Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_object_analytics.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_object_analytics.py · 9.8 KB · 197 lines
1"""Analytics: traffic capture (object_analytics + the server hook), the
2retention/rotation primitive (prune_collection_records), and the daemon pass.
3"""
4
5import asyncio
6from datetime import datetime, timedelta, timezone
7from pathlib import Path
8
9import object_analytics
10import object_daemon
11import object_packages
12import object_records
13import object_server
14
15PACKAGES_ROOT = Path(__file__).resolve().parents[1] / "packages"
16
17
18# ---- pure engine ----------------------------------------------------------
19
20def test_config_from_env():
21 assert object_analytics.analytics_enabled({"DBBASIC_ANALYTICS": "on"})
22 assert object_analytics.analytics_enabled({"DBBASIC_ANALYTICS": "1"})
23 assert not object_analytics.analytics_enabled({})
24 assert not object_analytics.analytics_enabled({"DBBASIC_ANALYTICS": "off"})
25 assert object_analytics.owner_ips({"DBBASIC_ANALYTICS_OWNER_IPS": "1.1.1.1, 2.2.2.2"}) == {"1.1.1.1", "2.2.2.2"}
26 assert object_analytics.retention_days({"DBBASIC_ANALYTICS_RETENTION_DAYS": "7"}) == 7
27 assert object_analytics.retention_days({}) == 30
28 assert object_analytics.retention_days({"DBBASIC_ANALYTICS_RETENTION_DAYS": "junk"}) == 30
29
30
31def test_should_capture_skips_assets_but_keeps_api_and_errors():
32 assert object_analytics.should_capture("/dashboard")
33 assert object_analytics.should_capture("/api/mcp") # bots hit APIs -- capture
34 assert object_analytics.should_capture("/some/missing") # 404 targets captured (status is separate)
35 assert not object_analytics.should_capture("/static/app.css")
36 assert not object_analytics.should_capture("/favicon.ico")
37 assert not object_analytics.should_capture("/healthz")
38 assert not object_analytics.should_capture("/realtime/stream")
39
40
41def test_build_page_view_extracts_request_fields():
42 row = object_analytics.build_page_view(
43 path="/pricing", method="get", status=200, ip="9.9.9.9",
44 headers={"user-agent": "curl/8", "referer": "https://x.com/", "cookie": "a=1; session_id=sess42"},
45 owners=frozenset({"9.9.9.9"}),
46 )
47 assert row["path"] == "/pricing" and row["method"] == "GET" and row["status"] == "200"
48 assert row["ip"] == "9.9.9.9" and row["user_agent"] == "curl/8"
49 assert row["referrer"] == "https://x.com/" and row["session_id"] == "sess42"
50 assert row["is_owner"] == "true" # ip is in owners
51 # a non-owner ip
52 row2 = object_analytics.build_page_view(
53 path="/", method="GET", status=404, ip="5.5.5.5", headers={}, owners=frozenset({"9.9.9.9"}))
54 assert row2["is_owner"] == "false" and row2["user_agent"] == "" and row2["session_id"] == ""
55
56
57# ---- retention primitive (prune) ------------------------------------------
58
59def _install(tmp_path):
60 data_dir = tmp_path / "data"
61 object_root = tmp_path / "objects"
62 object_root.mkdir()
63 for pkg in ("app-views", "app-rollup", "app-analytics"):
64 object_packages.install_package(pkg, root=PACKAGES_ROOT, base_dir=data_dir,
65 object_roots=[object_root], allow_replace=True)
66 return data_dir
67
68
69def _append(data_dir, path, created_at):
70 # created_at is read_only, so set it explicitly with preserve_read_only
71 return object_records.create_collection_record(
72 "page_views", {"path": path, "method": "GET", "status": "200", "created_at": created_at},
73 base_dir=data_dir, actor="t", preserve_read_only=True)
74
75
76def test_prune_drops_old_keeps_recent_and_undateable(tmp_path):
77 data_dir = _install(tmp_path)
78 _append(data_dir, "/old", "2026-01-01T00:00:00Z")
79 _append(data_dir, "/recent", "2026-07-20T00:00:00Z")
80 _append(data_dir, "/undateable", "") # no timestamp -> never dropped
81
82 result = object_records.prune_collection_records(
83 "page_views", keep_newer_than="2026-06-01T00:00:00Z", base_dir=data_dir)
84 assert result["pruned"] and result["removed"] == 1
85 paths = {r["path"] for r in object_records.read_collection_records("page_views", base_dir=data_dir)}
86 assert paths == {"/recent", "/undateable"} # /old aged out
87
88 # idempotent: nothing older than the cutoff remains
89 again = object_records.prune_collection_records(
90 "page_views", keep_newer_than="2026-06-01T00:00:00Z", base_dir=data_dir)
91 assert again["pruned"] is False and again["removed"] == 0
92
93
94# ---- daemon retention pass ------------------------------------------------
95
96def test_retention_pass_gated_and_prunes(tmp_path, monkeypatch):
97 data_dir = _install(tmp_path)
98 old = (datetime.now(timezone.utc) - timedelta(days=90)).isoformat().replace("+00:00", "Z")
99 fresh = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat().replace("+00:00", "Z")
100 _append(data_dir, "/old", old)
101 _append(data_dir, "/fresh", fresh)
102
103 # disabled -> no-op
104 monkeypatch.delenv("DBBASIC_ANALYTICS", raising=False)
105 assert object_daemon.process_analytics_retention(base_dir=data_dir) is None
106
107 # enabled, 30d retention -> the 90d-old row is aged out, the 1d row stays
108 monkeypatch.setenv("DBBASIC_ANALYTICS", "on")
109 monkeypatch.setenv("DBBASIC_ANALYTICS_RETENTION_DAYS", "30")
110 result = object_daemon.process_analytics_retention(base_dir=data_dir)
111 assert result and result["removed"] == 1
112 paths = {r["path"] for r in object_records.read_collection_records("page_views", base_dir=data_dir)}
113 assert paths == {"/fresh"}
114
115 # marker-gated: an immediate re-run is skipped (returns None, not a re-prune)
116 assert object_daemon.process_analytics_retention(base_dir=data_dir) is None
117
118
119# ---- the server capture hook ----------------------------------------------
120
121def test_capture_hook_appends_a_row_when_enabled(tmp_path, monkeypatch):
122 data_dir = _install(tmp_path)
123 monkeypatch.setenv(object_server.DATA_DIR_ENV, str(data_dir))
124 monkeypatch.setenv("DBBASIC_ANALYTICS", "on")
125 scope = {"client": ("8.8.8.8", 4321)}
126 headers = {"user-agent": "Mozilla", "referer": "https://ref/", "cookie": "session_id=s1"}
127
128 asyncio.run(object_server._capture_page_view(scope, "GET", "/pricing", 200, headers))
129 rows = object_records.read_collection_records("page_views", base_dir=data_dir)
130 assert len(rows) == 1 and rows[0]["path"] == "/pricing" and rows[0]["ip"] == "8.8.8.8"
131 assert rows[0]["session_id"] == "s1" and rows[0]["created_at"]
132
133 # a skip-path is not captured
134 asyncio.run(object_server._capture_page_view(scope, "GET", "/static/x.js", 200, headers))
135 assert len(object_records.read_collection_records("page_views", base_dir=data_dir)) == 1
136
137
138def test_rollups_aggregate_traffic_and_exclude_owner(tmp_path):
139 data_dir = tmp_path / "data"
140 object_root = tmp_path / "objects"
141 object_root.mkdir()
142 for pkg in ("app-views", "app-rollup", "app-analytics"):
143 object_packages.install_package(pkg, root=PACKAGES_ROOT, base_dir=data_dir,
144 object_roots=[object_root], allow_replace=True)
145 # the reporting rollups seeded cleanly into the shared rollup_definitions
146 defs = {d["id"] for d in object_records.read_collection_records("rollup_definitions", base_dir=data_dir)}
147 assert {"ana_top_paths", "ana_top_ips", "ana_status", "ana_daily"} <= defs
148
149 def hit(path, ip, status, owner=False):
150 object_records.create_collection_record(
151 "page_views",
152 {"path": path, "method": "GET", "status": str(status), "ip": ip,
153 "is_owner": "true" if owner else "false"},
154 base_dir=data_dir, actor="t")
155
156 for _ in range(3):
157 hit("/a", "6.6.6.6", 200) # a "bot" hammering /a from one IP
158 hit("/b", "1.2.3.4", 404) # a 404
159 hit("/admin", "9.9.9.9", 200, owner=True) # owner traffic -> excluded
160
161 result = object_daemon.process_rollups(base_dir=data_dir)
162 assert result is not None
163
164 top_paths = {r["path"]: int(r["hits"]) for r in object_records.read_collection_records("analytics_top_paths", base_dir=data_dir)}
165 assert top_paths == {"/a": 3, "/b": 1} # /admin excluded (is_owner)
166 top_ips = {r["ip"]: int(r["hits"]) for r in object_records.read_collection_records("analytics_top_ips", base_dir=data_dir)}
167 assert top_ips.get("6.6.6.6") == 3 # the bot is visible, ranked
168 status = {r["status"]: int(r["hits"]) for r in object_records.read_collection_records("analytics_status", base_dir=data_dir)}
169 assert status.get("404") == 1 and status.get("200") == 3
170
171
172def test_conversions_collection_and_analytics_view_seeded(tmp_path):
173 import json
174 data_dir = _install(tmp_path)
175 # conversions collection installs and records a goal event (app-driven write)
176 conv = object_records.create_collection_record(
177 "conversions", {"event_type": "signup", "session_id": "s1", "metadata": json.dumps({"plan": "pro"})},
178 base_dir=data_dir, actor="app")
179 assert conv["event_type"] == "signup"
180 assert object_records.read_collection_records("conversions", base_dir=data_dir)[0]["event_type"] == "signup"
181
182 # the /analytics generative view + route seeded
183 views = {v["id"]: v for v in object_records.read_collection_records("views", base_dir=data_dir)}
184 assert "view_analytics" in views and views["view_analytics"]["route"] == "/analytics"
185 blocks = json.loads(views["view_analytics"]["blocks"])
186 listed = {b["collection"] for b in blocks if b.get("kind") == "list"}
187 assert listed == {"analytics_top_ips", "analytics_top_paths", "analytics_status", "analytics_daily"}
188 routes = {r["pattern"]: r["object_id"] for r in object_records.read_collection_records("site_routes", base_dir=data_dir)}
189 assert routes.get("/analytics") == "site_view_render"
190
191
192def test_capture_hook_noop_when_disabled(tmp_path, monkeypatch):
193 data_dir = _install(tmp_path)
194 monkeypatch.setenv(object_server.DATA_DIR_ENV, str(data_dir))
195 monkeypatch.delenv("DBBASIC_ANALYTICS", raising=False)
196 asyncio.run(object_server._capture_page_view({"client": ("1.1.1.1", 1)}, "GET", "/", 200, {}))
197 assert object_records.read_collection_records("page_views", base_dir=data_dir) == []