askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_object_analytics.py · 9.8 KB · 197 lines
| 1 | """Analytics: traffic capture (object_analytics + the server hook), the |
| 2 | retention/rotation primitive (prune_collection_records), and the daemon pass. |
| 3 | """ |
| 4 | |
| 5 | import asyncio |
| 6 | from datetime import datetime, timedelta, timezone |
| 7 | from pathlib import Path |
| 8 | |
| 9 | import object_analytics |
| 10 | import object_daemon |
| 11 | import object_packages |
| 12 | import object_records |
| 13 | import object_server |
| 14 | |
| 15 | PACKAGES_ROOT = Path(__file__).resolve().parents[1] / "packages" |
| 16 | |
| 17 | |
| 18 | # ---- pure engine ---------------------------------------------------------- |
| 19 | |
| 20 | def test_config_from_env(): |
| 21 | assert object_analytics.analytics_enabled({"DBBASIC_ANALYTICS": "on"}) |
| 22 | assert object_analytics.analytics_enabled({"DBBASIC_ANALYTICS": "1"}) |
| 23 | assert not object_analytics.analytics_enabled({}) |
| 24 | assert not object_analytics.analytics_enabled({"DBBASIC_ANALYTICS": "off"}) |
| 25 | assert object_analytics.owner_ips({"DBBASIC_ANALYTICS_OWNER_IPS": "1.1.1.1, 2.2.2.2"}) == {"1.1.1.1", "2.2.2.2"} |
| 26 | assert object_analytics.retention_days({"DBBASIC_ANALYTICS_RETENTION_DAYS": "7"}) == 7 |
| 27 | assert object_analytics.retention_days({}) == 30 |
| 28 | assert object_analytics.retention_days({"DBBASIC_ANALYTICS_RETENTION_DAYS": "junk"}) == 30 |
| 29 | |
| 30 | |
| 31 | def test_should_capture_skips_assets_but_keeps_api_and_errors(): |
| 32 | assert object_analytics.should_capture("/dashboard") |
| 33 | assert object_analytics.should_capture("/api/mcp") # bots hit APIs -- capture |
| 34 | assert object_analytics.should_capture("/some/missing") # 404 targets captured (status is separate) |
| 35 | assert not object_analytics.should_capture("/static/app.css") |
| 36 | assert not object_analytics.should_capture("/favicon.ico") |
| 37 | assert not object_analytics.should_capture("/healthz") |
| 38 | assert not object_analytics.should_capture("/realtime/stream") |
| 39 | |
| 40 | |
| 41 | def test_build_page_view_extracts_request_fields(): |
| 42 | row = object_analytics.build_page_view( |
| 43 | path="/pricing", method="get", status=200, ip="9.9.9.9", |
| 44 | headers={"user-agent": "curl/8", "referer": "https://x.com/", "cookie": "a=1; session_id=sess42"}, |
| 45 | owners=frozenset({"9.9.9.9"}), |
| 46 | ) |
| 47 | assert row["path"] == "/pricing" and row["method"] == "GET" and row["status"] == "200" |
| 48 | assert row["ip"] == "9.9.9.9" and row["user_agent"] == "curl/8" |
| 49 | assert row["referrer"] == "https://x.com/" and row["session_id"] == "sess42" |
| 50 | assert row["is_owner"] == "true" # ip is in owners |
| 51 | # a non-owner ip |
| 52 | row2 = object_analytics.build_page_view( |
| 53 | path="/", method="GET", status=404, ip="5.5.5.5", headers={}, owners=frozenset({"9.9.9.9"})) |
| 54 | assert row2["is_owner"] == "false" and row2["user_agent"] == "" and row2["session_id"] == "" |
| 55 | |
| 56 | |
| 57 | # ---- retention primitive (prune) ------------------------------------------ |
| 58 | |
| 59 | def _install(tmp_path): |
| 60 | data_dir = tmp_path / "data" |
| 61 | object_root = tmp_path / "objects" |
| 62 | object_root.mkdir() |
| 63 | for pkg in ("app-views", "app-rollup", "app-analytics"): |
| 64 | object_packages.install_package(pkg, root=PACKAGES_ROOT, base_dir=data_dir, |
| 65 | object_roots=[object_root], allow_replace=True) |
| 66 | return data_dir |
| 67 | |
| 68 | |
| 69 | def _append(data_dir, path, created_at): |
| 70 | # created_at is read_only, so set it explicitly with preserve_read_only |
| 71 | return object_records.create_collection_record( |
| 72 | "page_views", {"path": path, "method": "GET", "status": "200", "created_at": created_at}, |
| 73 | base_dir=data_dir, actor="t", preserve_read_only=True) |
| 74 | |
| 75 | |
| 76 | def test_prune_drops_old_keeps_recent_and_undateable(tmp_path): |
| 77 | data_dir = _install(tmp_path) |
| 78 | _append(data_dir, "/old", "2026-01-01T00:00:00Z") |
| 79 | _append(data_dir, "/recent", "2026-07-20T00:00:00Z") |
| 80 | _append(data_dir, "/undateable", "") # no timestamp -> never dropped |
| 81 | |
| 82 | result = object_records.prune_collection_records( |
| 83 | "page_views", keep_newer_than="2026-06-01T00:00:00Z", base_dir=data_dir) |
| 84 | assert result["pruned"] and result["removed"] == 1 |
| 85 | paths = {r["path"] for r in object_records.read_collection_records("page_views", base_dir=data_dir)} |
| 86 | assert paths == {"/recent", "/undateable"} # /old aged out |
| 87 | |
| 88 | # idempotent: nothing older than the cutoff remains |
| 89 | again = object_records.prune_collection_records( |
| 90 | "page_views", keep_newer_than="2026-06-01T00:00:00Z", base_dir=data_dir) |
| 91 | assert again["pruned"] is False and again["removed"] == 0 |
| 92 | |
| 93 | |
| 94 | # ---- daemon retention pass ------------------------------------------------ |
| 95 | |
| 96 | def test_retention_pass_gated_and_prunes(tmp_path, monkeypatch): |
| 97 | data_dir = _install(tmp_path) |
| 98 | old = (datetime.now(timezone.utc) - timedelta(days=90)).isoformat().replace("+00:00", "Z") |
| 99 | fresh = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat().replace("+00:00", "Z") |
| 100 | _append(data_dir, "/old", old) |
| 101 | _append(data_dir, "/fresh", fresh) |
| 102 | |
| 103 | # disabled -> no-op |
| 104 | monkeypatch.delenv("DBBASIC_ANALYTICS", raising=False) |
| 105 | assert object_daemon.process_analytics_retention(base_dir=data_dir) is None |
| 106 | |
| 107 | # enabled, 30d retention -> the 90d-old row is aged out, the 1d row stays |
| 108 | monkeypatch.setenv("DBBASIC_ANALYTICS", "on") |
| 109 | monkeypatch.setenv("DBBASIC_ANALYTICS_RETENTION_DAYS", "30") |
| 110 | result = object_daemon.process_analytics_retention(base_dir=data_dir) |
| 111 | assert result and result["removed"] == 1 |
| 112 | paths = {r["path"] for r in object_records.read_collection_records("page_views", base_dir=data_dir)} |
| 113 | assert paths == {"/fresh"} |
| 114 | |
| 115 | # marker-gated: an immediate re-run is skipped (returns None, not a re-prune) |
| 116 | assert object_daemon.process_analytics_retention(base_dir=data_dir) is None |
| 117 | |
| 118 | |
| 119 | # ---- the server capture hook ---------------------------------------------- |
| 120 | |
| 121 | def test_capture_hook_appends_a_row_when_enabled(tmp_path, monkeypatch): |
| 122 | data_dir = _install(tmp_path) |
| 123 | monkeypatch.setenv(object_server.DATA_DIR_ENV, str(data_dir)) |
| 124 | monkeypatch.setenv("DBBASIC_ANALYTICS", "on") |
| 125 | scope = {"client": ("8.8.8.8", 4321)} |
| 126 | headers = {"user-agent": "Mozilla", "referer": "https://ref/", "cookie": "session_id=s1"} |
| 127 | |
| 128 | asyncio.run(object_server._capture_page_view(scope, "GET", "/pricing", 200, headers)) |
| 129 | rows = object_records.read_collection_records("page_views", base_dir=data_dir) |
| 130 | assert len(rows) == 1 and rows[0]["path"] == "/pricing" and rows[0]["ip"] == "8.8.8.8" |
| 131 | assert rows[0]["session_id"] == "s1" and rows[0]["created_at"] |
| 132 | |
| 133 | # a skip-path is not captured |
| 134 | asyncio.run(object_server._capture_page_view(scope, "GET", "/static/x.js", 200, headers)) |
| 135 | assert len(object_records.read_collection_records("page_views", base_dir=data_dir)) == 1 |
| 136 | |
| 137 | |
| 138 | def test_rollups_aggregate_traffic_and_exclude_owner(tmp_path): |
| 139 | data_dir = tmp_path / "data" |
| 140 | object_root = tmp_path / "objects" |
| 141 | object_root.mkdir() |
| 142 | for pkg in ("app-views", "app-rollup", "app-analytics"): |
| 143 | object_packages.install_package(pkg, root=PACKAGES_ROOT, base_dir=data_dir, |
| 144 | object_roots=[object_root], allow_replace=True) |
| 145 | # the reporting rollups seeded cleanly into the shared rollup_definitions |
| 146 | defs = {d["id"] for d in object_records.read_collection_records("rollup_definitions", base_dir=data_dir)} |
| 147 | assert {"ana_top_paths", "ana_top_ips", "ana_status", "ana_daily"} <= defs |
| 148 | |
| 149 | def hit(path, ip, status, owner=False): |
| 150 | object_records.create_collection_record( |
| 151 | "page_views", |
| 152 | {"path": path, "method": "GET", "status": str(status), "ip": ip, |
| 153 | "is_owner": "true" if owner else "false"}, |
| 154 | base_dir=data_dir, actor="t") |
| 155 | |
| 156 | for _ in range(3): |
| 157 | hit("/a", "6.6.6.6", 200) # a "bot" hammering /a from one IP |
| 158 | hit("/b", "1.2.3.4", 404) # a 404 |
| 159 | hit("/admin", "9.9.9.9", 200, owner=True) # owner traffic -> excluded |
| 160 | |
| 161 | result = object_daemon.process_rollups(base_dir=data_dir) |
| 162 | assert result is not None |
| 163 | |
| 164 | top_paths = {r["path"]: int(r["hits"]) for r in object_records.read_collection_records("analytics_top_paths", base_dir=data_dir)} |
| 165 | assert top_paths == {"/a": 3, "/b": 1} # /admin excluded (is_owner) |
| 166 | top_ips = {r["ip"]: int(r["hits"]) for r in object_records.read_collection_records("analytics_top_ips", base_dir=data_dir)} |
| 167 | assert top_ips.get("6.6.6.6") == 3 # the bot is visible, ranked |
| 168 | status = {r["status"]: int(r["hits"]) for r in object_records.read_collection_records("analytics_status", base_dir=data_dir)} |
| 169 | assert status.get("404") == 1 and status.get("200") == 3 |
| 170 | |
| 171 | |
| 172 | def test_conversions_collection_and_analytics_view_seeded(tmp_path): |
| 173 | import json |
| 174 | data_dir = _install(tmp_path) |
| 175 | # conversions collection installs and records a goal event (app-driven write) |
| 176 | conv = object_records.create_collection_record( |
| 177 | "conversions", {"event_type": "signup", "session_id": "s1", "metadata": json.dumps({"plan": "pro"})}, |
| 178 | base_dir=data_dir, actor="app") |
| 179 | assert conv["event_type"] == "signup" |
| 180 | assert object_records.read_collection_records("conversions", base_dir=data_dir)[0]["event_type"] == "signup" |
| 181 | |
| 182 | # the /analytics generative view + route seeded |
| 183 | views = {v["id"]: v for v in object_records.read_collection_records("views", base_dir=data_dir)} |
| 184 | assert "view_analytics" in views and views["view_analytics"]["route"] == "/analytics" |
| 185 | blocks = json.loads(views["view_analytics"]["blocks"]) |
| 186 | listed = {b["collection"] for b in blocks if b.get("kind") == "list"} |
| 187 | assert listed == {"analytics_top_ips", "analytics_top_paths", "analytics_status", "analytics_daily"} |
| 188 | routes = {r["pattern"]: r["object_id"] for r in object_records.read_collection_records("site_routes", base_dir=data_dir)} |
| 189 | assert routes.get("/analytics") == "site_view_render" |
| 190 | |
| 191 | |
| 192 | def test_capture_hook_noop_when_disabled(tmp_path, monkeypatch): |
| 193 | data_dir = _install(tmp_path) |
| 194 | monkeypatch.setenv(object_server.DATA_DIR_ENV, str(data_dir)) |
| 195 | monkeypatch.delenv("DBBASIC_ANALYTICS", raising=False) |
| 196 | asyncio.run(object_server._capture_page_view({"client": ("1.1.1.1", 1)}, "GET", "/", 200, {})) |
| 197 | assert object_records.read_collection_records("page_views", base_dir=data_dir) == [] |