Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_visitors.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_visitors.py · 12.0 KB · 296 lines
1"""Who actually turned up: visitors, bots, and us, counted apart.
2
3The pure half of visitor analytics -- one pass over page_views rows,
4no data directory, no I/O.
5
6The question this exists to answer is "I talked about the thing today,
7did anyone come?", and a raw unique-IP count answers it badly. On this
8server 45% of distinct paths were hit exactly once, by vulnerability
9scanners probing for /wp-login.php and /index1.php from a fresh address
10each time. Counting those as visitors turns a quiet day into a busy one
11and makes the number useless precisely when somebody is relying on it.
12
13So traffic is classified into three kinds and never silently merged:
14
15**operator** -- our own. The `is_owner` flag already marks owner IPs and
16admin-token requests, so deploys and scripts stop inflating the count.
17Shown, not hidden: seeing your own hits is how you sanity-check that the
18page is recording anything at all.
19
20**bot** -- a client that never behaved like a person. The test is
21behavioural rather than a user-agent blocklist, because the blocklist is
22always out of date and the behaviour is not: a scanner walks a list of
23URLs that do not exist here and collects 404s.
24
25A successful page load alone is NOT enough, and that correction came from
26real data: `/` returns 200 to everyone, so the first version of this
27promoted every prober that touched the homepage into the visitor column.
28An address must also have loaded a second distinct page, arrived from a
29referrer, or carried a session cookie -- any one of which a person does
30and a prober does not.
31
32A declared crawler stays a bot even when it fetches something real:
33Googlebot is honest about what it is, and honesty should not promote it.
34So does anything sending an attack payload in a header -- one arrived
35here as a Log4Shell probe with an AWS-credential exfiltration template in
36the Referer, and was counted as a visitor because it had also loaded the
37front page.
38
39**visitor** -- everything else. A human, probably.
40
41"Unique" means distinct IP, and that is an approximation worth stating
42out loud rather than dressing up: an office behind one NAT is one
43visitor, a phone moving between wifi and cellular is two. It stays the
44unit HERE even though there is now a first-party visitor cookie
45(object_analytics), and that is a deliberate split rather than an
46oversight: this fold classifies BEHAVIOUR, and behaviour belongs to an
47address -- a scanner does not accept cookies, so classifying by token
48would put every bot in one bucket called "no token" and make the
49visitor/bot line unanswerable. The cookie answers a different question,
50new versus returning, and object_conversions owns that fold. The
51alternative to both -- a cross-site identifier, or a fingerprint -- is a
52thing this system deliberately does not have.
53"""
54
55from __future__ import annotations
56
57from collections import defaultdict
58from datetime import datetime, timedelta, timezone
59from typing import Any, Iterable
60
61# Clients that say what they are. Matched case-insensitively against a
62# substring of the user agent -- deliberately short, because this list is
63# a courtesy to honest crawlers, not the mechanism that catches scanners.
64# The behavioural test below is what does that work.
65_DECLARED_BOTS = (
66 "bot", "crawler", "spider", "slurp", "curl/", "wget", "python-requests",
67 "httpx", "go-http-client", "scrapy", "headlesschrome", "monitoring",
68 "uptime", "pingdom", "semrush", "ahrefs", "facebookexternalhit",
69)
70
71# Paths that a person does not "visit" -- fetched by a page, not chosen by
72# a human. They still count as traffic; they just cannot be the ONLY thing
73# an address did and still make it a visitor.
74_NON_PAGE_PREFIXES = ("/api/", "/collections/", "/objects/", "/style", "/nav")
75
76# Attack payloads seen in a header. Not a defence -- none of these work
77# against this server, and the JNDI one is a Java vulnerability probing a
78# Python process -- but a client that sends one is definitively not a
79# person, and saying so is what keeps it out of the visitor column. The
80# real example that prompted this arrived in a Referer header:
81# ${jndi:ldap://…}AWS_SECRET_ACCESS_KEY=${env:AWS_SECRET_ACCESS_KEY:-}…
82# and was counted as a visitor, because it had also loaded the homepage.
83_HOSTILE_MARKERS = (
84 "${jndi:", "${env:", "../../", "..%2f", "<script", "union select",
85 "' or '1'='1", "/etc/passwd", "%00", "{{", "$(", "`",
86)
87
88
89def hostile(*parts: str) -> bool:
90 """True when any header carries an attack payload."""
91 haystack = " ".join(_text(part) for part in parts).lower()
92 return any(marker in haystack for marker in _HOSTILE_MARKERS)
93
94
95OPERATOR, BOT, VISITOR = "operator", "bot", "visitor"
96
97
98def _text(value: Any) -> str:
99 return str(value if value is not None else "").strip()
100
101
102def _truthy(value: Any) -> bool:
103 return _text(value).lower() in ("true", "1", "yes", "on")
104
105
106def _status(value: Any) -> int:
107 try:
108 return int(_text(value) or 0)
109 except (TypeError, ValueError):
110 return 0
111
112
113def is_page_path(path: str) -> bool:
114 """A path a person could have chosen to visit."""
115 clean = _text(path) or "/"
116 return not clean.startswith(_NON_PAGE_PREFIXES)
117
118
119def declared_bot(user_agent: str) -> bool:
120 lowered = _text(user_agent).lower()
121 if not lowered:
122 # No user agent at all is a script, not a browser. Every real
123 # browser sends one.
124 return True
125 return any(token in lowered for token in _DECLARED_BOTS)
126
127
128def classify(rows: Iterable[dict]) -> dict[str, str]:
129 """Decide what each IP was, from everything it did.
130
131 Classification is per ADDRESS rather than per request, because the
132 question is "how many of these were people" and a person generates
133 many rows.
134
135 A successful page load is necessary but NOT sufficient, and that
136 correction came from the data: `/` returns 200 to everyone, so the
137 first version promoted every scanner that touched the homepage into
138 the visitor column. On a real week that was a large share of the
139 "direct traffic" figure. So an address also has to have done one
140 thing a prober does not:
141
142 * loaded MORE THAN ONE distinct page, or
143 * arrived from somewhere (a referrer), or
144 * carried a session cookie.
145
146 A person browsing does at least one of those; something hitting root
147 and leaving does none. The rule is deliberately generous -- one real
148 page plus any one signal -- because over-counting a shy visitor is a
149 smaller error than reporting a busy day that never happened.
150 """
151 seen_pages: dict[str, set[str]] = defaultdict(set)
152 signalled: set[str] = set()
153 declared: set[str] = set()
154 operator: set[str] = set()
155 everyone: set[str] = set()
156
157 for row in rows:
158 ip = _text(row.get("ip"))
159 if not ip:
160 continue
161 everyone.add(ip)
162 if _truthy(row.get("is_owner")):
163 operator.add(ip)
164 referrer = _text(row.get("referrer"))
165 if (declared_bot(row.get("user_agent"))
166 or hostile(referrer, row.get("user_agent"), row.get("path"))):
167 declared.add(ip)
168 if (200 <= _status(row.get("status")) < 400
169 and is_page_path(row.get("path"))):
170 seen_pages[ip].add(_text(row.get("path")) or "/")
171 if referrer or _text(row.get("session_id")):
172 signalled.add(ip)
173
174 kinds: dict[str, str] = {}
175 for ip in everyone:
176 pages = seen_pages.get(ip, set())
177 looks_human = bool(pages) and (len(pages) > 1 or ip in signalled)
178 if ip in operator:
179 kinds[ip] = OPERATOR
180 elif ip in declared or not looks_human:
181 kinds[ip] = BOT
182 else:
183 kinds[ip] = VISITOR
184 return kinds
185
186
187def _bucket(stamp: str, *, hourly: bool) -> str:
188 text = _text(stamp)
189 if len(text) < 10:
190 return ""
191 return text[:13].replace("T", " ") if hourly else text[:10]
192
193
194def summarize(
195 rows: Iterable[dict],
196 *,
197 now: datetime | None = None,
198 days: int = 7,
199 host: str = "",
200) -> dict[str, Any]:
201 """Visitors, views and bots -- hourly for today, daily for the window.
202
203 Returns {"today", "days", "hours", "totals", "referrers", "hosts", ...}.
204 Counted in ONE pass with sets per bucket: the alternative, a distinct
205 count per bucket computed by re-scanning, is the shape that made a
206 rollup rewrite 1818 rows every five minutes on this very server.
207
208 `host` narrows to one site. One process here answers for a marketing
209 domain and the app domain both, and a combined figure answers neither
210 "did anyone read the pitch" nor "did anyone try the product". The
211 per-host split is always returned even when not filtering, because
212 the first useful thing to know is which site they went to.
213 """
214 rows = list(rows)
215 if host:
216 rows = [row for row in rows if _text(row.get("host")) == host]
217 kinds = classify(rows)
218 now = now or datetime.now(timezone.utc)
219 today = now.strftime("%Y-%m-%d")
220 earliest = (now - timedelta(days=days - 1)).strftime("%Y-%m-%d")
221
222 views: dict[tuple[str, str], int] = defaultdict(int)
223 uniques: dict[tuple[str, str], set[str]] = defaultdict(set)
224 hour_views: dict[tuple[str, str], int] = defaultdict(int)
225 hour_uniques: dict[tuple[str, str], set[str]] = defaultdict(set)
226 referrers: dict[str, set[str]] = defaultdict(set)
227 landing: dict[str, set[str]] = defaultdict(set)
228 hosts: dict[str, set[str]] = defaultdict(set)
229
230 for row in rows:
231 ip = _text(row.get("ip"))
232 stamp = _text(row.get("created_at"))
233 day = _bucket(stamp, hourly=False)
234 if not ip or not day or day < earliest:
235 continue
236 kind = kinds.get(ip, VISITOR)
237
238 views[(day, kind)] += 1
239 uniques[(day, kind)].add(ip)
240
241 if day == today:
242 hour = _bucket(stamp, hourly=True)
243 hour_views[(hour, kind)] += 1
244 hour_uniques[(hour, kind)].add(ip)
245
246 if kind != VISITOR:
247 continue
248 hosts[_text(row.get("host")) or "(not recorded)"].add(ip)
249 referrer = _text(row.get("referrer"))
250 # An empty referrer is "typed it, or came from a private link" --
251 # worth showing as its own row rather than dropped, because for a
252 # server nobody has linked to yet it is the whole story.
253 referrers[referrer or "(direct or private link)"].add(ip)
254 if is_page_path(row.get("path")) and 200 <= _status(row.get("status")) < 400:
255 landing[_text(row.get("path")) or "/"].add(ip)
256
257 def _series(bucket_views, bucket_uniques, keys):
258 return [{
259 "bucket": key,
260 "visitors": len(bucket_uniques.get((key, VISITOR), ())),
261 "views": bucket_views.get((key, VISITOR), 0),
262 "bots": len(bucket_uniques.get((key, BOT), ())),
263 "bot_views": bucket_views.get((key, BOT), 0),
264 "operator_views": bucket_views.get((key, OPERATOR), 0),
265 } for key in keys]
266
267 day_keys = [(now - timedelta(days=offset)).strftime("%Y-%m-%d")
268 for offset in range(days - 1, -1, -1)]
269 hour_keys = [f"{today} {hour:02d}" for hour in range(24)]
270
271 totals: dict[str, Any] = {}
272 for kind in (VISITOR, BOT, OPERATOR):
273 ips = {ip for ip, k in kinds.items() if k == kind}
274 totals[kind] = {
275 "unique": len(ips),
276 "views": sum(count for (day, k), count in views.items()
277 if k == kind and day >= earliest),
278 }
279
280 return {
281 "today": today,
282 "days": _series(views, uniques, day_keys),
283 "hours": _series(hour_views, hour_uniques, hour_keys),
284 "totals": totals,
285 "referrers": sorted(
286 ({"referrer": ref, "visitors": len(ips)} for ref, ips in referrers.items()),
287 key=lambda row: (-row["visitors"], row["referrer"]))[:12],
288 "hosts": sorted(
289 ({"host": name, "visitors": len(ips)} for name, ips in hosts.items()),
290 key=lambda row: (-row["visitors"], row["host"])),
291 "host": host,
292 "landing": sorted(
293 ({"path": path, "visitors": len(ips)} for path, ips in landing.items()),
294 key=lambda row: (-row["visitors"], row["path"]))[:12],
295 "window_days": days,
296 }