Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_worker_liveness.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_worker_liveness.py · 12.0 KB · 279 lines
1"""Is this server still doing its own work?
2
3The agents board answered that for EXTERNAL agents and nobody was asking
4it of the box itself. The evidence was already on disk: every scheduled
5execution writes a scheduler_runs row (object_id, started_at, ok,
6duration_ms, error), so "did the runner pass run in the last minute" has
7been answerable all along -- just never surfaced.
8
9So this is a fold, not a new heartbeat, and that is the design decision
10worth defending. Adding a writer that stamped "the daemon is alive"
11beside a log that already proves it would be a second, WEAKER account of
12the same fact: a heartbeat says a process reached the line that writes
13heartbeats, while a scheduler_runs row says the actual work ran and
14whether it worked.
15"""
16
17import pytest
18
19import object_agents
20
21NOW = "2026-07-31T14:00:00Z"
22
23
24def run(object_id, started_at, *, ok="true", error="", duration_ms="10"):
25 return {"object_id": object_id, "started_at": started_at, "ok": ok,
26 "error": error, "duration_ms": duration_ms}
27
28
29def task(object_id, *, schedule="* * * * *", status="active"):
30 return {"object_id": object_id, "schedule": schedule, "status": status}
31
32
33def fold(runs, tasks=None, **kw):
34 return object_agents.worker_liveness(runs, tasks=tasks, now=NOW, **kw)
35
36
37def by_id(result, object_id):
38 return next(w for w in result["workers"] if w["object_id"] == object_id)
39
40
41def test_a_pass_that_just_ran_is_live():
42 result = fold([run("system_template_runner", "2026-07-31T13:59:30Z")])
43 assert by_id(result, "system_template_runner")["liveness"] == "live"
44 assert by_id(result, "system_template_runner")["last_run_ago"]
45
46
47def test_a_pass_that_stopped_hours_ago_is_lost():
48 result = fold([run("system_run_sweeper", "2026-07-31T09:00:00Z")])
49 assert by_id(result, "system_run_sweeper")["liveness"] == "lost"
50
51
52def test_only_the_most_recent_run_decides():
53 """The log is append-only and full of history; liveness is about the
54 newest row, not the first one found."""
55 result = fold([run("p", "2026-07-30T01:00:00Z"),
56 run("p", "2026-07-31T13:59:00Z"),
57 run("p", "2026-07-31T02:00:00Z")])
58 assert by_id(result, "p")["liveness"] == "live"
59
60
61def test_a_task_that_has_never_run_is_visible_at_all():
62 """THE case a fold over the run log cannot see by itself: an object
63 with no rows is an ABSENCE, and absences are invisible to a fold over
64 presences. A pass somebody scheduled that has quietly done nothing
65 since the day it was added is the failure most worth catching."""
66 result = fold([], tasks=[task("system_never_ran", schedule="0 4 * * *")])
67 assert by_id(result, "system_never_ran")["liveness"] == "never"
68 assert by_id(result, "system_never_ran")["schedule"] == "0 4 * * *"
69
70
71def test_a_task_an_operator_switched_off_is_not_a_problem():
72 """Same distinction liveness() draws for a paused agent: it is not
73 silent, it is off, and rendering it as lost manufactures an alarm out
74 of somebody's own decision."""
75 result = fold([run("p", "2026-07-30T01:00:00Z")],
76 tasks=[task("p", status="paused")])
77 assert by_id(result, "p")["liveness"] == "paused"
78
79
80def test_a_pass_that_ran_and_threw_is_live_AND_failing():
81 """The state a liveness badge alone would render as healthy, and it
82 is worse than lost: the daemon is fine, the work is broken."""
83 result = fold([run("p", "2026-07-31T13:59:00Z", ok="false",
84 error="boom")])
85 row = by_id(result, "p")
86 assert row["liveness"] == "live"
87 assert row["failing"] is True
88 assert row["last_error"] == "boom"
89 assert result["failing"] == ["p"]
90
91
92def test_an_error_string_alone_counts_as_failing():
93 """ok can be absent on older rows; an error message is evidence
94 enough and must not be shrugged off because a column was blank."""
95 row = by_id(fold([run("p", "2026-07-31T13:59:00Z", ok="", error="nope")]), "p")
96 assert row["failing"] is True
97
98
99def test_worst_comes_first_and_failures_outrank_healthy_peers():
100 """A board exists to surface what needs attention."""
101 result = fold(
102 [run("healthy", "2026-07-31T13:59:00Z"),
103 run("broken", "2026-07-31T13:59:00Z", ok="false", error="x"),
104 run("gone", "2026-07-31T01:00:00Z")],
105 tasks=[task("unstarted")])
106 order = [w["object_id"] for w in result["workers"]]
107 assert order.index("unstarted") < order.index("healthy")
108 assert order.index("gone") < order.index("healthy")
109 assert order.index("broken") < order.index("healthy")
110
111
112def test_the_declared_schedule_rides_along_for_context():
113 """"lost" means nothing without knowing it was supposed to run
114 minutely rather than yearly."""
115 result = fold([run("p", "2026-07-31T09:00:00Z")],
116 tasks=[task("p", schedule="*/10 * * * *")])
117 assert by_id(result, "p")["schedule"] == "*/10 * * * *"
118
119
120def test_counts_and_totals_summarize_the_board():
121 result = fold([run("a", "2026-07-31T13:59:00Z"),
122 run("b", "2026-07-31T01:00:00Z")],
123 tasks=[task("c")])
124 assert result["total"] == 3
125 assert result["live"] == 1
126 assert result["counts"]["never"] == 1
127
128
129def test_an_empty_box_folds_to_nothing_rather_than_erroring():
130 assert fold([], tasks=[]) == {"workers": [], "counts": {}, "live": 0,
131 "total": 0, "failing": []}
132
133
134def test_the_liveness_rule_is_the_SAME_one_agents_use():
135 """Two rules that could drift apart is how one board starts lying, so
136 a worker and an agent that have both been silent for the same
137 interval must get the same verdict."""
138 silent = "2026-07-31T13:40:00Z"
139 worker = by_id(fold([run("p", silent)]), "p")["liveness"]
140 agent = object_agents.liveness({"status": "active", "heartbeat_at": silent},
141 now=NOW)
142 assert worker == agent
143
144
145# --- timestamp forms: the bug that only real data had -------------------------
146
147def test_an_offset_aware_stamp_is_not_silently_aged():
148 """THE live bug, and the quiet half of it.
149
150 The board had only ever seen timestamps the heartbeat action writes,
151 which end in `Z`. scheduler_runs writes `+00:00`, and the module's
152 lexicographic ISO comparison is WRONG across the two forms: "+"
153 (0x2B) sorts below "Z" (0x5A), so the same instant in the offset form
154 reads as OLDER. A worker that had just run could be reported stale,
155 and no fixture using one format could ever catch it.
156 """
157 assert "2026-07-31T14:00:00+00:00" < "2026-07-31T14:00:00Z" # the hazard
158
159 just_ran = object_agents.liveness(
160 {"status": "active", "heartbeat_at": "2026-07-31T13:59:00+00:00"},
161 now=NOW)
162 assert just_ran == "live"
163
164
165def test_the_two_forms_of_the_same_instant_agree():
166 """Agreement is the property, whatever the verdict happens to be --
167 30 minutes past a 15-minute window is legitimately `stale`, and both
168 spellings of that instant must say so identically."""
169 verdicts = {object_agents.liveness(
170 {"status": "active", "heartbeat_at": stamp}, now=NOW)
171 for stamp in ("2026-07-31T13:30:00Z", "2026-07-31T13:30:00+00:00")}
172 assert verdicts == {"stale"}
173
174 fresh = {object_agents.liveness(
175 {"status": "active", "heartbeat_at": stamp}, now=NOW)
176 for stamp in ("2026-07-31T13:59:00Z", "2026-07-31T13:59:00+00:00")}
177 assert fresh == {"live"}
178
179
180def test_a_real_scheduler_runs_stamp_folds(tmp_path=None):
181 """Copied verbatim off the live box, microseconds and all -- the
182 shape that produced 'can't subtract offset-naive and offset-aware
183 datetimes' on a page that had passed every test."""
184 result = fold([run("system_invoice_aging", "2026-07-31T13:59:27.944184+00:00")])
185 row = by_id(result, "system_invoice_aging")
186 assert row["liveness"] == "live"
187 assert row["last_run_ago"].endswith("ago") or row["last_run_ago"] == "just now"
188
189
190def test_an_odd_fractional_precision_still_parses():
191 """Python 3.10's fromisoformat takes only 3 or 6 fractional digits,
192 and writers emit whatever they emit. The droplet runs 3.12 and is
193 lenient; trusting that is how a page works in production and breaks
194 on a developer's machine."""
195 assert object_agents.relative_time("2026-07-31T13:00:00.9+00:00", NOW) \
196 == "1 hour ago" or "minutes ago" in object_agents.relative_time(
197 "2026-07-31T13:00:00.9+00:00", NOW)
198
199
200def test_an_unparseable_stamp_degrades_rather_than_raising():
201 assert object_agents.relative_time("not a date", NOW) == "not a date"
202 assert object_agents.liveness(
203 {"status": "active", "heartbeat_at": "not a date"}, now=NOW) == "never"
204
205
206# --- silence only means something against a schedule --------------------------
207
208def test_a_daily_pass_seven_hours_quiet_is_not_lost():
209 """THE false alarm the live board produced on its first deploy: ten
210 of fourteen passes reported `lost` for the crime of being daily. A
211 cron that fires at 06:55 and last ran seven hours ago is not lost, it
212 is waiting -- and a board that says otherwise is one an operator
213 learns to ignore."""
214 result = fold([run("nightly", "2026-07-31T06:55:00Z")],
215 tasks=[task("nightly", schedule="55 6 * * *")])
216 assert by_id(result, "nightly")["liveness"] == "live"
217
218
219def test_a_minutely_pass_ten_minutes_quiet_IS_a_problem():
220 """The same silence, the opposite verdict. The schedule is what tells
221 them apart, which is the whole point of judging against cadence."""
222 result = fold([run("minutely", "2026-07-31T13:50:00Z")],
223 tasks=[task("minutely", schedule="* * * * *")])
224 assert by_id(result, "minutely")["liveness"] in ("stale", "lost")
225
226
227def test_a_daily_pass_genuinely_missing_for_days_is_still_caught():
228 """Cadence-relative must not mean unfalsifiable."""
229 result = fold([run("nightly", "2026-07-20T06:55:00Z")],
230 tasks=[task("nightly", schedule="55 6 * * *")])
231 assert by_id(result, "nightly")["liveness"] == "lost"
232
233
234def test_a_pass_may_miss_a_firing_or_two_before_anyone_is_told():
235 """Cron drift, a long previous pass and a restart are ordinary. A
236 board that shouts on the first missed beat is noise."""
237 result = fold([run("minutely", "2026-07-31T13:58:00Z")],
238 tasks=[task("minutely", schedule="* * * * *")])
239 assert by_id(result, "minutely")["liveness"] == "live"
240
241
242def test_the_verdict_carries_what_it_was_judged_by():
243 """"lost" means nothing without knowing the thing was expected
244 minutely rather than monthly -- the same reason a quality score
245 states its method."""
246 result = fold([run("p", "2026-07-31T13:59:00Z")],
247 tasks=[task("p", schedule="*/10 * * * *")])
248 assert by_id(result, "p")["expected_every"] == 600
249
250
251def test_an_unparseable_schedule_falls_back_to_the_agent_windows():
252 """Not to a guess. A schedule nobody can read is a reason to use the
253 default, not to invent an interval."""
254 assert object_agents.cron_interval_seconds("nonsense") is None
255 assert object_agents.worker_windows("nonsense", stale_seconds=900,
256 lost_seconds=3600) == (900, 3600)
257
258
259def test_scheduled_work_is_judged_more_tightly_than_an_agent():
260 """The first version floored these at the AGENT windows and was wrong
261 in the other direction: a minutely pass could go quiet for fifteen
262 minutes and still read `live`. The two cases genuinely differ -- an
263 agent's beat is best-effort ("reasoning for ten minutes writes
264 nothing and is perfectly alive"), while a cron either fired or it did
265 not. Scheduled work can be held to a tighter standard, and should
266 be."""
267 stale, lost = object_agents.worker_windows("* * * * *", stale_seconds=900,
268 lost_seconds=3600)
269 assert stale < 900 and lost < 3600
270 assert (stale, lost) == (object_agents.JITTER_STALE_SECONDS,
271 object_agents.JITTER_LOST_SECONDS)
272
273
274@pytest.mark.parametrize("schedule,expected", [
275 ("* * * * *", 60), ("*/10 * * * *", 600), ("5 * * * *", 3600),
276 ("55 6 * * *", 86400), ("0 3 * * 1", 604800),
277])
278def test_cron_cadence_estimates(schedule, expected):
279 assert object_agents.cron_interval_seconds(schedule) == expected