askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_worker_liveness.py · 12.0 KB · 279 lines
| 1 | """Is this server still doing its own work? |
| 2 | |
| 3 | The agents board answered that for EXTERNAL agents and nobody was asking |
| 4 | it of the box itself. The evidence was already on disk: every scheduled |
| 5 | execution writes a scheduler_runs row (object_id, started_at, ok, |
| 6 | duration_ms, error), so "did the runner pass run in the last minute" has |
| 7 | been answerable all along -- just never surfaced. |
| 8 | |
| 9 | So this is a fold, not a new heartbeat, and that is the design decision |
| 10 | worth defending. Adding a writer that stamped "the daemon is alive" |
| 11 | beside a log that already proves it would be a second, WEAKER account of |
| 12 | the same fact: a heartbeat says a process reached the line that writes |
| 13 | heartbeats, while a scheduler_runs row says the actual work ran and |
| 14 | whether it worked. |
| 15 | """ |
| 16 | |
| 17 | import pytest |
| 18 | |
| 19 | import object_agents |
| 20 | |
| 21 | NOW = "2026-07-31T14:00:00Z" |
| 22 | |
| 23 | |
| 24 | def run(object_id, started_at, *, ok="true", error="", duration_ms="10"): |
| 25 | return {"object_id": object_id, "started_at": started_at, "ok": ok, |
| 26 | "error": error, "duration_ms": duration_ms} |
| 27 | |
| 28 | |
| 29 | def task(object_id, *, schedule="* * * * *", status="active"): |
| 30 | return {"object_id": object_id, "schedule": schedule, "status": status} |
| 31 | |
| 32 | |
| 33 | def fold(runs, tasks=None, **kw): |
| 34 | return object_agents.worker_liveness(runs, tasks=tasks, now=NOW, **kw) |
| 35 | |
| 36 | |
| 37 | def by_id(result, object_id): |
| 38 | return next(w for w in result["workers"] if w["object_id"] == object_id) |
| 39 | |
| 40 | |
| 41 | def test_a_pass_that_just_ran_is_live(): |
| 42 | result = fold([run("system_template_runner", "2026-07-31T13:59:30Z")]) |
| 43 | assert by_id(result, "system_template_runner")["liveness"] == "live" |
| 44 | assert by_id(result, "system_template_runner")["last_run_ago"] |
| 45 | |
| 46 | |
| 47 | def test_a_pass_that_stopped_hours_ago_is_lost(): |
| 48 | result = fold([run("system_run_sweeper", "2026-07-31T09:00:00Z")]) |
| 49 | assert by_id(result, "system_run_sweeper")["liveness"] == "lost" |
| 50 | |
| 51 | |
| 52 | def test_only_the_most_recent_run_decides(): |
| 53 | """The log is append-only and full of history; liveness is about the |
| 54 | newest row, not the first one found.""" |
| 55 | result = fold([run("p", "2026-07-30T01:00:00Z"), |
| 56 | run("p", "2026-07-31T13:59:00Z"), |
| 57 | run("p", "2026-07-31T02:00:00Z")]) |
| 58 | assert by_id(result, "p")["liveness"] == "live" |
| 59 | |
| 60 | |
| 61 | def test_a_task_that_has_never_run_is_visible_at_all(): |
| 62 | """THE case a fold over the run log cannot see by itself: an object |
| 63 | with no rows is an ABSENCE, and absences are invisible to a fold over |
| 64 | presences. A pass somebody scheduled that has quietly done nothing |
| 65 | since the day it was added is the failure most worth catching.""" |
| 66 | result = fold([], tasks=[task("system_never_ran", schedule="0 4 * * *")]) |
| 67 | assert by_id(result, "system_never_ran")["liveness"] == "never" |
| 68 | assert by_id(result, "system_never_ran")["schedule"] == "0 4 * * *" |
| 69 | |
| 70 | |
| 71 | def test_a_task_an_operator_switched_off_is_not_a_problem(): |
| 72 | """Same distinction liveness() draws for a paused agent: it is not |
| 73 | silent, it is off, and rendering it as lost manufactures an alarm out |
| 74 | of somebody's own decision.""" |
| 75 | result = fold([run("p", "2026-07-30T01:00:00Z")], |
| 76 | tasks=[task("p", status="paused")]) |
| 77 | assert by_id(result, "p")["liveness"] == "paused" |
| 78 | |
| 79 | |
| 80 | def test_a_pass_that_ran_and_threw_is_live_AND_failing(): |
| 81 | """The state a liveness badge alone would render as healthy, and it |
| 82 | is worse than lost: the daemon is fine, the work is broken.""" |
| 83 | result = fold([run("p", "2026-07-31T13:59:00Z", ok="false", |
| 84 | error="boom")]) |
| 85 | row = by_id(result, "p") |
| 86 | assert row["liveness"] == "live" |
| 87 | assert row["failing"] is True |
| 88 | assert row["last_error"] == "boom" |
| 89 | assert result["failing"] == ["p"] |
| 90 | |
| 91 | |
| 92 | def test_an_error_string_alone_counts_as_failing(): |
| 93 | """ok can be absent on older rows; an error message is evidence |
| 94 | enough and must not be shrugged off because a column was blank.""" |
| 95 | row = by_id(fold([run("p", "2026-07-31T13:59:00Z", ok="", error="nope")]), "p") |
| 96 | assert row["failing"] is True |
| 97 | |
| 98 | |
| 99 | def test_worst_comes_first_and_failures_outrank_healthy_peers(): |
| 100 | """A board exists to surface what needs attention.""" |
| 101 | result = fold( |
| 102 | [run("healthy", "2026-07-31T13:59:00Z"), |
| 103 | run("broken", "2026-07-31T13:59:00Z", ok="false", error="x"), |
| 104 | run("gone", "2026-07-31T01:00:00Z")], |
| 105 | tasks=[task("unstarted")]) |
| 106 | order = [w["object_id"] for w in result["workers"]] |
| 107 | assert order.index("unstarted") < order.index("healthy") |
| 108 | assert order.index("gone") < order.index("healthy") |
| 109 | assert order.index("broken") < order.index("healthy") |
| 110 | |
| 111 | |
| 112 | def test_the_declared_schedule_rides_along_for_context(): |
| 113 | """"lost" means nothing without knowing it was supposed to run |
| 114 | minutely rather than yearly.""" |
| 115 | result = fold([run("p", "2026-07-31T09:00:00Z")], |
| 116 | tasks=[task("p", schedule="*/10 * * * *")]) |
| 117 | assert by_id(result, "p")["schedule"] == "*/10 * * * *" |
| 118 | |
| 119 | |
| 120 | def test_counts_and_totals_summarize_the_board(): |
| 121 | result = fold([run("a", "2026-07-31T13:59:00Z"), |
| 122 | run("b", "2026-07-31T01:00:00Z")], |
| 123 | tasks=[task("c")]) |
| 124 | assert result["total"] == 3 |
| 125 | assert result["live"] == 1 |
| 126 | assert result["counts"]["never"] == 1 |
| 127 | |
| 128 | |
| 129 | def test_an_empty_box_folds_to_nothing_rather_than_erroring(): |
| 130 | assert fold([], tasks=[]) == {"workers": [], "counts": {}, "live": 0, |
| 131 | "total": 0, "failing": []} |
| 132 | |
| 133 | |
| 134 | def test_the_liveness_rule_is_the_SAME_one_agents_use(): |
| 135 | """Two rules that could drift apart is how one board starts lying, so |
| 136 | a worker and an agent that have both been silent for the same |
| 137 | interval must get the same verdict.""" |
| 138 | silent = "2026-07-31T13:40:00Z" |
| 139 | worker = by_id(fold([run("p", silent)]), "p")["liveness"] |
| 140 | agent = object_agents.liveness({"status": "active", "heartbeat_at": silent}, |
| 141 | now=NOW) |
| 142 | assert worker == agent |
| 143 | |
| 144 | |
| 145 | # --- timestamp forms: the bug that only real data had ------------------------- |
| 146 | |
| 147 | def test_an_offset_aware_stamp_is_not_silently_aged(): |
| 148 | """THE live bug, and the quiet half of it. |
| 149 | |
| 150 | The board had only ever seen timestamps the heartbeat action writes, |
| 151 | which end in `Z`. scheduler_runs writes `+00:00`, and the module's |
| 152 | lexicographic ISO comparison is WRONG across the two forms: "+" |
| 153 | (0x2B) sorts below "Z" (0x5A), so the same instant in the offset form |
| 154 | reads as OLDER. A worker that had just run could be reported stale, |
| 155 | and no fixture using one format could ever catch it. |
| 156 | """ |
| 157 | assert "2026-07-31T14:00:00+00:00" < "2026-07-31T14:00:00Z" # the hazard |
| 158 | |
| 159 | just_ran = object_agents.liveness( |
| 160 | {"status": "active", "heartbeat_at": "2026-07-31T13:59:00+00:00"}, |
| 161 | now=NOW) |
| 162 | assert just_ran == "live" |
| 163 | |
| 164 | |
| 165 | def test_the_two_forms_of_the_same_instant_agree(): |
| 166 | """Agreement is the property, whatever the verdict happens to be -- |
| 167 | 30 minutes past a 15-minute window is legitimately `stale`, and both |
| 168 | spellings of that instant must say so identically.""" |
| 169 | verdicts = {object_agents.liveness( |
| 170 | {"status": "active", "heartbeat_at": stamp}, now=NOW) |
| 171 | for stamp in ("2026-07-31T13:30:00Z", "2026-07-31T13:30:00+00:00")} |
| 172 | assert verdicts == {"stale"} |
| 173 | |
| 174 | fresh = {object_agents.liveness( |
| 175 | {"status": "active", "heartbeat_at": stamp}, now=NOW) |
| 176 | for stamp in ("2026-07-31T13:59:00Z", "2026-07-31T13:59:00+00:00")} |
| 177 | assert fresh == {"live"} |
| 178 | |
| 179 | |
| 180 | def test_a_real_scheduler_runs_stamp_folds(tmp_path=None): |
| 181 | """Copied verbatim off the live box, microseconds and all -- the |
| 182 | shape that produced 'can't subtract offset-naive and offset-aware |
| 183 | datetimes' on a page that had passed every test.""" |
| 184 | result = fold([run("system_invoice_aging", "2026-07-31T13:59:27.944184+00:00")]) |
| 185 | row = by_id(result, "system_invoice_aging") |
| 186 | assert row["liveness"] == "live" |
| 187 | assert row["last_run_ago"].endswith("ago") or row["last_run_ago"] == "just now" |
| 188 | |
| 189 | |
| 190 | def test_an_odd_fractional_precision_still_parses(): |
| 191 | """Python 3.10's fromisoformat takes only 3 or 6 fractional digits, |
| 192 | and writers emit whatever they emit. The droplet runs 3.12 and is |
| 193 | lenient; trusting that is how a page works in production and breaks |
| 194 | on a developer's machine.""" |
| 195 | assert object_agents.relative_time("2026-07-31T13:00:00.9+00:00", NOW) \ |
| 196 | == "1 hour ago" or "minutes ago" in object_agents.relative_time( |
| 197 | "2026-07-31T13:00:00.9+00:00", NOW) |
| 198 | |
| 199 | |
| 200 | def test_an_unparseable_stamp_degrades_rather_than_raising(): |
| 201 | assert object_agents.relative_time("not a date", NOW) == "not a date" |
| 202 | assert object_agents.liveness( |
| 203 | {"status": "active", "heartbeat_at": "not a date"}, now=NOW) == "never" |
| 204 | |
| 205 | |
| 206 | # --- silence only means something against a schedule -------------------------- |
| 207 | |
| 208 | def test_a_daily_pass_seven_hours_quiet_is_not_lost(): |
| 209 | """THE false alarm the live board produced on its first deploy: ten |
| 210 | of fourteen passes reported `lost` for the crime of being daily. A |
| 211 | cron that fires at 06:55 and last ran seven hours ago is not lost, it |
| 212 | is waiting -- and a board that says otherwise is one an operator |
| 213 | learns to ignore.""" |
| 214 | result = fold([run("nightly", "2026-07-31T06:55:00Z")], |
| 215 | tasks=[task("nightly", schedule="55 6 * * *")]) |
| 216 | assert by_id(result, "nightly")["liveness"] == "live" |
| 217 | |
| 218 | |
| 219 | def test_a_minutely_pass_ten_minutes_quiet_IS_a_problem(): |
| 220 | """The same silence, the opposite verdict. The schedule is what tells |
| 221 | them apart, which is the whole point of judging against cadence.""" |
| 222 | result = fold([run("minutely", "2026-07-31T13:50:00Z")], |
| 223 | tasks=[task("minutely", schedule="* * * * *")]) |
| 224 | assert by_id(result, "minutely")["liveness"] in ("stale", "lost") |
| 225 | |
| 226 | |
| 227 | def test_a_daily_pass_genuinely_missing_for_days_is_still_caught(): |
| 228 | """Cadence-relative must not mean unfalsifiable.""" |
| 229 | result = fold([run("nightly", "2026-07-20T06:55:00Z")], |
| 230 | tasks=[task("nightly", schedule="55 6 * * *")]) |
| 231 | assert by_id(result, "nightly")["liveness"] == "lost" |
| 232 | |
| 233 | |
| 234 | def test_a_pass_may_miss_a_firing_or_two_before_anyone_is_told(): |
| 235 | """Cron drift, a long previous pass and a restart are ordinary. A |
| 236 | board that shouts on the first missed beat is noise.""" |
| 237 | result = fold([run("minutely", "2026-07-31T13:58:00Z")], |
| 238 | tasks=[task("minutely", schedule="* * * * *")]) |
| 239 | assert by_id(result, "minutely")["liveness"] == "live" |
| 240 | |
| 241 | |
| 242 | def test_the_verdict_carries_what_it_was_judged_by(): |
| 243 | """"lost" means nothing without knowing the thing was expected |
| 244 | minutely rather than monthly -- the same reason a quality score |
| 245 | states its method.""" |
| 246 | result = fold([run("p", "2026-07-31T13:59:00Z")], |
| 247 | tasks=[task("p", schedule="*/10 * * * *")]) |
| 248 | assert by_id(result, "p")["expected_every"] == 600 |
| 249 | |
| 250 | |
| 251 | def test_an_unparseable_schedule_falls_back_to_the_agent_windows(): |
| 252 | """Not to a guess. A schedule nobody can read is a reason to use the |
| 253 | default, not to invent an interval.""" |
| 254 | assert object_agents.cron_interval_seconds("nonsense") is None |
| 255 | assert object_agents.worker_windows("nonsense", stale_seconds=900, |
| 256 | lost_seconds=3600) == (900, 3600) |
| 257 | |
| 258 | |
| 259 | def test_scheduled_work_is_judged_more_tightly_than_an_agent(): |
| 260 | """The first version floored these at the AGENT windows and was wrong |
| 261 | in the other direction: a minutely pass could go quiet for fifteen |
| 262 | minutes and still read `live`. The two cases genuinely differ -- an |
| 263 | agent's beat is best-effort ("reasoning for ten minutes writes |
| 264 | nothing and is perfectly alive"), while a cron either fired or it did |
| 265 | not. Scheduled work can be held to a tighter standard, and should |
| 266 | be.""" |
| 267 | stale, lost = object_agents.worker_windows("* * * * *", stale_seconds=900, |
| 268 | lost_seconds=3600) |
| 269 | assert stale < 900 and lost < 3600 |
| 270 | assert (stale, lost) == (object_agents.JITTER_STALE_SECONDS, |
| 271 | object_agents.JITTER_LOST_SECONDS) |
| 272 | |
| 273 | |
| 274 | @pytest.mark.parametrize("schedule,expected", [ |
| 275 | ("* * * * *", 60), ("*/10 * * * *", 600), ("5 * * * *", 3600), |
| 276 | ("55 6 * * *", 86400), ("0 3 * * 1", 604800), |
| 277 | ]) |
| 278 | def test_cron_cadence_estimates(schedule, expected): |
| 279 | assert object_agents.cron_interval_seconds(schedule) == expected |