askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_agents.py · 22.0 KB · 537 lines
| 1 | """object_agents -- who is here, what they can do, and what they have |
| 2 | committed. Pure: no I/O, no clock, no data directory. |
| 3 | |
| 4 | Same posture as object_cart, object_rates, object_ledger_head: the |
| 5 | package objects in packages/app-agents do the reading and writing and call |
| 6 | these. |
| 7 | |
| 8 | ## Liveness is the one thing the change log cannot tell you |
| 9 | |
| 10 | Everything else about an agent is already recorded. Every write on this |
| 11 | box carries an actor; app-activity renders per-actor history; identity |
| 12 | owns who the agent is. The single question none of that answers is |
| 13 | **"is it still there?"** -- because an agent reasoning for ten minutes |
| 14 | writes nothing at all and is perfectly alive, while a crashed one looks |
| 15 | alive for as long as its last write stays recent. |
| 16 | |
| 17 | So liveness is a heartbeat compared against a supplied `now`, and both |
| 18 | are parameters here rather than read from a clock, which is what makes it |
| 19 | testable without sleeping. |
| 20 | |
| 21 | ## Three states, not two |
| 22 | |
| 23 | `live` / `stale` / `lost` rather than a boolean, because the useful thing |
| 24 | an operator wants is not "up or down" but "should I be worried yet". A |
| 25 | missed beat is ordinary; several missed beats is a question; a long |
| 26 | silence is an answer. The thresholds are the caller's -- 25-agents-spec |
| 27 | deliberately declined to name a number ("Open Questions #5"), and this |
| 28 | module declines too, defaulting only so a caller that has no opinion |
| 29 | still gets something sensible. |
| 30 | |
| 31 | ## Spend is a fold over the ledger, never a stored counter |
| 32 | |
| 33 | `committed_minor` sums an agent's holds and debits out of wallet_entries. |
| 34 | It is NOT a column on agent_registry, for the reason the whole codebase |
| 35 | repeats: a counter beside a log of movements is two truths that drift, |
| 36 | and when they drift the gate that reads the counter authorises a spend |
| 37 | that should have been refused. The identical argument |
| 38 | hook_wallet_entries makes about wallets.balance_minor, and |
| 39 | object_promotions makes about redemptions_used. |
| 40 | """ |
| 41 | |
| 42 | DEFAULT_STALE_SECONDS = 900 # 15 minutes: a missed beat or two |
| 43 | DEFAULT_LOST_SECONDS = 3600 # an hour of silence is an answer |
| 44 | |
| 45 | # Wallet entry kinds that represent money an agent has COMMITTED. A hold |
| 46 | # is money reserved and not yet spent; a debit is money spent. Both count |
| 47 | # against a cap, because a cap that ignored holds would let an agent queue |
| 48 | # a hundred runs and only notice at settlement -- which is exactly the |
| 49 | # failure the hold model exists to prevent (plan/template-runner-spec.md). |
| 50 | COMMITTING_KINDS = ("hold", "debit", "auto_topup") |
| 51 | |
| 52 | # Board ordering, worst first. `never` outranks `lost` because an agent |
| 53 | # that registered and never spoke is almost always a misconfiguration |
| 54 | # somebody can fix right now, where one that went quiet after working may |
| 55 | # simply be busy. Paused and retired sort BELOW live: they are settings, |
| 56 | # not problems, and an operator's own decision should never sit at the top |
| 57 | # of a page whose job is to show what needs attention. |
| 58 | _LIVENESS_RANK = { |
| 59 | "never": 0, "lost": 1, "stale": 2, "live": 3, |
| 60 | "suspended": 4, "paused": 5, "retired": 6, |
| 61 | } |
| 62 | |
| 63 | |
| 64 | def _text(value): |
| 65 | return str(value if value is not None else "").strip() |
| 66 | |
| 67 | |
| 68 | def _int(value, default=0): |
| 69 | try: |
| 70 | return int(_text(value) or default) |
| 71 | except (TypeError, ValueError): |
| 72 | return default |
| 73 | |
| 74 | |
| 75 | def capabilities(agent): |
| 76 | """The tags this agent accepts work for, normalised. |
| 77 | |
| 78 | Lower-cased and de-duplicated, order preserved. Empty is the default |
| 79 | and means "accepts nothing": a registration that advertises nothing |
| 80 | must never be routed work, because opting in is the whole posture |
| 81 | (nobody's laptop joins a compute pool by accident). |
| 82 | """ |
| 83 | seen, out = set(), [] |
| 84 | for part in _text((agent or {}).get("capabilities")).split(","): |
| 85 | tag = part.strip().lower() |
| 86 | if tag and tag not in seen: |
| 87 | seen.add(tag) |
| 88 | out.append(tag) |
| 89 | return out |
| 90 | |
| 91 | |
| 92 | def can_serve(agent, requirement): |
| 93 | """Whether this agent advertises what a piece of work requires. |
| 94 | |
| 95 | A blank requirement is servable by ANYONE (ordinary work needs no |
| 96 | special capability). A blank capability list serves NOTHING, including |
| 97 | blank requirements -- an agent that advertised nothing is not |
| 98 | volunteering for general duty either, it simply has not opted in. |
| 99 | """ |
| 100 | wanted = _text(requirement).lower() |
| 101 | have = capabilities(agent) |
| 102 | if not have: |
| 103 | return False |
| 104 | if not wanted: |
| 105 | return True |
| 106 | return all(tag in have for tag in |
| 107 | [part.strip() for part in wanted.split(",") if part.strip()]) |
| 108 | |
| 109 | |
| 110 | def liveness(agent, *, now, stale_seconds=DEFAULT_STALE_SECONDS, |
| 111 | lost_seconds=DEFAULT_LOST_SECONDS): |
| 112 | """`live` | `stale` | `lost` | `never` for one agent. |
| 113 | |
| 114 | `never` is its own answer rather than being folded into `lost`, |
| 115 | because "registered and has not yet said anything" is a different |
| 116 | problem from "was here and stopped" -- the first is usually a |
| 117 | misconfiguration, the second an incident. |
| 118 | |
| 119 | An agent that is not `active` reports its own status instead: a paused |
| 120 | or retired agent is not silent, it is off, and rendering it as `lost` |
| 121 | would manufacture an alarm out of an operator's own decision. |
| 122 | """ |
| 123 | status = _text((agent or {}).get("status")) or "active" |
| 124 | if status != "active": |
| 125 | return status |
| 126 | |
| 127 | beat = _text((agent or {}).get("heartbeat_at")) |
| 128 | if not beat: |
| 129 | return "never" |
| 130 | |
| 131 | # Datetimes, not strings: see _utc_naive for why comparing ISO text |
| 132 | # across `Z` and `+00:00` silently ages a timestamp. |
| 133 | beat_at, right_now = _utc_naive(beat), _utc_naive(now) |
| 134 | if beat_at is None or right_now is None: |
| 135 | return "never" |
| 136 | age = (right_now - beat_at).total_seconds() |
| 137 | if age <= abs(int(stale_seconds)): |
| 138 | return "live" |
| 139 | if age <= abs(int(lost_seconds)): |
| 140 | return "stale" |
| 141 | return "lost" |
| 142 | |
| 143 | |
| 144 | def committed_minor(agent_id, wallet_entries): |
| 145 | """What this agent has committed: holds plus debits, as a positive |
| 146 | number. A FOLD over the ledger -- never a stored counter. |
| 147 | |
| 148 | Reads `owner_id` on the entry, which is the agent's user_id, the same |
| 149 | value every write it makes already carries as its actor. |
| 150 | """ |
| 151 | target = _text(agent_id) |
| 152 | if not target: |
| 153 | return 0 |
| 154 | total = 0 |
| 155 | for row in wallet_entries or (): |
| 156 | if _text(row.get("owner_id")) != target: |
| 157 | continue |
| 158 | if _text(row.get("kind")) not in COMMITTING_KINDS: |
| 159 | continue |
| 160 | amount = _int(row.get("amount_minor")) |
| 161 | if amount < 0: |
| 162 | total += -amount |
| 163 | return total |
| 164 | |
| 165 | |
| 166 | def over_cap(agent, wallet_entries): |
| 167 | """Has this agent spent past its cap? (cap, committed, over) -- or |
| 168 | None when it has no cap. |
| 169 | |
| 170 | A cap of 0 means NO CAP, matching billing.wallet.overdraft_minor's |
| 171 | convention. Stating that here because the opposite reading -- zero as |
| 172 | "may spend nothing" -- would silently suspend every agent the moment |
| 173 | the field shipped, which is the worst possible default for a field |
| 174 | added to existing rows. |
| 175 | """ |
| 176 | cap = _int((agent or {}).get("spend_cap_minor")) |
| 177 | if cap <= 0: |
| 178 | return None |
| 179 | committed = committed_minor((agent or {}).get("agent_id"), wallet_entries) |
| 180 | return {"cap_minor": cap, "committed_minor": committed, |
| 181 | "over": committed > cap, |
| 182 | "remaining_minor": max(0, cap - committed)} |
| 183 | |
| 184 | |
| 185 | def board(agents, wallet_entries=None, *, now, |
| 186 | stale_seconds=DEFAULT_STALE_SECONDS, |
| 187 | lost_seconds=DEFAULT_LOST_SECONDS): |
| 188 | """Everything the agent board renders, as one fold. |
| 189 | |
| 190 | One shape for the page and the JSON, so the two cannot disagree -- |
| 191 | the posture site_privacy and site_ledger_integrity both take. |
| 192 | """ |
| 193 | rows = [] |
| 194 | for agent in agents or (): |
| 195 | state = liveness(agent, now=now, stale_seconds=stale_seconds, |
| 196 | lost_seconds=lost_seconds) |
| 197 | rows.append({ |
| 198 | "agent_id": _text(agent.get("agent_id")), |
| 199 | "label": _text(agent.get("label")) or _text(agent.get("agent_id")), |
| 200 | "purpose": _text(agent.get("purpose")), |
| 201 | "capabilities": capabilities(agent), |
| 202 | "endpoint": _text(agent.get("endpoint")), |
| 203 | "status": _text(agent.get("status")) or "active", |
| 204 | "heartbeat_at": _text(agent.get("heartbeat_at")), |
| 205 | # Both forms: the stamp for anything computing, the phrase for |
| 206 | # the human being asked "is it still there". |
| 207 | "heartbeat_ago": relative_time(agent.get("heartbeat_at"), now), |
| 208 | "liveness": state, |
| 209 | "spend": over_cap(agent, wallet_entries or []), |
| 210 | }) |
| 211 | # WORST FIRST. A board exists to surface what needs attention, so the |
| 212 | # silent agents lead and the working ones sink -- the opposite of the |
| 213 | # obvious sort, and the reason to write the order down rather than |
| 214 | # leave it to a boolean. An operator's own decisions (paused, retired) |
| 215 | # rank below live: they are not problems, they are settings. |
| 216 | rows.sort(key=lambda row: (_LIVENESS_RANK.get(row["liveness"], 50), |
| 217 | row["label"].lower())) |
| 218 | |
| 219 | counts = {} |
| 220 | for row in rows: |
| 221 | counts[row["liveness"]] = counts.get(row["liveness"], 0) + 1 |
| 222 | return { |
| 223 | "agents": rows, |
| 224 | "counts": counts, |
| 225 | "live": counts.get("live", 0), |
| 226 | "total": len(rows), |
| 227 | "over_cap": [row["agent_id"] for row in rows |
| 228 | if row["spend"] and row["spend"]["over"]], |
| 229 | "capabilities": sorted({tag for row in rows for tag in row["capabilities"]}), |
| 230 | } |
| 231 | |
| 232 | |
| 233 | def _utc_naive(stamp): |
| 234 | """Any ISO-8601 form -> a naive UTC datetime, or None. |
| 235 | |
| 236 | The board was only ever fed timestamps the heartbeat action writes, |
| 237 | which end in `Z`. The moment it also folded scheduler_runs it met |
| 238 | `+00:00`, and TWO things broke at once: |
| 239 | |
| 240 | - `rstrip("Z")` leaves an offset intact, so one side of a subtraction |
| 241 | stayed timezone-aware and the other did not: TypeError, live, on a |
| 242 | page that had passed every test. |
| 243 | - Worse and quieter, the lexicographic comparison this module leans |
| 244 | on is WRONG across the two forms. "+" (0x2B) sorts below "Z" |
| 245 | (0x5A), so `...14:00:00+00:00` < `...14:00:00Z` -- the same instant |
| 246 | reads as older, and a worker that had just run could be reported |
| 247 | stale. A test would never catch it while every fixture used one |
| 248 | format. |
| 249 | |
| 250 | So every comparison goes through here now, and the answer is a real |
| 251 | datetime rather than a string that happens to sort. |
| 252 | """ |
| 253 | from datetime import datetime, timezone |
| 254 | import re |
| 255 | |
| 256 | text = _text(stamp) |
| 257 | if not text: |
| 258 | return None |
| 259 | if text.endswith("Z"): |
| 260 | text = text[:-1] + "+00:00" |
| 261 | # Python 3.10's fromisoformat accepts only 3 or 6 fractional digits, |
| 262 | # and writers in the wild emit whatever they emit. Normalizing the |
| 263 | # fraction here rather than trusting it is the difference between a |
| 264 | # page that works on 3.12 and one that works. |
| 265 | match = re.match(r"^(.*?)\.(\d+)(.*)$", text) |
| 266 | if match: |
| 267 | head, fraction, tail = match.groups() |
| 268 | text = f"{head}.{fraction[:6].ljust(6, '0')}{tail}" |
| 269 | try: |
| 270 | parsed = datetime.fromisoformat(text) |
| 271 | except ValueError: |
| 272 | return None |
| 273 | if parsed.tzinfo is not None: |
| 274 | parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None) |
| 275 | return parsed |
| 276 | |
| 277 | |
| 278 | def relative_time(stamp, now): |
| 279 | """"4 hours ago" rather than "2026-07-27T03:12:44Z". |
| 280 | |
| 281 | A liveness verdict is supposed to be stated WITH its evidence, and an |
| 282 | ISO timestamp is not evidence a human can act on -- an operator asked |
| 283 | "is it still there" does not want to subtract two datetimes in their |
| 284 | head. "last beat 4 hours ago" answers the question the page exists to |
| 285 | answer; the raw stamp is kept in the JSON for anything that needs to |
| 286 | compute rather than read. |
| 287 | |
| 288 | Pure, `now` passed in, same as everything else here. Deliberately |
| 289 | coarse: nobody needs "3 hours, 14 minutes, 9 seconds", they need to |
| 290 | know whether it is minutes or days. |
| 291 | """ |
| 292 | if not _text(stamp): |
| 293 | return "never" |
| 294 | then, right_now = _utc_naive(stamp), _utc_naive(now) |
| 295 | if then is None or right_now is None: |
| 296 | return _text(stamp) |
| 297 | |
| 298 | seconds = (right_now - then).total_seconds() |
| 299 | if seconds < 0: |
| 300 | return "just now" # clock skew reads as now, never "in 3 hours" |
| 301 | for limit, divisor, unit in ((60, 1, "second"), (3600, 60, "minute"), |
| 302 | (86400, 3600, "hour"), (604800, 86400, "day"), |
| 303 | (2629800, 604800, "week")): |
| 304 | if seconds < limit: |
| 305 | value = int(seconds // divisor) |
| 306 | if value <= 0: |
| 307 | return "just now" |
| 308 | return f"{value} {unit}{'' if value == 1 else 's'} ago" |
| 309 | months = int(seconds // 2629800) |
| 310 | return f"{months} month{'' if months == 1 else 's'} ago" |
| 311 | |
| 312 | |
| 313 | def _shift_iso(stamp, seconds): |
| 314 | """An ISO timestamp moved by N seconds, staying pure (no clock). |
| 315 | |
| 316 | Same helper object_template_runs uses for staleness, and for the same |
| 317 | reason: ISO-8601 strings sort lexicographically, so a comparison |
| 318 | against a shifted stamp is the whole of the arithmetic. |
| 319 | """ |
| 320 | from datetime import datetime, timedelta |
| 321 | |
| 322 | text = stamp.rstrip("Z") |
| 323 | try: |
| 324 | parsed = datetime.fromisoformat(text) |
| 325 | except ValueError: |
| 326 | return stamp |
| 327 | shifted = parsed + timedelta(seconds=seconds) |
| 328 | out = shifted.isoformat() |
| 329 | return out + "Z" if stamp.endswith("Z") else out |
| 330 | |
| 331 | |
| 332 | # === the box's own workers ===================================================== |
| 333 | # |
| 334 | # The board above answers "are the external agents alive". Nobody was |
| 335 | # asking the same question of THIS server's scheduled work, and the |
| 336 | # answer was already on disk: every scheduled execution writes a |
| 337 | # scheduler_runs row (object_id, started_at, ok, duration_ms, error), so |
| 338 | # "did the runner pass run in the last minute" has been answerable all |
| 339 | # along -- just never surfaced. |
| 340 | # |
| 341 | # So this is a fold, not a new heartbeat. Adding a writer that stamped |
| 342 | # "the daemon is alive" beside a log that already proves it would be a |
| 343 | # second, weaker account of the same fact: a heartbeat says a process |
| 344 | # reached the line that writes heartbeats, while a scheduler_runs row |
| 345 | # says the actual work ran and whether it worked. The evidence that |
| 346 | # already exists is the better evidence. |
| 347 | # |
| 348 | # The liveness verdict deliberately reuses `liveness()` above, by handing |
| 349 | # it a synthetic agent shaped like a registry row. A worker that has gone |
| 350 | # quiet and an agent that has gone quiet are the same question, and two |
| 351 | # rules that could drift apart is exactly how one board starts lying. |
| 352 | |
| 353 | # --- a schedule is what makes silence mean something -------------------------- |
| 354 | # |
| 355 | # The first version of the workers panel judged every scheduled pass |
| 356 | # against the agent windows (15 minutes stale, an hour lost) and the live |
| 357 | # box immediately reported TEN of fourteen as `lost` -- every daily cron, |
| 358 | # for the crime of being daily. A pass that runs at 06:55 and last ran |
| 359 | # seven hours ago is not lost, it is waiting, and a board that says |
| 360 | # otherwise is one an operator learns to ignore. That is the same |
| 361 | # cries-wolf failure the customer-funds reconciliation had to design |
| 362 | # around, arriving from the opposite direction. |
| 363 | # |
| 364 | # So silence is judged against the DECLARED INTERVAL. A minutely task |
| 365 | # quiet for ten minutes is broken; a daily task quiet for ten hours is |
| 366 | # early. Same evidence, opposite verdicts, and the schedule is what tells |
| 367 | # them apart. |
| 368 | |
| 369 | _CRON_FIELD_PERIODS = (60, 3600, 86400, 2629800, 604800) # min, hour, dom, mon, dow |
| 370 | |
| 371 | |
| 372 | def cron_interval_seconds(schedule): |
| 373 | """Roughly how often a 5-field cron expression fires, in seconds. |
| 374 | |
| 375 | Deliberately approximate: this feeds a staleness multiplier, not a |
| 376 | scheduler, and being out by a factor under two changes no verdict. |
| 377 | Returns None for anything it does not understand, which is the signal |
| 378 | to fall back on the fixed windows rather than to invent a number. |
| 379 | """ |
| 380 | fields = _text(schedule).split() |
| 381 | if len(fields) != 5: |
| 382 | return None |
| 383 | |
| 384 | minute, hour, dom, month, dow = fields |
| 385 | |
| 386 | def step(field, period): |
| 387 | """The period implied by one field, or None when it says nothing.""" |
| 388 | if field == "*": |
| 389 | return None # "every" -- the field below decides |
| 390 | if field.startswith("*/"): |
| 391 | try: |
| 392 | every = int(field[2:]) |
| 393 | except ValueError: |
| 394 | return None |
| 395 | return period * max(1, every) |
| 396 | return None # a fixed value pins the field above |
| 397 | |
| 398 | # Walk from the finest field upward. The first field that is not "*" |
| 399 | # and not a step sets the cadence to the NEXT unit up: "5 * * * *" |
| 400 | # fires hourly, "55 6 * * *" daily. |
| 401 | if minute == "*": |
| 402 | base = 60 |
| 403 | elif (stepped := step(minute, 60)) is not None: |
| 404 | base = stepped |
| 405 | elif hour == "*": |
| 406 | base = 3600 |
| 407 | elif (stepped := step(hour, 3600)) is not None: |
| 408 | base = stepped |
| 409 | elif dom == "*" and dow == "*": |
| 410 | base = 86400 |
| 411 | elif dow != "*": |
| 412 | base = 604800 |
| 413 | else: |
| 414 | base = 2629800 |
| 415 | return base |
| 416 | |
| 417 | |
| 418 | #: Absorbs ordinary jitter -- the scheduler polls on a loop, a long pass |
| 419 | #: delays the next one, a restart skips a beat. Deliberately small: these |
| 420 | #: are floors for the FASTEST schedules, not a global window. |
| 421 | JITTER_STALE_SECONDS = 180 |
| 422 | JITTER_LOST_SECONDS = 600 |
| 423 | |
| 424 | |
| 425 | def worker_windows(schedule, *, stale_seconds, lost_seconds, |
| 426 | stale_multiple=3, lost_multiple=8): |
| 427 | """The (stale, lost) thresholds this schedule should be judged by. |
| 428 | |
| 429 | A pass may miss a couple of firings before anyone is told and several |
| 430 | before it is called lost, because cron drift, a long previous pass |
| 431 | and a restart are all ordinary. |
| 432 | |
| 433 | The floors are deliberately NOT the agent windows, which was the |
| 434 | first version and was wrong in the other direction: it let a minutely |
| 435 | pass go quiet for fifteen minutes and still read `live`. The two |
| 436 | cases are genuinely different. An agent's heartbeat is best-effort -- |
| 437 | "an agent reasoning for ten minutes writes nothing and is perfectly |
| 438 | alive" is the reason that window is generous. A cron either fired or |
| 439 | it did not, and the scheduler is reliable, so scheduled work can and |
| 440 | should be judged more tightly than an agent. |
| 441 | |
| 442 | `stale_seconds`/`lost_seconds` remain the answer for a schedule |
| 443 | nobody can parse: an unreadable schedule is a reason to fall back, |
| 444 | never to invent an interval. |
| 445 | """ |
| 446 | interval = cron_interval_seconds(schedule) |
| 447 | if interval is None: |
| 448 | return abs(int(stale_seconds)), abs(int(lost_seconds)) |
| 449 | return (max(JITTER_STALE_SECONDS, interval * stale_multiple), |
| 450 | max(JITTER_LOST_SECONDS, interval * lost_multiple)) |
| 451 | |
| 452 | |
| 453 | def worker_liveness(scheduler_runs, *, now, tasks=None, |
| 454 | stale_seconds=DEFAULT_STALE_SECONDS, |
| 455 | lost_seconds=DEFAULT_LOST_SECONDS): |
| 456 | """Fold scheduler_runs into one liveness row per scheduled object. |
| 457 | |
| 458 | `tasks` (the scheduler's declared tasks, each {object_id, schedule, |
| 459 | status}) is optional but changes the answer in one important way: a |
| 460 | task declared and NEVER run is `never`, which no amount of reading |
| 461 | the run log alone could tell you -- an object that has never run has |
| 462 | no rows, and a fold over rows cannot see an absence. That is the |
| 463 | failure this is most likely to catch: a pass somebody scheduled that |
| 464 | has been silently doing nothing since the day it was added. |
| 465 | |
| 466 | Pure: no clock, no I/O. |
| 467 | """ |
| 468 | latest = {} |
| 469 | for row in scheduler_runs or (): |
| 470 | object_id = _text(row.get("object_id")) |
| 471 | if not object_id: |
| 472 | continue |
| 473 | started = _text(row.get("started_at")) |
| 474 | current = latest.get(object_id) |
| 475 | if current is None or started > _text(current.get("started_at")): |
| 476 | latest[object_id] = row |
| 477 | |
| 478 | declared = {} |
| 479 | for task in tasks or (): |
| 480 | object_id = _text(task.get("object_id")) |
| 481 | if object_id: |
| 482 | declared[object_id] = task |
| 483 | |
| 484 | rows = [] |
| 485 | for object_id in sorted(set(latest) | set(declared)): |
| 486 | run = latest.get(object_id) or {} |
| 487 | task = declared.get(object_id) or {} |
| 488 | # A task an operator switched off is not silent, it is off -- |
| 489 | # the same distinction liveness() draws for a paused agent, and |
| 490 | # the same reason it must not be rendered as a problem. |
| 491 | status = _text(task.get("status")) or "active" |
| 492 | synthetic = { |
| 493 | "status": "active" if status == "active" else "paused", |
| 494 | "heartbeat_at": _text(run.get("started_at")), |
| 495 | } |
| 496 | # Judged against ITS OWN cadence, not a global window. |
| 497 | own_stale, own_lost = worker_windows( |
| 498 | task.get("schedule"), stale_seconds=stale_seconds, |
| 499 | lost_seconds=lost_seconds) |
| 500 | state = liveness(synthetic, now=now, stale_seconds=own_stale, |
| 501 | lost_seconds=own_lost) |
| 502 | failed = _text(run.get("ok")).lower() in ("false", "0", "no") |
| 503 | rows.append({ |
| 504 | "object_id": object_id, |
| 505 | "schedule": _text(task.get("schedule")), |
| 506 | "description": _text(task.get("description")), |
| 507 | "status": status, |
| 508 | "last_run_at": _text(run.get("started_at")), |
| 509 | "last_run_ago": relative_time(run.get("started_at"), now), |
| 510 | "duration_ms": _int(run.get("duration_ms")), |
| 511 | "run_count": _int(task.get("run_count")), |
| 512 | "liveness": state, |
| 513 | # The verdict states what it was judged by, the same way a |
| 514 | # quality score states its method: "lost" means nothing |
| 515 | # without knowing it was expected minutely rather than |
| 516 | # monthly. |
| 517 | "expected_every": cron_interval_seconds(_text(task.get("schedule"))), |
| 518 | # A pass that RAN and THREW is the case a liveness verdict |
| 519 | # alone would call healthy, so the error rides beside it |
| 520 | # rather than replacing it: "live, and failing" is a real |
| 521 | # and much worse state than "lost". |
| 522 | "last_error": _text(run.get("error")), |
| 523 | "failing": failed or bool(_text(run.get("error"))), |
| 524 | }) |
| 525 | |
| 526 | rows.sort(key=lambda row: (_LIVENESS_RANK.get(row["liveness"], 50), |
| 527 | not row["failing"], row["object_id"])) |
| 528 | counts = {} |
| 529 | for row in rows: |
| 530 | counts[row["liveness"]] = counts.get(row["liveness"], 0) + 1 |
| 531 | return { |
| 532 | "workers": rows, |
| 533 | "counts": counts, |
| 534 | "live": counts.get("live", 0), |
| 535 | "total": len(rows), |
| 536 | "failing": [row["object_id"] for row in rows if row["failing"]], |
| 537 | } |