askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
template_runner.py · 20.6 KB · 485 lines
| 1 | """system_template_runner -- claim queued runs, execute them, settle the |
| 2 | money. The engine; every handler is a plug in it. |
| 3 | |
| 4 | Runs every minute from the package schedule, and by hand from the same |
| 5 | object, so the button and the pass cannot drift apart. |
| 6 | |
| 7 | ## One engine, pluggable handlers |
| 8 | |
| 9 | The handler owns exactly one thing: turning a claimed run into an output |
| 10 | or an error. Claiming, charging, settling, recording and reporting belong |
| 11 | to the engine, and no handler gets opinions about them. "Screenshot Page" |
| 12 | will be one handler of this engine, not a feature -- the uniform layer |
| 13 | the predecessor hand-built four times. |
| 14 | |
| 15 | Slice 1 ships two: `echo` (the rendered body straight back -- free, no |
| 16 | provider, and the handler that proves the money model without spending |
| 17 | anything) and `ai_text` (the rendered body to a configured provider |
| 18 | through object_ai, which already exists). |
| 19 | |
| 20 | ## What a handler must never do |
| 21 | |
| 22 | **Silently degrade.** No "provider unavailable, here is a placeholder". |
| 23 | A run that did not do the thing fails, releases its hold, and says why in |
| 24 | words a person can act on -- which capability is missing and what to |
| 25 | configure. |
| 26 | |
| 27 | ## Settlement, and the crash it is designed around |
| 28 | |
| 29 | Settlement is release-in-full plus an ordinary debit on success, release |
| 30 | alone on failure (object_template_runs.settlement). It is idempotent by |
| 31 | provenance marker, checked against the WALLET ENTRIES rather than the |
| 32 | run's status, because the crash worth designing for is exactly the one |
| 33 | between writing money and updating status: replayed at the next pass, the |
| 34 | marker is found and nothing posts twice (doctrine #7). |
| 35 | |
| 36 | Money is written FIRST, status second. If the pass dies between them the |
| 37 | run re-settles as a no-op on the next pass and the status catches up. The |
| 38 | other order would leave a run marked settled whose money never moved. |
| 39 | |
| 40 | The settlement writes are in-process and deliberately NOT gated: a |
| 41 | release is money coming back (never gated anyway), and the debit is the |
| 42 | recording of a spend that already happened at the provider. Refusing to |
| 43 | record reality does not undo it. |
| 44 | |
| 45 | ## What the engine will not do |
| 46 | |
| 47 | **Retry.** A run that fails is failed. A claimed run whose heartbeat |
| 48 | stops is the sweeper's, becomes abandoned, and stays abandoned. The |
| 49 | provider call is not idempotent and is not ours -- re-running a job that |
| 50 | may already have been billed is paying twice for one thing, and a retry |
| 51 | loop around a paid non-idempotent call is an unbounded spend wearing a |
| 52 | robustness costume. A retry is a new run, with a new hold, started by a |
| 53 | person who can see what happened. |
| 54 | """ |
| 55 | |
| 56 | import json |
| 57 | import os |
| 58 | import urllib.error |
| 59 | import urllib.request |
| 60 | from datetime import datetime, timezone |
| 61 | |
| 62 | import object_ai |
| 63 | import object_records |
| 64 | import object_service_keys |
| 65 | import object_template_runs |
| 66 | |
| 67 | ACTOR = "system_template_runner" |
| 68 | RUNS = "template_runs" |
| 69 | |
| 70 | BATCH_SETTING = "runner.batch" |
| 71 | TIMEOUT_SETTING = "runner.timeout_seconds" |
| 72 | DEFAULT_BATCH = 3 |
| 73 | DEFAULT_TIMEOUT = 120 |
| 74 | |
| 75 | |
| 76 | def _base_dir(): |
| 77 | return os.environ.get("DBBASIC_DATA_DIR", "data") |
| 78 | |
| 79 | |
| 80 | def _text(value): |
| 81 | return str(value if value is not None else "").strip() |
| 82 | |
| 83 | |
| 84 | def _int(value, default=0): |
| 85 | try: |
| 86 | return int(_text(value) or default) |
| 87 | except (TypeError, ValueError): |
| 88 | return default |
| 89 | |
| 90 | |
| 91 | def _now(): |
| 92 | return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") |
| 93 | |
| 94 | |
| 95 | def _setting(base, key, default): |
| 96 | try: |
| 97 | for row in object_records.read_collection_records("app_settings", |
| 98 | base_dir=base): |
| 99 | if _text(row.get("key")) == key: |
| 100 | return _text(row.get("value")) or default |
| 101 | except Exception: |
| 102 | pass |
| 103 | return default |
| 104 | |
| 105 | |
| 106 | # === handlers ================================================================ |
| 107 | # |
| 108 | # handler(run, base, timeout) -> {"ok", "output", "error", |
| 109 | # "provider_cost_cents", "provider_job_id"} |
| 110 | # The whole contract. A handler that needs more than this is trying to own |
| 111 | # something the engine owns. |
| 112 | # |
| 113 | # ASYNC handlers are a PAIR in ASYNC_HANDLERS: (submit, poll). |
| 114 | # |
| 115 | # submit(run, base, timeout) -> {"provider_job_id", "error"} |
| 116 | # poll(run, base, timeout) -> {"done": bool, ...the sync shape when done} |
| 117 | # |
| 118 | # The engine, not the handler, owns what that means: a submit that |
| 119 | # returns a job id moves the run to `running` and RETURNS, leaving the |
| 120 | # pass free; later passes poll. A Sora job takes minutes and q9 budgets |
| 121 | # thirty, so executing it inline would block the batch and eventually |
| 122 | # collide with the sweeper writing off a job that is legitimately still |
| 123 | # running. Short handlers (echo, ai_text) stay inline -- the engine |
| 124 | # supports both shapes and the handler declares which it is by which |
| 125 | # registry it appears in. |
| 126 | |
| 127 | def _handle_echo(run, base, timeout): |
| 128 | """The rendered body straight back. Free, instant, and the handler |
| 129 | slice 1 exists for: it exercises queue, claim, settle and report with |
| 130 | no provider and no spend, which is how the money model gets proven |
| 131 | before a dollar rides on it.""" |
| 132 | return {"ok": True, "output": _text(run.get("rendered_body")), |
| 133 | "error": "", "provider_cost_cents": 0, "provider_job_id": ""} |
| 134 | |
| 135 | |
| 136 | def _handle_ai_text(run, base, timeout): |
| 137 | """The rendered body to the stamped model, through object_ai. |
| 138 | |
| 139 | The model comes off the RUN, not the settings -- stamped at queue |
| 140 | time, so a model swap in settings changes the next run and never |
| 141 | restates what produced this one. The key is re-read now because keys |
| 142 | get deleted between queue and run; that failure is the rare race, and |
| 143 | it fails with the same words the queue-time check uses. |
| 144 | """ |
| 145 | model = _text(run.get("model")) |
| 146 | if ":" not in model: |
| 147 | return {"ok": False, "output": "", |
| 148 | "error": "No model was stamped on this run; it should have " |
| 149 | "been refused at queue time.", |
| 150 | "provider_cost_cents": 0, "provider_job_id": ""} |
| 151 | service, model_name = object_ai.split_model(model) |
| 152 | key = object_service_keys.get_service_key( |
| 153 | _text(run.get("owner_id")), service, base_dir=base) |
| 154 | if not key: |
| 155 | return {"ok": False, "output": "", |
| 156 | "error": f"No {service} API key is stored for " |
| 157 | f"{_text(run.get('owner_id'))} -- it existed at queue " |
| 158 | f"time and is gone now. Store one and start a new " |
| 159 | f"run; this one was not charged.", |
| 160 | "provider_cost_cents": 0, "provider_job_id": ""} |
| 161 | |
| 162 | def send_http(url, headers, body): |
| 163 | request = urllib.request.Request(url, data=body, headers=dict(headers), |
| 164 | method="POST") |
| 165 | try: |
| 166 | with urllib.request.urlopen(request, timeout=timeout) as response: |
| 167 | return response.status, response.read() |
| 168 | except urllib.error.HTTPError as exc: |
| 169 | return exc.code, exc.read() |
| 170 | |
| 171 | def refuse_tool(name, arguments): |
| 172 | # No tools are offered, so this cannot be reached; refusing rather |
| 173 | # than passing silently keeps that a fact instead of an assumption. |
| 174 | return {"http_status": 400, |
| 175 | "response": {"error": "Template runs offer no tools."}} |
| 176 | |
| 177 | try: |
| 178 | result = object_ai.run_chat( |
| 179 | send_http=send_http, |
| 180 | dispatch_tool=refuse_tool, |
| 181 | service=service, |
| 182 | model=model_name, |
| 183 | key=key, |
| 184 | message=_text(run.get("rendered_body")), |
| 185 | system=_text(run.get("instructions")) or None, |
| 186 | ) |
| 187 | except object_ai.AIProviderError as exc: |
| 188 | return {"ok": False, "output": "", |
| 189 | "error": f"{service} refused the request: {str(exc)[:300]}", |
| 190 | "provider_cost_cents": 0, "provider_job_id": ""} |
| 191 | except Exception as exc: |
| 192 | return {"ok": False, "output": "", |
| 193 | "error": f"The provider call failed: {str(exc)[:300]}", |
| 194 | "provider_cost_cents": 0, "provider_job_id": ""} |
| 195 | |
| 196 | usage = result.get("usage") or {} |
| 197 | cost = 0 |
| 198 | try: |
| 199 | prices = object_records.read_collection_records("ai_prices", |
| 200 | base_dir=base) |
| 201 | row = object_ai.select_price_row(prices, provider=service, |
| 202 | model=model_name) |
| 203 | computed = object_ai.compute_cost_cents( |
| 204 | _int(usage.get("input_tokens")), _int(usage.get("output_tokens")), |
| 205 | row) |
| 206 | cost = computed if computed is not None else 0 |
| 207 | except Exception: |
| 208 | cost = 0 # no price table is expected, not an error |
| 209 | |
| 210 | return {"ok": True, "output": _text(result.get("reply")), |
| 211 | "error": "", "provider_cost_cents": cost, "provider_job_id": ""} |
| 212 | |
| 213 | |
| 214 | HANDLERS = { |
| 215 | "echo": _handle_echo, |
| 216 | "ai_text": _handle_ai_text, |
| 217 | } |
| 218 | |
| 219 | #: name -> (submit, poll). Empty until a media handler lands; the engine |
| 220 | #: below is what that handler will plug into, and it is tested with a |
| 221 | #: stand-in so the money rule is proven before a provider rides on it. |
| 222 | ASYNC_HANDLERS = {} |
| 223 | |
| 224 | |
| 225 | # === the engine ============================================================== |
| 226 | |
| 227 | def _settle(base, run, *, succeeded): |
| 228 | """Write the run's settlement, exactly once, money before status. |
| 229 | |
| 230 | Returns a note for the pass report. Reads the entries fresh and checks |
| 231 | the release marker -- never run.status -- because the crash this |
| 232 | protects is the one between money and status. |
| 233 | """ |
| 234 | try: |
| 235 | entries = object_records.read_collection_records("wallet_entries", |
| 236 | base_dir=base) |
| 237 | except Exception: |
| 238 | entries = [] |
| 239 | if object_template_runs.already_settled(run.get("id"), entries): |
| 240 | return "already settled" |
| 241 | |
| 242 | for entry in object_template_runs.settlement(run, succeeded=succeeded): |
| 243 | object_records.create_collection_record( |
| 244 | "wallet_entries", entry, base_dir=base, actor=ACTOR) |
| 245 | return "settled" |
| 246 | |
| 247 | |
| 248 | def _claim(base, run): |
| 249 | """Take exclusive ownership of a queued run, or return None if |
| 250 | somebody else got there first. |
| 251 | |
| 252 | THE claim is a compare-and-set (63, plan/vocabulary/63-concurrency-spec.md), |
| 253 | not a plain write, and this is the difference between one provider |
| 254 | call and two. Without the precondition: two passes -- the minutely |
| 255 | schedule and an operator pressing Run Now -- both read the same row as |
| 256 | `queued`, both write `claimed` (the transition check is a no-op when |
| 257 | old == new), and both execute. Settlement is idempotent so the USER is |
| 258 | charged once, which is exactly what makes it dangerous: the duplicate |
| 259 | lands on OUR side of the ledger, as a second provider bill nothing in |
| 260 | the money model was watching. |
| 261 | |
| 262 | `expected_rev` closes it inside the write lock: the second pass's rev |
| 263 | no longer matches the row the first pass just claimed, so it 409s and |
| 264 | never runs the handler. Same primitive, same idiom as |
| 265 | app-timers/objects/site/timer_actions.py, and the same one an agent |
| 266 | uses to claim a task. |
| 267 | """ |
| 268 | return object_records.update_collection_record( |
| 269 | RUNS, _text(run.get("id")), |
| 270 | {"status": "claimed", "claimed_by": ACTOR, |
| 271 | "claimed_at": _now(), "heartbeat_at": _now()}, |
| 272 | base_dir=base, actor=ACTOR, |
| 273 | expected_rev=object_records.compute_record_rev(run)) |
| 274 | |
| 275 | |
| 276 | def _submit_one(base, run, timeout, pair): |
| 277 | """Claim a run, hand it to the provider, and leave it `running`. |
| 278 | |
| 279 | The pass ends here. Nothing is settled, because nothing has finished |
| 280 | -- and the hold placed at queue time stays exactly where it is, which |
| 281 | is the whole reason a hold exists rather than a charge on completion. |
| 282 | """ |
| 283 | run_id = _text(run.get("id")) |
| 284 | submit, _poll = pair |
| 285 | try: |
| 286 | _claim(base, run) |
| 287 | except object_records.VersionConflictError: |
| 288 | return {"run_id": run_id, "handler": _text(run.get("handler")), |
| 289 | "status": "lost_claim", "settlement": "not ours", "error": ""} |
| 290 | |
| 291 | try: |
| 292 | outcome = submit(run, base, timeout) |
| 293 | except Exception as exc: |
| 294 | outcome = {"provider_job_id": "", |
| 295 | "error": f"Submission crashed: {str(exc)[:300]}"} |
| 296 | |
| 297 | job_id = _text(outcome.get("provider_job_id")) |
| 298 | if not job_id: |
| 299 | # Nothing reached the provider, so nothing was spent: this settles |
| 300 | # as an ordinary failure and the hold comes back in full. |
| 301 | settled = _settle(base, run, succeeded=False) |
| 302 | object_records.update_collection_record( |
| 303 | RUNS, run_id, |
| 304 | {"status": "failed", |
| 305 | "error": _text(outcome.get("error")) or "Submission returned no job id.", |
| 306 | "finished_at": _now(), "heartbeat_at": _now()}, |
| 307 | base_dir=base, actor=ACTOR) |
| 308 | return {"run_id": run_id, "handler": _text(run.get("handler")), |
| 309 | "status": "failed", "settlement": settled, |
| 310 | "error": _text(outcome.get("error"))[:160]} |
| 311 | |
| 312 | # THE ORDERING THAT MATTERS: the job id is written before anything |
| 313 | # else can happen to this run. Once a provider is holding the job, |
| 314 | # the id is the only evidence that money may have been spent, and |
| 315 | # poll_disposition reads it to refuse to abandon the run. A crash |
| 316 | # between the call and this write is the one case that loses the |
| 317 | # evidence, which is why submit returns the id rather than writing |
| 318 | # it -- the window is as small as it can be made. |
| 319 | object_records.update_collection_record( |
| 320 | RUNS, run_id, |
| 321 | {"status": object_template_runs.RUNNING_STATUS, |
| 322 | "provider_job_id": job_id, "heartbeat_at": _now()}, |
| 323 | base_dir=base, actor=ACTOR) |
| 324 | return {"run_id": run_id, "handler": _text(run.get("handler")), |
| 325 | "status": "running", "settlement": "held", "error": "", |
| 326 | "provider_job_id": job_id} |
| 327 | |
| 328 | |
| 329 | def _poll_one(base, run, timeout, pair): |
| 330 | """Ask the provider whether a running job is done. |
| 331 | |
| 332 | Not done is not an error: it touches the heartbeat and returns, which |
| 333 | is what stops the sweeper writing off a job that is legitimately |
| 334 | still running. That heartbeat is the entire reason |
| 335 | runner.stale_seconds must exceed the POLL interval rather than the |
| 336 | job duration. |
| 337 | """ |
| 338 | run_id = _text(run.get("id")) |
| 339 | _submit, poll = pair |
| 340 | try: |
| 341 | outcome = poll(run, base, timeout) |
| 342 | except Exception as exc: |
| 343 | # A failed poll is not a failed job. The provider still has it, |
| 344 | # so touch the heartbeat and try again next pass; only |
| 345 | # max_run_seconds ends this loop, and it ends it at a human. |
| 346 | object_records.update_collection_record( |
| 347 | RUNS, run_id, {"heartbeat_at": _now()}, base_dir=base, actor=ACTOR) |
| 348 | return {"run_id": run_id, "handler": _text(run.get("handler")), |
| 349 | "status": "running", "settlement": "held", |
| 350 | "error": f"Poll failed, still running: {str(exc)[:160]}"} |
| 351 | |
| 352 | if not outcome.get("done"): |
| 353 | object_records.update_collection_record( |
| 354 | RUNS, run_id, {"heartbeat_at": _now()}, base_dir=base, actor=ACTOR) |
| 355 | return {"run_id": run_id, "handler": _text(run.get("handler")), |
| 356 | "status": "running", "settlement": "held", "error": ""} |
| 357 | |
| 358 | settled = _settle(base, run, succeeded=bool(outcome.get("ok"))) |
| 359 | final = "succeeded" if outcome.get("ok") else "failed" |
| 360 | object_records.update_collection_record( |
| 361 | RUNS, run_id, |
| 362 | {"status": final, |
| 363 | "output": _text(outcome.get("output"))[:20000], |
| 364 | "error": _text(outcome.get("error"))[:2000], |
| 365 | "provider_cost_cents": str(_int(outcome.get("provider_cost_cents"))), |
| 366 | "finished_at": _now(), "heartbeat_at": _now()}, |
| 367 | base_dir=base, actor=ACTOR) |
| 368 | return {"run_id": run_id, "handler": _text(run.get("handler")), |
| 369 | "status": final, "settlement": settled, |
| 370 | "error": _text(outcome.get("error"))[:160]} |
| 371 | |
| 372 | |
| 373 | def _execute_one(base, run, timeout): |
| 374 | run_id = _text(run.get("id")) |
| 375 | |
| 376 | try: |
| 377 | _claim(base, run) |
| 378 | except object_records.VersionConflictError: |
| 379 | # Another pass claimed it between our read and our write. Not an |
| 380 | # error and not worth a retry: the run is somebody else's now, and |
| 381 | # the whole point is that we did NOT call the provider. |
| 382 | return {"run_id": run_id, "handler": _text(run.get("handler")), |
| 383 | "status": "lost_claim", "settlement": "not ours", |
| 384 | "error": ""} |
| 385 | |
| 386 | handler = HANDLERS.get(_text(run.get("handler"))) |
| 387 | if handler is None: |
| 388 | outcome = {"ok": False, "output": "", |
| 389 | "error": f"No handler named {_text(run.get('handler'))!r} " |
| 390 | f"is installed on this server.", |
| 391 | "provider_cost_cents": 0, "provider_job_id": ""} |
| 392 | else: |
| 393 | try: |
| 394 | outcome = handler(run, base, timeout) |
| 395 | except Exception as exc: |
| 396 | outcome = {"ok": False, "output": "", |
| 397 | "error": f"The handler crashed: {str(exc)[:300]}", |
| 398 | "provider_cost_cents": 0, "provider_job_id": ""} |
| 399 | |
| 400 | settled = _settle(base, run, succeeded=bool(outcome["ok"])) |
| 401 | |
| 402 | final = "succeeded" if outcome["ok"] else "failed" |
| 403 | try: |
| 404 | object_records.update_collection_record( |
| 405 | RUNS, run_id, |
| 406 | {"status": final, |
| 407 | "output": _text(outcome.get("output"))[:20000], |
| 408 | "error": _text(outcome.get("error"))[:2000], |
| 409 | "provider_cost_cents": str(_int(outcome.get("provider_cost_cents"))), |
| 410 | "provider_job_id": _text(outcome.get("provider_job_id")), |
| 411 | "finished_at": _now(), "heartbeat_at": _now()}, |
| 412 | base_dir=base, actor=ACTOR) |
| 413 | except Exception as exc: |
| 414 | # The one way this write legitimately fails: the sweeper decided |
| 415 | # our heartbeat had stopped and marked the run `abandoned`, which |
| 416 | # the transition map declares terminal -- so there is no move from |
| 417 | # it to `succeeded`, by design. The work IS done and the money is |
| 418 | # already right (the sweeper released the hold; `_settle` found |
| 419 | # that marker and charged nothing), so this must not crash the |
| 420 | # pass and take the rest of the batch with it. Report it: a run |
| 421 | # that completed after being written off is a signal that |
| 422 | # runner.stale_seconds is too short for this handler. |
| 423 | return {"run_id": run_id, "handler": _text(run.get("handler")), |
| 424 | "status": "finished_after_sweep", "settlement": settled, |
| 425 | "error": (f"Completed, but the run had already been swept as " |
| 426 | f"abandoned, so its result could not be recorded " |
| 427 | f"({str(exc)[:120]}). Raise runner.stale_seconds.")} |
| 428 | |
| 429 | return {"run_id": run_id, "handler": _text(run.get("handler")), |
| 430 | "status": final, "settlement": settled, |
| 431 | "error": _text(outcome.get("error"))[:160]} |
| 432 | |
| 433 | |
| 434 | def EVENT(request): |
| 435 | """The daemon entry point. EVENT with POST aliased, because the |
| 436 | scheduler calls EVENT and declaring only POST is a pass that silently |
| 437 | never runs -- a real production bug here, twice.""" |
| 438 | request = request or {} |
| 439 | base = _base_dir() |
| 440 | |
| 441 | batch = max(1, _int(_setting(base, BATCH_SETTING, DEFAULT_BATCH), |
| 442 | DEFAULT_BATCH)) |
| 443 | timeout = max(5, _int(_setting(base, TIMEOUT_SETTING, DEFAULT_TIMEOUT), |
| 444 | DEFAULT_TIMEOUT)) |
| 445 | |
| 446 | try: |
| 447 | runs = object_records.read_collection_records(RUNS, base_dir=base) |
| 448 | except Exception: |
| 449 | return {"ok": True, "claimed": 0, |
| 450 | "note": "No template_runs collection on this box."} |
| 451 | |
| 452 | queued = sorted((row for row in runs |
| 453 | if _text(row.get("status")) == "queued"), |
| 454 | key=lambda row: _text(row.get("created_at"))) |
| 455 | |
| 456 | results = [] |
| 457 | for run in queued[:batch]: |
| 458 | pair = ASYNC_HANDLERS.get(_text(run.get("handler"))) |
| 459 | results.append(_submit_one(base, run, timeout, pair) if pair |
| 460 | else _execute_one(base, run, timeout)) |
| 461 | |
| 462 | # Then poll everything already at the provider. These are NOT new |
| 463 | # claims -- the run is already ours -- so they are reported |
| 464 | # separately and never counted as work this pass started. |
| 465 | polled = [] |
| 466 | for run in runs: |
| 467 | if _text(run.get("status")) != object_template_runs.RUNNING_STATUS: |
| 468 | continue |
| 469 | pair = ASYNC_HANDLERS.get(_text(run.get("handler"))) |
| 470 | if pair is None: |
| 471 | continue # handler uninstalled; the sweeper will escalate |
| 472 | polled.append(_poll_one(base, run, timeout, pair)) |
| 473 | # `claimed` counts runs this pass actually OWNED and executed. A lost |
| 474 | # claim is another pass's run, and counting it here would report two |
| 475 | # passes as having done the same work -- the exact confusion the |
| 476 | # compare-and-set exists to make impossible. |
| 477 | executed = [r for r in results if r["status"] != "lost_claim"] |
| 478 | lost = len(results) - len(executed) |
| 479 | return {"ok": True, "claimed": len(executed), "lost_claims": lost, |
| 480 | "queued_remaining": max(0, len(queued) - len(results)), |
| 481 | "polled": len(polled), "poll_results": polled, |
| 482 | "results": results} |
| 483 | |
| 484 | |
| 485 | POST = EVENT |