askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_runner_async.py · 15.1 KB · 373 lines
| 1 | """Asynchronous runs, and the one place the sweeper was a money bug. |
| 2 | |
| 3 | A Sora job takes minutes and q9 budgets thirty. Executing it inline would |
| 4 | block the pass, stall the batch, and eventually collide with the sweeper |
| 5 | writing off a job that is legitimately still running. So a run gains a |
| 6 | `running` status between claimed and terminal: submit, store the |
| 7 | provider job id, RETURN; later passes poll. |
| 8 | |
| 9 | **The money rule is the reason this file exists.** The original sweeper |
| 10 | released the hold on every stale run, which is exactly right when nothing |
| 11 | was submitted -- an unclaimed job that never started cost nothing. It is |
| 12 | exactly wrong once a provider is holding the job: a submitted job may |
| 13 | already have cost everything, and handing the money back on a timer means |
| 14 | paying for it ourselves. |
| 15 | |
| 16 | And the mirror is just as wrong. Charging a run whose outcome we never |
| 17 | learned bills a customer for something they may never have received. |
| 18 | |
| 19 | The box genuinely cannot tell which, so it refuses to guess: a submitted |
| 20 | run that never resolves goes to `needs_review` with the hold still on, |
| 21 | and a person decides. That is why needs_review is deliberately NOT in |
| 22 | TERMINAL_STATUSES -- that tuple is what drives the automatic release. |
| 23 | """ |
| 24 | |
| 25 | import pathlib |
| 26 | |
| 27 | import pytest |
| 28 | from conftest import stage_collection |
| 29 | |
| 30 | import object_execution |
| 31 | import object_records |
| 32 | import object_template_runs |
| 33 | import python_object_runtime |
| 34 | |
| 35 | REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] |
| 36 | RUNNER_OBJECTS = REPO_ROOT / "packages" / "app-runner" / "objects" |
| 37 | RUNTIME = python_object_runtime.PythonObjectRuntime() |
| 38 | |
| 39 | NOW = "2026-07-31T12:00:00Z" |
| 40 | |
| 41 | |
| 42 | def run_row(**fields): |
| 43 | row = {"id": "r1", "template_id": "tpl-image", "template_name": "Picture", |
| 44 | "status": "claimed", "handler": "ai_image", |
| 45 | "claimed_at": "2026-07-31T11:00:00Z", |
| 46 | "heartbeat_at": "2026-07-31T11:00:00Z", |
| 47 | "provider_job_id": "", "price_cents": "600", "wallet_id": "w1", |
| 48 | "owner_id": "dan"} |
| 49 | row.update(fields) |
| 50 | return row |
| 51 | |
| 52 | |
| 53 | # --- the decision, pure --------------------------------------------------------- |
| 54 | |
| 55 | def disposition(run, *, stale=300, max_run=3600): |
| 56 | return object_template_runs.poll_disposition( |
| 57 | run, now=NOW, stale_seconds=stale, max_run_seconds=max_run) |
| 58 | |
| 59 | |
| 60 | def test_a_stale_claim_that_never_submitted_is_abandoned(): |
| 61 | """Unchanged, and still right: nothing reached a provider, so nothing |
| 62 | was spent and the hold comes back in full.""" |
| 63 | assert disposition(run_row()) == "abandon" |
| 64 | |
| 65 | |
| 66 | def test_a_stale_claim_that_DID_submit_is_never_abandoned(): |
| 67 | """THE money bug. Same dead worker, same stopped heartbeat -- but a |
| 68 | provider is holding this job and may already have charged for it. |
| 69 | Abandoning is what releases the hold, so abandoning here would hand |
| 70 | back money we had already spent.""" |
| 71 | assert disposition(run_row(provider_job_id="job-abc")) == "poll" |
| 72 | |
| 73 | |
| 74 | def test_a_running_job_is_polled_not_swept(): |
| 75 | assert disposition(run_row(status="running", |
| 76 | provider_job_id="job-abc")) == "poll" |
| 77 | |
| 78 | |
| 79 | def test_a_running_job_past_the_ceiling_stops_for_a_person(): |
| 80 | """Polling has not resolved it for longer than any job should take. |
| 81 | It stops being a machine's decision -- and the hold stays put, since |
| 82 | both automatic answers are wrong.""" |
| 83 | old = run_row(status="running", provider_job_id="job-abc", |
| 84 | claimed_at="2026-07-31T10:00:00Z", |
| 85 | heartbeat_at="2026-07-31T11:59:00Z") |
| 86 | assert disposition(old, max_run=3600) == "review" |
| 87 | |
| 88 | |
| 89 | def test_a_live_claim_is_left_alone(): |
| 90 | assert disposition(run_row(heartbeat_at="2026-07-31T11:59:00Z")) is None |
| 91 | |
| 92 | |
| 93 | def test_terminal_and_reviewed_runs_are_never_reconsidered(): |
| 94 | for status in ("succeeded", "failed", "abandoned", "needs_review"): |
| 95 | assert disposition(run_row(status=status, |
| 96 | provider_job_id="job-abc")) is None, status |
| 97 | |
| 98 | |
| 99 | def test_a_job_id_outranks_a_status_a_crash_prevented(): |
| 100 | """A run still marked `claimed` but carrying a job id was submitted; |
| 101 | the status write simply never landed. The id is the evidence, and |
| 102 | evidence outranks a field a crash may have stopped.""" |
| 103 | assert disposition(run_row(status="claimed", |
| 104 | provider_job_id="job-abc")) == "poll" |
| 105 | |
| 106 | |
| 107 | # --- the sweeper, end to end ---------------------------------------------------- |
| 108 | |
| 109 | def setup_env(tmp_path, monkeypatch, runs, entries=()): |
| 110 | data_dir = tmp_path / "data" |
| 111 | for pkg, name in (("app-runner", "template_runs"), |
| 112 | ("app-templates", "templates"), |
| 113 | ("app-billing", "wallets"), |
| 114 | ("app-billing", "wallet_entries"), |
| 115 | ("app-settings", "app_settings")): |
| 116 | stage_collection(data_dir, pkg, name) |
| 117 | monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir)) |
| 118 | object_records.create_collection_record( |
| 119 | "templates", {"id": "tpl-image", "name": "Picture", "handler": "ai_image", |
| 120 | "body": "a picture of {thing}", "owner_id": "dan"}, |
| 121 | base_dir=data_dir) |
| 122 | object_records.create_collection_record( |
| 123 | "wallets", {"id": "w1", "owner_id": "dan", "is_active": "true"}, |
| 124 | base_dir=data_dir) |
| 125 | for row in runs: |
| 126 | object_records.create_collection_record("template_runs", row, |
| 127 | base_dir=data_dir) |
| 128 | for entry in entries: |
| 129 | object_records.create_collection_record("wallet_entries", entry, |
| 130 | base_dir=data_dir) |
| 131 | return data_dir |
| 132 | |
| 133 | |
| 134 | def hold_for(run_id, amount=600): |
| 135 | return {"wallet_id": "w1", "amount_minor": str(-amount), "kind": "hold", |
| 136 | "description": f"Hold: {run_id}", |
| 137 | "generated_from": f"template_run/{run_id}/hold", "owner_id": "dan"} |
| 138 | |
| 139 | |
| 140 | def sweep(): |
| 141 | return object_execution.execute_object( |
| 142 | RUNTIME, |
| 143 | object_execution.ObjectExecutionRequest( |
| 144 | "system_run_sweeper", method="EVENT", payload={"now": NOW}), |
| 145 | roots=[RUNNER_OBJECTS]).result |
| 146 | |
| 147 | |
| 148 | def entries_for(data_dir, run_id): |
| 149 | return [e for e in object_records.read_collection_records( |
| 150 | "wallet_entries", base_dir=data_dir) |
| 151 | if f"template_run/{run_id}/" in str(e.get("generated_from"))] |
| 152 | |
| 153 | |
| 154 | def test_the_sweeper_releases_a_hold_for_a_run_that_never_submitted( |
| 155 | tmp_path, monkeypatch): |
| 156 | data_dir = setup_env(tmp_path, monkeypatch, [run_row()], |
| 157 | entries=[hold_for("r1")]) |
| 158 | result = sweep() |
| 159 | assert result["runs"] == ["r1"] |
| 160 | kinds = [e["kind"] for e in entries_for(data_dir, "r1")] |
| 161 | assert "release" in kinds |
| 162 | |
| 163 | |
| 164 | def test_the_sweeper_does_not_touch_the_hold_of_a_submitted_run( |
| 165 | tmp_path, monkeypatch): |
| 166 | """THE test. The worker died, the heartbeat stopped, and the money |
| 167 | stays exactly where it is because a provider is holding the job.""" |
| 168 | data_dir = setup_env(tmp_path, monkeypatch, |
| 169 | [run_row(provider_job_id="job-abc")], |
| 170 | entries=[hold_for("r1")]) |
| 171 | result = sweep() |
| 172 | assert result["runs"] == [] |
| 173 | assert result["needs_review"] == [] |
| 174 | |
| 175 | kinds = [e["kind"] for e in entries_for(data_dir, "r1")] |
| 176 | assert kinds == ["hold"] # no release, no debit |
| 177 | run = object_records.get_collection_record("template_runs", "r1", |
| 178 | base_dir=data_dir) |
| 179 | assert run["status"] == "claimed" # untouched, waiting to be polled |
| 180 | |
| 181 | |
| 182 | def test_an_unresolved_submitted_run_escalates_with_the_hold_intact( |
| 183 | tmp_path, monkeypatch): |
| 184 | data_dir = setup_env(tmp_path, monkeypatch, |
| 185 | [run_row(status="running", provider_job_id="job-abc", |
| 186 | claimed_at="2026-07-31T10:00:00Z")], |
| 187 | entries=[hold_for("r1")]) |
| 188 | result = sweep() |
| 189 | assert result["needs_review"] == ["r1"] |
| 190 | |
| 191 | run = object_records.get_collection_record("template_runs", "r1", |
| 192 | base_dir=data_dir) |
| 193 | assert run["status"] == "needs_review" |
| 194 | assert "may already have been charged" in run["error"] |
| 195 | |
| 196 | kinds = [e["kind"] for e in entries_for(data_dir, "r1")] |
| 197 | assert kinds == ["hold"] # still held: a person decides |
| 198 | |
| 199 | |
| 200 | def test_needs_review_is_not_terminal_so_nothing_auto_releases_it(): |
| 201 | """The property the whole design rests on. TERMINAL_STATUSES drives |
| 202 | the sweeper's automatic hold release; needs_review must never join |
| 203 | it, or the escalation quietly becomes the release it exists to |
| 204 | prevent.""" |
| 205 | assert object_template_runs.NEEDS_REVIEW_STATUS \ |
| 206 | not in object_template_runs.TERMINAL_STATUSES |
| 207 | |
| 208 | |
| 209 | # --- the engine: submit, poll, settle ------------------------------------------- |
| 210 | |
| 211 | @pytest.fixture |
| 212 | def async_handler(monkeypatch): |
| 213 | """A stand-in provider, so the money rule is proven before a real one |
| 214 | rides on it. `state` lets a test decide what the provider says.""" |
| 215 | import importlib.util |
| 216 | spec = importlib.util.spec_from_file_location( |
| 217 | "template_runner_mod", |
| 218 | RUNNER_OBJECTS / "system" / "template_runner.py") |
| 219 | module = importlib.util.module_from_spec(spec) |
| 220 | spec.loader.exec_module(module) |
| 221 | |
| 222 | state = {"submitted": [], "done": False, "ok": True, "fail_submit": False} |
| 223 | |
| 224 | def submit(run, base, timeout): |
| 225 | if state["fail_submit"]: |
| 226 | return {"provider_job_id": "", "error": "provider refused"} |
| 227 | state["submitted"].append(run["id"]) |
| 228 | return {"provider_job_id": f"job-{run['id']}", "error": ""} |
| 229 | |
| 230 | def poll(run, base, timeout): |
| 231 | if not state["done"]: |
| 232 | return {"done": False} |
| 233 | return {"done": True, "ok": state["ok"], "output": "a picture", |
| 234 | "error": "" if state["ok"] else "provider failed", |
| 235 | "provider_cost_cents": 12} |
| 236 | |
| 237 | module.ASYNC_HANDLERS["ai_image"] = (submit, poll) |
| 238 | return module, state |
| 239 | |
| 240 | |
| 241 | def pass_once(module): |
| 242 | return module.EVENT({}) |
| 243 | |
| 244 | |
| 245 | def test_a_submit_leaves_the_run_running_and_the_money_held( |
| 246 | tmp_path, monkeypatch, async_handler): |
| 247 | module, state = async_handler |
| 248 | data_dir = setup_env(tmp_path, monkeypatch, |
| 249 | [run_row(status="queued", claimed_at="", |
| 250 | heartbeat_at="")], |
| 251 | entries=[hold_for("r1")]) |
| 252 | result = pass_once(module) |
| 253 | assert result["results"][0]["status"] == "running" |
| 254 | |
| 255 | run = object_records.get_collection_record("template_runs", "r1", |
| 256 | base_dir=data_dir) |
| 257 | assert run["status"] == "running" |
| 258 | assert run["provider_job_id"] == "job-r1" |
| 259 | assert [e["kind"] for e in entries_for(data_dir, "r1")] == ["hold"] |
| 260 | |
| 261 | |
| 262 | def test_polling_an_unfinished_job_only_touches_the_heartbeat( |
| 263 | tmp_path, monkeypatch, async_handler): |
| 264 | """What stops the sweeper writing off a five-minute job.""" |
| 265 | module, state = async_handler |
| 266 | data_dir = setup_env(tmp_path, monkeypatch, |
| 267 | [run_row(status="queued", claimed_at="", |
| 268 | heartbeat_at="")], |
| 269 | entries=[hold_for("r1")]) |
| 270 | pass_once(module) |
| 271 | before = object_records.get_collection_record( |
| 272 | "template_runs", "r1", base_dir=data_dir)["heartbeat_at"] |
| 273 | |
| 274 | second = pass_once(module) |
| 275 | assert second["polled"] == 1 |
| 276 | after = object_records.get_collection_record("template_runs", "r1", |
| 277 | base_dir=data_dir) |
| 278 | assert after["status"] == "running" |
| 279 | assert after["heartbeat_at"] >= before |
| 280 | assert [e["kind"] for e in entries_for(data_dir, "r1")] == ["hold"] |
| 281 | |
| 282 | |
| 283 | def test_a_finished_job_settles_on_a_later_pass( |
| 284 | tmp_path, monkeypatch, async_handler): |
| 285 | module, state = async_handler |
| 286 | data_dir = setup_env(tmp_path, monkeypatch, |
| 287 | [run_row(status="queued", claimed_at="", |
| 288 | heartbeat_at="")], |
| 289 | entries=[hold_for("r1")]) |
| 290 | pass_once(module) |
| 291 | state["done"] = True |
| 292 | pass_once(module) |
| 293 | |
| 294 | run = object_records.get_collection_record("template_runs", "r1", |
| 295 | base_dir=data_dir) |
| 296 | assert run["status"] == "succeeded" |
| 297 | assert run["output"] == "a picture" |
| 298 | assert run["provider_cost_cents"] == "12" |
| 299 | |
| 300 | kinds = [e["kind"] for e in entries_for(data_dir, "r1")] |
| 301 | assert kinds == ["hold", "release", "debit"] |
| 302 | |
| 303 | |
| 304 | def test_a_job_that_fails_at_the_provider_costs_the_user_nothing( |
| 305 | tmp_path, monkeypatch, async_handler): |
| 306 | module, state = async_handler |
| 307 | data_dir = setup_env(tmp_path, monkeypatch, |
| 308 | [run_row(status="queued", claimed_at="", |
| 309 | heartbeat_at="")], |
| 310 | entries=[hold_for("r1")]) |
| 311 | pass_once(module) |
| 312 | state["done"], state["ok"] = True, False |
| 313 | pass_once(module) |
| 314 | |
| 315 | kinds = [e["kind"] for e in entries_for(data_dir, "r1")] |
| 316 | assert kinds == ["hold", "release"] # released, never debited |
| 317 | |
| 318 | |
| 319 | def test_a_submission_that_never_reached_the_provider_refunds_in_full( |
| 320 | tmp_path, monkeypatch, async_handler): |
| 321 | """No job id means nothing was spent, so this is an ordinary failure |
| 322 | and the hold comes straight back.""" |
| 323 | module, state = async_handler |
| 324 | state["fail_submit"] = True |
| 325 | data_dir = setup_env(tmp_path, monkeypatch, |
| 326 | [run_row(status="queued", claimed_at="", |
| 327 | heartbeat_at="")], |
| 328 | entries=[hold_for("r1")]) |
| 329 | pass_once(module) |
| 330 | |
| 331 | run = object_records.get_collection_record("template_runs", "r1", |
| 332 | base_dir=data_dir) |
| 333 | assert run["status"] == "failed" |
| 334 | assert [e["kind"] for e in entries_for(data_dir, "r1")] == ["hold", "release"] |
| 335 | |
| 336 | |
| 337 | def test_a_failing_poll_is_not_a_failing_job( |
| 338 | tmp_path, monkeypatch, async_handler): |
| 339 | """A network blip while asking must not fail a job the provider is |
| 340 | still working on -- and must not settle anything.""" |
| 341 | module, state = async_handler |
| 342 | data_dir = setup_env(tmp_path, monkeypatch, |
| 343 | [run_row(status="queued", claimed_at="", |
| 344 | heartbeat_at="")], |
| 345 | entries=[hold_for("r1")]) |
| 346 | pass_once(module) |
| 347 | |
| 348 | def exploding_poll(run, base, timeout): |
| 349 | raise RuntimeError("connection reset") |
| 350 | |
| 351 | module.ASYNC_HANDLERS["ai_image"] = ( |
| 352 | module.ASYNC_HANDLERS["ai_image"][0], exploding_poll) |
| 353 | result = pass_once(module) |
| 354 | |
| 355 | assert result["poll_results"][0]["status"] == "running" |
| 356 | run = object_records.get_collection_record("template_runs", "r1", |
| 357 | base_dir=data_dir) |
| 358 | assert run["status"] == "running" |
| 359 | assert [e["kind"] for e in entries_for(data_dir, "r1")] == ["hold"] |
| 360 | |
| 361 | |
| 362 | def test_a_sync_handler_still_runs_inline(tmp_path, monkeypatch, async_handler): |
| 363 | """Both shapes coexist: echo has no async pair and must not be |
| 364 | dragged through submit/poll.""" |
| 365 | module, state = async_handler |
| 366 | data_dir = setup_env(tmp_path, monkeypatch, |
| 367 | [run_row(id="r2", status="queued", handler="echo", |
| 368 | price_cents="0", claimed_at="", |
| 369 | heartbeat_at="", rendered_body="hello")]) |
| 370 | result = pass_once(module) |
| 371 | assert result["results"][0]["status"] == "succeeded" |
| 372 | assert object_records.get_collection_record( |
| 373 | "template_runs", "r2", base_dir=data_dir)["output"] == "hello" |