askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_template_runner.py · 28.9 KB · 698 lines
| 1 | """The template runner: hold the money, run the job, settle exactly once. |
| 2 | |
| 3 | Slice 1 of plan/template-runner-spec.md, and the money model is the part |
| 4 | that cannot be retrofitted, so it is the part this file leans on: |
| 5 | |
| 6 | THE GATE FIRES AT COMMITMENT. The hold is a negative wallet entry placed |
| 7 | at queue time through hook_wallet_entries, so a user with five cents |
| 8 | cannot queue four hundred runs and present the provider bill later. The |
| 9 | four-hundredth hold fails the gate. |
| 10 | |
| 11 | SETTLEMENT IS RELEASE + ORDINARY DEBIT, EXACTLY ONCE. Success releases |
| 12 | the hold in full and charges a plain debit for the stamped price; failure |
| 13 | releases alone. Idempotent by provenance marker checked against the |
| 14 | ENTRIES, never run.status, because the crash worth designing for is the |
| 15 | one between writing money and updating status. |
| 16 | |
| 17 | A STALE RUN IS NEVER RETRIED. The provider call is not idempotent and is |
| 18 | not ours; abandoned is a terminal status with no exits in the declared |
| 19 | transition map, and the sweeper releases the hold and stops. |
| 20 | """ |
| 21 | |
| 22 | import json |
| 23 | import pathlib |
| 24 | import shutil |
| 25 | |
| 26 | import pytest |
| 27 | from conftest import stage_collection |
| 28 | |
| 29 | import object_execution |
| 30 | import object_records |
| 31 | import object_template_runs |
| 32 | import python_object_runtime |
| 33 | |
| 34 | REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] |
| 35 | PACKAGES = REPO_ROOT / "packages" |
| 36 | RUNNER_OBJECTS = PACKAGES / "app-runner" / "objects" |
| 37 | BILLING_OBJECTS = PACKAGES / "app-billing" / "objects" |
| 38 | RUNTIME = python_object_runtime.PythonObjectRuntime() |
| 39 | |
| 40 | |
| 41 | # --- the pure module ----------------------------------------------------------- |
| 42 | |
| 43 | def template(**fields): |
| 44 | row = {"id": "t1", "name": "Greeting", "handler": "echo", |
| 45 | "body": "Hello {name}, welcome to {place}.", |
| 46 | "run_cost_cents": "50", |
| 47 | "schema": json.dumps({"properties": {"name": {"title": "Name"}, |
| 48 | "place": {}}, |
| 49 | "required": ["name"]})} |
| 50 | row.update({k: str(v) for k, v in fields.items()}) |
| 51 | return row |
| 52 | |
| 53 | |
| 54 | def test_placeholders_fill_from_the_form_and_unknown_ones_survive(): |
| 55 | """A brace in prose must not kill a run, and a silently emptied |
| 56 | placeholder is a prompt that quietly lost its subject -- the worst |
| 57 | outcome, because the provider answers SOMETHING and charges for it.""" |
| 58 | out = object_template_runs.render("Hi {name}, re: {TBD}", {"name": "Ada"}) |
| 59 | assert out == "Hi Ada, re: {TBD}" |
| 60 | |
| 61 | |
| 62 | def test_every_missing_required_field_is_reported_together(): |
| 63 | found = object_template_runs.problems(template(handler=""), {}) |
| 64 | joined = " ".join(found) |
| 65 | assert len(found) == 2, found |
| 66 | assert "declares no handler" in joined |
| 67 | assert "'Name' is required" in joined |
| 68 | |
| 69 | |
| 70 | def test_the_stamp_carries_everything_execution_needs(): |
| 71 | """The run must be immune to every future edit of the template -- |
| 72 | including re-rendering, because rendering reads the template and by |
| 73 | execution time the template may say something else.""" |
| 74 | stamped = object_template_runs.stamp(template(), {"name": "Ada", |
| 75 | "place": "the shop"}, |
| 76 | model="anthropic:claude-sonnet-4-5") |
| 77 | assert stamped["rendered_body"] == "Hello Ada, welcome to the shop." |
| 78 | assert stamped["price_cents"] == "50" |
| 79 | assert stamped["handler"] == "echo" |
| 80 | assert stamped["model"] == "anthropic:claude-sonnet-4-5" |
| 81 | assert json.loads(stamped["form_data"]) == {"name": "Ada", |
| 82 | "place": "the shop"} |
| 83 | |
| 84 | |
| 85 | def run_row(**fields): |
| 86 | row = {"id": "r1", "price_cents": "50", "wallet_id": "w1", |
| 87 | "owner_id": "dan", "template_name": "Greeting", |
| 88 | "status": "claimed"} |
| 89 | row.update({k: str(v) for k, v in fields.items()}) |
| 90 | return row |
| 91 | |
| 92 | |
| 93 | def test_success_settles_as_release_plus_an_ordinary_debit(): |
| 94 | entries = object_template_runs.settlement(run_row(), succeeded=True) |
| 95 | assert [e["kind"] for e in entries] == ["release", "debit"] |
| 96 | assert [int(e["amount_minor"]) for e in entries] == [50, -50] |
| 97 | assert entries[0]["generated_from"] == "template_run/r1/release" |
| 98 | assert entries[1]["generated_from"] == "template_run/r1/charge" |
| 99 | |
| 100 | |
| 101 | def test_failure_settles_as_release_alone(): |
| 102 | """The user pays nothing for a run that did not do the thing, |
| 103 | whatever it cost US at the provider.""" |
| 104 | entries = object_template_runs.settlement(run_row(), succeeded=False) |
| 105 | assert [e["kind"] for e in entries] == ["release"] |
| 106 | assert int(entries[0]["amount_minor"]) == 50 |
| 107 | |
| 108 | |
| 109 | def test_a_free_run_touches_no_wallet_at_all(): |
| 110 | assert object_template_runs.settlement(run_row(price_cents="0"), |
| 111 | succeeded=True) == [] |
| 112 | assert object_template_runs.hold_entry("r1", "w1", 0, "dan") is None |
| 113 | |
| 114 | |
| 115 | def test_the_charge_is_a_debit_and_the_release_is_not_a_refund(): |
| 116 | """The reason hold/release are the only new kinds: every report that |
| 117 | already reads debits keeps working, and a released hold never |
| 118 | masquerades as money returned to a customer.""" |
| 119 | entries = object_template_runs.settlement(run_row(), succeeded=True) |
| 120 | kinds = {e["kind"] for e in entries} |
| 121 | assert "refund" not in kinds |
| 122 | assert "debit" in kinds |
| 123 | |
| 124 | |
| 125 | def test_already_settled_reads_the_entries_and_never_the_status(): |
| 126 | entries = [{"generated_from": "template_run/r1/release"}] |
| 127 | assert object_template_runs.already_settled("r1", entries) is True |
| 128 | assert object_template_runs.already_settled("r2", entries) is False |
| 129 | |
| 130 | |
| 131 | def test_outstanding_holds_is_a_fold_over_the_entries(): |
| 132 | entries = [ |
| 133 | {"generated_from": "template_run/r1/hold", "amount_minor": "-50"}, |
| 134 | {"generated_from": "template_run/r2/hold", "amount_minor": "-30"}, |
| 135 | {"generated_from": "template_run/r1/release", "amount_minor": "50"}, |
| 136 | {"generated_from": "checkout/x", "amount_minor": "-99"}, |
| 137 | ] |
| 138 | assert object_template_runs.outstanding_holds(entries) == {"r2": 30} |
| 139 | |
| 140 | |
| 141 | def test_staleness_is_pure_and_only_claimed_runs_can_be_stale(): |
| 142 | now = "2026-07-26T12:00:00Z" |
| 143 | old = {"status": "claimed", "heartbeat_at": "2026-07-26T11:30:00Z"} |
| 144 | fresh = {"status": "claimed", "heartbeat_at": "2026-07-26T11:59:00Z"} |
| 145 | queued = {"status": "queued", "heartbeat_at": "2026-07-26T01:00:00Z"} |
| 146 | assert object_template_runs.is_stale(old, now=now, stale_seconds=900) |
| 147 | assert not object_template_runs.is_stale(fresh, now=now, stale_seconds=900) |
| 148 | assert not object_template_runs.is_stale(queued, now=now, stale_seconds=900) |
| 149 | |
| 150 | |
| 151 | # --- through the objects, money and all ---------------------------------------- |
| 152 | |
| 153 | @pytest.fixture |
| 154 | def data_dir(tmp_path, monkeypatch): |
| 155 | base = tmp_path / "data" |
| 156 | stage_collection(base, "app-runner", "template_runs") |
| 157 | stage_collection(base, "app-templates", "templates") |
| 158 | stage_collection(base, "app-billing", "wallets") |
| 159 | stage_collection(base, "app-billing", "wallet_entries") |
| 160 | stage_collection(base, "app-settings", "app_settings") |
| 161 | # Both roots merged into one tree and exported via the environment, |
| 162 | # the way test_promotions does it: action_run_template asks |
| 163 | # hook_wallet_entries through a FRESH in-process runtime (the same |
| 164 | # mechanism the shop uses), and that runtime resolves objects from the |
| 165 | # environment, not from this test's execute_object roots. |
| 166 | objects_root = tmp_path / "objects" |
| 167 | for source in (RUNNER_OBJECTS, BILLING_OBJECTS): |
| 168 | shutil.copytree(source, objects_root, dirs_exist_ok=True) |
| 169 | monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(objects_root)) |
| 170 | monkeypatch.setenv("DBBASIC_DATA_DIR", str(base)) |
| 171 | return base |
| 172 | |
| 173 | |
| 174 | def run_object(object_id, payload, *, method="POST"): |
| 175 | return object_execution.execute_object( |
| 176 | RUNTIME, |
| 177 | object_execution.ObjectExecutionRequest(object_id, method=method, |
| 178 | payload=payload)).result |
| 179 | |
| 180 | |
| 181 | def seed_template(data_dir, **fields): |
| 182 | return object_records.create_collection_record( |
| 183 | "templates", template(**fields), base_dir=data_dir) |
| 184 | |
| 185 | |
| 186 | def seed_wallet(data_dir, cents, wallet_id="w1", owner="dan"): |
| 187 | object_records.create_collection_record( |
| 188 | "wallets", {"id": wallet_id, "owner_id": owner, "kind": "balance", |
| 189 | "is_active": "true"}, |
| 190 | base_dir=data_dir) |
| 191 | if cents: |
| 192 | object_records.create_collection_record( |
| 193 | "wallet_entries", |
| 194 | {"wallet_id": wallet_id, "amount_minor": str(cents), |
| 195 | "kind": "topup", "owner_id": owner}, |
| 196 | base_dir=data_dir) |
| 197 | |
| 198 | |
| 199 | def queue(data_dir, **payload): |
| 200 | payload.setdefault("template_id", "t1") |
| 201 | payload.setdefault("form_data", {"name": "Ada", "place": "the shop"}) |
| 202 | payload.setdefault("_identity", {"user_id": "dan"}) |
| 203 | return run_object("action_run_template", payload) |
| 204 | |
| 205 | |
| 206 | def balance(data_dir, wallet_id="w1"): |
| 207 | return sum(int(row.get("amount_minor") or 0) |
| 208 | for row in object_records.read_collection_records( |
| 209 | "wallet_entries", base_dir=data_dir) |
| 210 | if row.get("wallet_id") == wallet_id) |
| 211 | |
| 212 | |
| 213 | def test_queueing_places_the_hold_and_stamps_the_terms(data_dir): |
| 214 | seed_template(data_dir) |
| 215 | seed_wallet(data_dir, 200) |
| 216 | |
| 217 | queued = queue(data_dir) |
| 218 | assert queued["ok"] is True and queued["held"] is True |
| 219 | |
| 220 | assert balance(data_dir) == 150 # 200 - 50 held |
| 221 | |
| 222 | run = object_records.read_collection_records("template_runs", |
| 223 | base_dir=data_dir)[0] |
| 224 | assert run["status"] == "queued" |
| 225 | assert run["rendered_body"] == "Hello Ada, welcome to the shop." |
| 226 | assert run["price_cents"] == "50" |
| 227 | assert run["idempotency_key"] # minted before any call |
| 228 | |
| 229 | |
| 230 | def test_the_gate_refuses_the_run_the_balance_cannot_cover(data_dir): |
| 231 | """THE reason holds exist: the gate is asked at commitment. Three runs |
| 232 | fit a 120-cent balance twice; the third is refused with the wallet |
| 233 | still intact and nothing queued.""" |
| 234 | seed_template(data_dir) |
| 235 | seed_wallet(data_dir, 120) |
| 236 | |
| 237 | assert queue(data_dir)["ok"] is True |
| 238 | assert queue(data_dir)["ok"] is True |
| 239 | refused = queue(data_dir) |
| 240 | assert refused["status"] == 402 |
| 241 | assert "Insufficient wallet balance" in refused["error"] |
| 242 | |
| 243 | assert balance(data_dir) == 20 # two holds, no third |
| 244 | runs = object_records.read_collection_records("template_runs", |
| 245 | base_dir=data_dir) |
| 246 | assert len(runs) == 2 |
| 247 | |
| 248 | |
| 249 | def test_editing_the_template_after_queueing_restates_nothing(data_dir): |
| 250 | """Doctrine #1, on runs: the stamped terms are the terms.""" |
| 251 | seed_template(data_dir) |
| 252 | seed_wallet(data_dir, 200) |
| 253 | queue(data_dir) |
| 254 | |
| 255 | object_records.update_collection_record( |
| 256 | "templates", "t1", |
| 257 | {"body": "Completely different {name}", "run_cost_cents": "9000"}, |
| 258 | base_dir=data_dir) |
| 259 | |
| 260 | result = run_object("system_template_runner", {}) |
| 261 | assert result["claimed"] == 1 |
| 262 | run = object_records.read_collection_records("template_runs", |
| 263 | base_dir=data_dir)[0] |
| 264 | assert run["status"] == "succeeded" |
| 265 | assert run["output"] == "Hello Ada, welcome to the shop." |
| 266 | assert balance(data_dir) == 150 # charged 50, not 9000 |
| 267 | |
| 268 | |
| 269 | def test_a_successful_run_settles_release_plus_debit_and_only_once(data_dir): |
| 270 | seed_template(data_dir) |
| 271 | seed_wallet(data_dir, 200) |
| 272 | queue(data_dir) |
| 273 | |
| 274 | run_object("system_template_runner", {}) |
| 275 | assert balance(data_dir) == 150 # 200 - 50 charged |
| 276 | |
| 277 | entries = object_records.read_collection_records("wallet_entries", |
| 278 | base_dir=data_dir) |
| 279 | kinds = sorted(e["kind"] for e in entries) |
| 280 | assert kinds == ["debit", "hold", "release", "topup"] |
| 281 | |
| 282 | # A second pass finds nothing queued and, crucially, re-settles nothing. |
| 283 | again = run_object("system_template_runner", {}) |
| 284 | assert again["claimed"] == 0 |
| 285 | assert balance(data_dir) == 150 |
| 286 | |
| 287 | |
| 288 | def test_a_failed_run_costs_the_user_nothing(data_dir): |
| 289 | seed_template(data_dir, handler="ai_text") # queue-time check passes... |
| 290 | seed_wallet(data_dir, 200) |
| 291 | object_records.create_collection_record( |
| 292 | "app_settings", {"key": "runner.ai_model", |
| 293 | "value": "anthropic:claude-sonnet-4-5"}, |
| 294 | base_dir=data_dir) |
| 295 | import object_service_keys |
| 296 | object_service_keys.set_service_key("dan", "anthropic", "sk-test-not-real", |
| 297 | base_dir=data_dir) |
| 298 | queue(data_dir) |
| 299 | import object_service_keys as keys |
| 300 | keys.remove_service_key("dan", "anthropic", base_dir=data_dir) |
| 301 | |
| 302 | result = run_object("system_template_runner", {}) # ...and execution fails |
| 303 | assert result["results"][0]["status"] == "failed" |
| 304 | |
| 305 | run = object_records.read_collection_records("template_runs", |
| 306 | base_dir=data_dir)[0] |
| 307 | assert run["status"] == "failed" |
| 308 | assert "API key" in run["error"] |
| 309 | assert balance(data_dir) == 200 # hold released in full |
| 310 | kinds = sorted(e["kind"] for e in object_records.read_collection_records( |
| 311 | "wallet_entries", base_dir=data_dir)) |
| 312 | assert "debit" not in kinds |
| 313 | |
| 314 | |
| 315 | def test_a_free_template_runs_without_a_wallet_existing_at_all(data_dir): |
| 316 | """Money machinery that runs for free jobs is machinery that fails |
| 317 | for free jobs -- so for a zero-price template there must be none.""" |
| 318 | seed_template(data_dir, run_cost_cents="0") |
| 319 | queued = queue(data_dir) |
| 320 | assert queued["ok"] is True and queued["held"] is False |
| 321 | |
| 322 | run_object("system_template_runner", {}) |
| 323 | run = object_records.read_collection_records("template_runs", |
| 324 | base_dir=data_dir)[0] |
| 325 | assert run["status"] == "succeeded" |
| 326 | assert not object_records.read_collection_records("wallet_entries", |
| 327 | base_dir=data_dir) |
| 328 | |
| 329 | |
| 330 | def test_an_ai_template_with_no_provider_is_refused_at_queue_time(data_dir): |
| 331 | """The capability boundary, refused while the person who can fix it |
| 332 | is looking at the screen -- not discovered on the daemon an hour |
| 333 | later. Nothing queued, nothing held.""" |
| 334 | seed_template(data_dir, handler="ai_text") |
| 335 | seed_wallet(data_dir, 200) |
| 336 | |
| 337 | refused = queue(data_dir) |
| 338 | assert refused["status"] == 409 |
| 339 | assert "runner.ai_model" in refused["error"] |
| 340 | assert balance(data_dir) == 200 |
| 341 | assert not object_records.read_collection_records("template_runs", |
| 342 | base_dir=data_dir) |
| 343 | |
| 344 | |
| 345 | def test_an_anonymous_run_is_refused_because_runs_cost_money(data_dir): |
| 346 | seed_template(data_dir) |
| 347 | refused = run_object("action_run_template", |
| 348 | {"template_id": "t1", "form_data": {"name": "x"}}) |
| 349 | assert refused["status"] == 401 |
| 350 | assert "open tap" in refused["error"] |
| 351 | |
| 352 | |
| 353 | def test_a_template_with_no_handler_is_a_document_not_a_job(data_dir): |
| 354 | seed_template(data_dir, handler="") |
| 355 | refused = queue(data_dir) |
| 356 | assert refused["status"] == 400 |
| 357 | assert "declares no handler" in refused["error"] |
| 358 | |
| 359 | |
| 360 | # --- the sweeper ---------------------------------------------------------------- |
| 361 | |
| 362 | def test_a_stale_run_is_abandoned_refunded_and_never_requeued(data_dir): |
| 363 | seed_template(data_dir) |
| 364 | seed_wallet(data_dir, 200) |
| 365 | queue(data_dir) |
| 366 | |
| 367 | run = object_records.read_collection_records("template_runs", |
| 368 | base_dir=data_dir)[0] |
| 369 | object_records.update_collection_record( |
| 370 | "template_runs", run["id"], |
| 371 | {"status": "claimed", "claimed_by": "a-runner-that-died", |
| 372 | "claimed_at": "2026-07-26T01:00:00Z", |
| 373 | "heartbeat_at": "2026-07-26T01:00:00Z"}, |
| 374 | base_dir=data_dir) |
| 375 | |
| 376 | result = run_object("system_run_sweeper", {"now": "2026-07-26T12:00:00Z"}) |
| 377 | assert result["swept"] == 1 |
| 378 | |
| 379 | swept = object_records.read_collection_records("template_runs", |
| 380 | base_dir=data_dir)[0] |
| 381 | assert swept["status"] == "abandoned" |
| 382 | assert "NOT retried" in swept["error"] |
| 383 | assert balance(data_dir) == 200 # the hold came back in full |
| 384 | |
| 385 | # And the runner never picks it up again: abandoned is not queued. |
| 386 | after = run_object("system_template_runner", {}) |
| 387 | assert after["claimed"] == 0 |
| 388 | |
| 389 | # Sweeping again releases nothing twice. |
| 390 | again = run_object("system_run_sweeper", {"now": "2026-07-26T13:00:00Z"}) |
| 391 | assert again["swept"] == 0 or all(r["hold"] == "already settled" |
| 392 | for r in again["runs"]) |
| 393 | assert balance(data_dir) == 200 |
| 394 | |
| 395 | |
| 396 | def test_abandoned_is_a_dead_end_in_the_declared_state_machine(): |
| 397 | """The transition map is the enforcement: no status may move a run |
| 398 | out of abandoned, so nothing -- not a bug, not a helpful operator |
| 399 | endpoint added later -- can quietly re-run a paid provider call.""" |
| 400 | schema = json.loads((PACKAGES / "app-runner" / "schemas" |
| 401 | / "template_runs.json").read_text()) |
| 402 | status = next(f for f in schema["fields"] if f["name"] == "status") |
| 403 | transitions = status["transitions"] |
| 404 | assert "abandoned" not in transitions # no exits |
| 405 | assert "succeeded" not in transitions |
| 406 | assert "failed" not in transitions |
| 407 | for moves in transitions.values(): |
| 408 | assert "queued" not in moves # nothing moves BACK to queued |
| 409 | |
| 410 | |
| 411 | def test_nothing_in_the_package_retries_a_run(): |
| 412 | """Named as a grep on the whole package, the same way app-notary pins |
| 413 | its no-delete rule: the risk is not today's code, it is somebody later |
| 414 | adding retry as an improvement to a paid non-idempotent call.""" |
| 415 | for path in (PACKAGES / "app-runner").rglob("*.py"): |
| 416 | source = path.read_text() |
| 417 | assert '"queued"}' not in source.replace("status\": \"queued", ""), path |
| 418 | # No update may ever set a run BACK to queued. |
| 419 | assert '{"status": "queued"' not in source, path |
| 420 | |
| 421 | |
| 422 | def test_the_runner_and_sweeper_declare_EVENT(): |
| 423 | for name in ("template_runner", "run_sweeper"): |
| 424 | source = (PACKAGES / "app-runner" / "objects" / "system" |
| 425 | / f"{name}.py").read_text() |
| 426 | assert "def EVENT(" in source, name |
| 427 | assert "POST = EVENT" in source, name |
| 428 | |
| 429 | |
| 430 | def test_wallet_entries_gained_hold_and_release_and_nothing_was_removed(): |
| 431 | schema = json.loads((PACKAGES / "app-billing" / "schemas" |
| 432 | / "wallet_entries.json").read_text()) |
| 433 | kind = next(f for f in schema["fields"] if f["name"] == "kind") |
| 434 | assert set(kind["enum"]) == {"topup", "auto_topup", "debit", "refund", |
| 435 | "promo", "adjustment", "hold", "release"} |
| 436 | |
| 437 | |
| 438 | # --- the claim race (63) -------------------------------------------------------- |
| 439 | # |
| 440 | # The bug this closes was shipped and deployed before it was found, and it |
| 441 | # is worth stating exactly: two passes -- the minutely schedule and an |
| 442 | # operator pressing Run Now -- both read the same row as `queued`, both |
| 443 | # wrote `claimed` (the transition check is a no-op when old == new), and |
| 444 | # both ran the handler. Settlement is idempotent, so the USER was charged |
| 445 | # once. That is what made it dangerous: the duplicate landed on OUR side |
| 446 | # of the ledger as a second provider bill, which nothing in the money |
| 447 | # model was watching. |
| 448 | |
| 449 | def _runner_module(): |
| 450 | import importlib.util |
| 451 | spec = importlib.util.spec_from_file_location( |
| 452 | "_tr", PACKAGES / "app-runner" / "objects" / "system" |
| 453 | / "template_runner.py") |
| 454 | module = importlib.util.module_from_spec(spec) |
| 455 | spec.loader.exec_module(module) |
| 456 | return module |
| 457 | |
| 458 | |
| 459 | def test_a_pass_holding_a_stale_row_loses_the_claim_and_never_executes(data_dir): |
| 460 | """Deterministic version: read a run, let somebody else move it, then |
| 461 | try to claim it with the row we read. The handler must not run.""" |
| 462 | seed_template(data_dir) |
| 463 | seed_wallet(data_dir, 200) |
| 464 | queue(data_dir) |
| 465 | |
| 466 | stale = object_records.read_collection_records("template_runs", |
| 467 | base_dir=data_dir)[0] |
| 468 | |
| 469 | # Another pass gets there first. |
| 470 | object_records.update_collection_record( |
| 471 | "template_runs", stale["id"], |
| 472 | {"status": "claimed", "claimed_by": "the-other-pass"}, |
| 473 | base_dir=data_dir) |
| 474 | |
| 475 | result = _runner_module()._execute_one(data_dir, stale, 30) |
| 476 | assert result["status"] == "lost_claim" |
| 477 | assert result["settlement"] == "not ours" |
| 478 | |
| 479 | run = object_records.read_collection_records("template_runs", |
| 480 | base_dir=data_dir)[0] |
| 481 | assert run["claimed_by"] == "the-other-pass" # not overwritten |
| 482 | assert run["output"] == "" # handler never ran |
| 483 | assert balance(data_dir) == 150 # still just the hold |
| 484 | |
| 485 | |
| 486 | def test_two_concurrent_passes_execute_a_run_exactly_once(data_dir): |
| 487 | """The integration proof. Before the fix this reported [1, 1] and two |
| 488 | executions; a provider handler would have been called twice and billed |
| 489 | twice. Cannot fail spuriously: if the threads happen to serialise, the |
| 490 | second pass simply finds nothing queued -- still one execution.""" |
| 491 | import threading |
| 492 | |
| 493 | seed_template(data_dir) |
| 494 | seed_wallet(data_dir, 200) |
| 495 | queue(data_dir) |
| 496 | |
| 497 | results = [] |
| 498 | threads = [threading.Thread( |
| 499 | target=lambda: results.append(run_object("system_template_runner", {}))) |
| 500 | for _ in range(2)] |
| 501 | for thread in threads: |
| 502 | thread.start() |
| 503 | for thread in threads: |
| 504 | thread.join() |
| 505 | |
| 506 | assert sum(r["claimed"] for r in results) == 1, results |
| 507 | |
| 508 | runs = object_records.read_collection_records("template_runs", |
| 509 | base_dir=data_dir) |
| 510 | assert len(runs) == 1 and runs[0]["status"] == "succeeded" |
| 511 | assert balance(data_dir) == 150 # charged exactly once |
| 512 | |
| 513 | |
| 514 | def test_the_sweeper_will_not_write_off_a_run_that_moved_under_it(data_dir): |
| 515 | """The sweeper races the runner it is sweeping. A run that heartbeats |
| 516 | between the staleness read and the write has changed the row, so the |
| 517 | write-off loses -- otherwise the sweeper hands back money for work |
| 518 | that was in the middle of succeeding.""" |
| 519 | seed_template(data_dir) |
| 520 | seed_wallet(data_dir, 200) |
| 521 | queue(data_dir) |
| 522 | |
| 523 | stale = object_records.read_collection_records("template_runs", |
| 524 | base_dir=data_dir)[0] |
| 525 | object_records.update_collection_record( |
| 526 | "template_runs", stale["id"], |
| 527 | {"status": "claimed", "claimed_by": "runner", |
| 528 | "claimed_at": "2026-07-26T01:00:00Z", |
| 529 | "heartbeat_at": "2026-07-26T01:00:00Z"}, |
| 530 | base_dir=data_dir) |
| 531 | |
| 532 | import object_records as records |
| 533 | with pytest.raises(records.VersionConflictError): |
| 534 | records.update_collection_record( |
| 535 | "template_runs", stale["id"], {"status": "abandoned"}, |
| 536 | base_dir=data_dir, |
| 537 | expected_rev=records.compute_record_rev(stale)) |
| 538 | |
| 539 | |
| 540 | def test_the_sweeper_repairs_a_hold_a_crash_left_behind(data_dir): |
| 541 | """Step 2 is driven by STATE, not by 'did step 1 just run'. A crash |
| 542 | between writing off a run and releasing its hold used to leave money |
| 543 | reserved forever, because a run already marked abandoned is no longer |
| 544 | stale and would never be looked at again.""" |
| 545 | seed_template(data_dir) |
| 546 | seed_wallet(data_dir, 200) |
| 547 | queue(data_dir) |
| 548 | assert balance(data_dir) == 150 # held |
| 549 | |
| 550 | run = object_records.read_collection_records("template_runs", |
| 551 | base_dir=data_dir)[0] |
| 552 | # Exactly the state a crash between the two steps leaves: terminal, |
| 553 | # with the hold still on. |
| 554 | object_records.update_collection_record( |
| 555 | "template_runs", run["id"], {"status": "abandoned"}, |
| 556 | base_dir=data_dir) |
| 557 | |
| 558 | result = run_object("system_run_sweeper", {"now": "2026-07-26T12:00:00Z"}) |
| 559 | assert run["id"] in result["holds_released"] |
| 560 | assert balance(data_dir) == 200 # money back, without a retry |
| 561 | |
| 562 | # And it is not released a second time. |
| 563 | again = run_object("system_run_sweeper", {"now": "2026-07-26T13:00:00Z"}) |
| 564 | assert again["holds_released"] == [] |
| 565 | assert balance(data_dir) == 200 |
| 566 | |
| 567 | |
| 568 | # --- task_id: one execution model, not a second one (q9's tasks.Work) ---------- |
| 569 | |
| 570 | def seed_task(data_dir, task_id="task-1", owner="dan", **fields): |
| 571 | stage_collection(data_dir, "app-tasks", "tasks") |
| 572 | row = {"id": task_id, "title": "Review the homepage", "owner_id": owner} |
| 573 | row.update(fields) |
| 574 | return object_records.create_collection_record("tasks", row, base_dir=data_dir) |
| 575 | |
| 576 | |
| 577 | def test_a_run_can_be_queued_against_a_task_the_caller_owns(data_dir): |
| 578 | seed_template(data_dir) |
| 579 | seed_wallet(data_dir, 200) |
| 580 | seed_task(data_dir, owner="dan") |
| 581 | |
| 582 | queued = queue(data_dir, task_id="task-1") |
| 583 | assert queued["ok"] is True |
| 584 | |
| 585 | run = object_records.read_collection_records("template_runs", |
| 586 | base_dir=data_dir)[0] |
| 587 | assert run["task_id"] == "task-1" |
| 588 | |
| 589 | |
| 590 | def test_task_id_is_entirely_optional_and_unchanged_by_default(data_dir): |
| 591 | """The regression that would matter most: a template runner that |
| 592 | could only run FROM a task would break every ad-hoc run (a SWOT from |
| 593 | the shell, a one-off from Talk) that has nothing to do with one.""" |
| 594 | seed_template(data_dir) |
| 595 | seed_wallet(data_dir, 200) |
| 596 | |
| 597 | queued = queue(data_dir) # no task_id at all |
| 598 | assert queued["ok"] is True |
| 599 | run = object_records.read_collection_records("template_runs", |
| 600 | base_dir=data_dir)[0] |
| 601 | assert run["task_id"] == "" |
| 602 | |
| 603 | |
| 604 | def test_a_run_cannot_be_linked_to_a_task_that_does_not_exist(data_dir): |
| 605 | seed_template(data_dir) |
| 606 | seed_wallet(data_dir, 200) |
| 607 | |
| 608 | result = queue(data_dir, task_id="no-such-task") |
| 609 | assert result["status"] == 404 |
| 610 | assert object_records.read_collection_records("template_runs", |
| 611 | base_dir=data_dir) == [] |
| 612 | |
| 613 | |
| 614 | def test_a_run_cannot_be_linked_to_someone_elses_task(data_dir): |
| 615 | """Refused at queue time, same 'refused while somebody is looking' |
| 616 | posture as the capability-boundary checks beside it -- letting the |
| 617 | link through would leak a stranger's task existence and status |
| 618 | through what the run settles against.""" |
| 619 | seed_template(data_dir) |
| 620 | seed_wallet(data_dir, 200) |
| 621 | seed_task(data_dir, task_id="not-mine", owner="mallory") |
| 622 | |
| 623 | result = queue(data_dir, task_id="not-mine") |
| 624 | assert result["status"] == 403 |
| 625 | assert object_records.read_collection_records("template_runs", |
| 626 | base_dir=data_dir) == [] |
| 627 | # And nothing was held either -- the check runs before the hold. |
| 628 | assert balance(data_dir) == 200 |
| 629 | |
| 630 | |
| 631 | def test_an_admin_can_link_a_run_to_anyones_task(data_dir): |
| 632 | seed_template(data_dir) |
| 633 | seed_task(data_dir, task_id="someones", owner="mallory") |
| 634 | # The admin's OWN wallet holds the money -- linking to someone else's |
| 635 | # task changes whose task this serves, never who pays for it. |
| 636 | seed_wallet(data_dir, 200, wallet_id="w-ops", owner="ops") |
| 637 | |
| 638 | result = queue(data_dir, task_id="someones", |
| 639 | **{"_identity": {"user_id": "ops", "is_admin": True}}) |
| 640 | assert result["ok"] is True, result |
| 641 | |
| 642 | |
| 643 | def test_multiple_attempts_against_one_task_are_just_multiple_runs(data_dir): |
| 644 | """No second history collection -- q9's Work stored attempts in its |
| 645 | own rows; here it is a fold over template_runs sharing a task_id, |
| 646 | the same doctrine as every other one-to-many relationship in this |
| 647 | system.""" |
| 648 | seed_template(data_dir) |
| 649 | seed_wallet(data_dir, 200) |
| 650 | seed_task(data_dir, owner="dan") |
| 651 | |
| 652 | queue(data_dir, task_id="task-1") |
| 653 | queue(data_dir, task_id="task-1") |
| 654 | |
| 655 | runs = [r for r in object_records.read_collection_records( |
| 656 | "template_runs", base_dir=data_dir) |
| 657 | if r.get("task_id") == "task-1"] |
| 658 | assert len(runs) == 2 |
| 659 | |
| 660 | |
| 661 | def test_a_run_never_mutates_the_tasks_own_status(data_dir): |
| 662 | """Deliberate. A run succeeding is not the same claim as 'this task |
| 663 | is finished' -- that is a human's call, or an agent's, looking at |
| 664 | the run's own output. Same 'the server notices, the person decides' |
| 665 | posture as system_reorder_check's suggestions.""" |
| 666 | seed_template(data_dir) |
| 667 | seed_wallet(data_dir, 200) |
| 668 | seed_task(data_dir, owner="dan", status="open") |
| 669 | |
| 670 | queue(data_dir, task_id="task-1") |
| 671 | |
| 672 | task = object_records.get_collection_record("tasks", "task-1", |
| 673 | base_dir=data_dir) |
| 674 | assert task["status"] == "open" |
| 675 | |
| 676 | |
| 677 | def test_the_task_detail_view_renders_the_runs_related_to_it(): |
| 678 | """No new UI: the same `related` block order->order_lines already |
| 679 | uses, so a task's attempt history costs zero new client code.""" |
| 680 | import csv |
| 681 | seed = (REPO_ROOT / "packages" / "app-tasks" / "seed" / "views.tsv") |
| 682 | rows = list(csv.DictReader(seed.read_text().splitlines(), delimiter="\t")) |
| 683 | detail = next(r for r in rows if r["id"] == "view_tasks_detail") |
| 684 | blocks = json.loads(detail["blocks"]) |
| 685 | related = next((b for b in blocks if b.get("kind") == "related"), None) |
| 686 | assert related is not None, "no related block on the task detail view" |
| 687 | assert related["collection"] == "template_runs" |
| 688 | assert related["fk_field"] == "task_id" |
| 689 | assert related["match"] == "$record_id" |
| 690 | |
| 691 | |
| 692 | def test_the_schema_declares_task_id_as_a_relation_to_tasks(): |
| 693 | schema = json.loads((REPO_ROOT / "packages" / "app-runner" / "schemas" |
| 694 | / "template_runs.json").read_text()) |
| 695 | field = next(f for f in schema["fields"] if f["name"] == "task_id") |
| 696 | assert field["type"] == "relation" |
| 697 | assert field["relation"]["collection"] == "tasks" |
| 698 | assert schema["version"] >= 2 |