Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_runner_async.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_runner_async.py · 15.1 KB · 373 lines
1"""Asynchronous runs, and the one place the sweeper was a money bug.
2
3A Sora job takes minutes and q9 budgets thirty. Executing it inline would
4block the pass, stall the batch, and eventually collide with the sweeper
5writing off a job that is legitimately still running. So a run gains a
6`running` status between claimed and terminal: submit, store the
7provider job id, RETURN; later passes poll.
8
9**The money rule is the reason this file exists.** The original sweeper
10released the hold on every stale run, which is exactly right when nothing
11was submitted -- an unclaimed job that never started cost nothing. It is
12exactly wrong once a provider is holding the job: a submitted job may
13already have cost everything, and handing the money back on a timer means
14paying for it ourselves.
15
16And the mirror is just as wrong. Charging a run whose outcome we never
17learned bills a customer for something they may never have received.
18
19The box genuinely cannot tell which, so it refuses to guess: a submitted
20run that never resolves goes to `needs_review` with the hold still on,
21and a person decides. That is why needs_review is deliberately NOT in
22TERMINAL_STATUSES -- that tuple is what drives the automatic release.
23"""
24
25import pathlib
26
27import pytest
28from conftest import stage_collection
29
30import object_execution
31import object_records
32import object_template_runs
33import python_object_runtime
34
35REPO_ROOT = pathlib.Path(__file__).resolve().parents[1]
36RUNNER_OBJECTS = REPO_ROOT / "packages" / "app-runner" / "objects"
37RUNTIME = python_object_runtime.PythonObjectRuntime()
38
39NOW = "2026-07-31T12:00:00Z"
40
41
42def run_row(**fields):
43 row = {"id": "r1", "template_id": "tpl-image", "template_name": "Picture",
44 "status": "claimed", "handler": "ai_image",
45 "claimed_at": "2026-07-31T11:00:00Z",
46 "heartbeat_at": "2026-07-31T11:00:00Z",
47 "provider_job_id": "", "price_cents": "600", "wallet_id": "w1",
48 "owner_id": "dan"}
49 row.update(fields)
50 return row
51
52
53# --- the decision, pure ---------------------------------------------------------
54
55def disposition(run, *, stale=300, max_run=3600):
56 return object_template_runs.poll_disposition(
57 run, now=NOW, stale_seconds=stale, max_run_seconds=max_run)
58
59
60def test_a_stale_claim_that_never_submitted_is_abandoned():
61 """Unchanged, and still right: nothing reached a provider, so nothing
62 was spent and the hold comes back in full."""
63 assert disposition(run_row()) == "abandon"
64
65
66def test_a_stale_claim_that_DID_submit_is_never_abandoned():
67 """THE money bug. Same dead worker, same stopped heartbeat -- but a
68 provider is holding this job and may already have charged for it.
69 Abandoning is what releases the hold, so abandoning here would hand
70 back money we had already spent."""
71 assert disposition(run_row(provider_job_id="job-abc")) == "poll"
72
73
74def test_a_running_job_is_polled_not_swept():
75 assert disposition(run_row(status="running",
76 provider_job_id="job-abc")) == "poll"
77
78
79def test_a_running_job_past_the_ceiling_stops_for_a_person():
80 """Polling has not resolved it for longer than any job should take.
81 It stops being a machine's decision -- and the hold stays put, since
82 both automatic answers are wrong."""
83 old = run_row(status="running", provider_job_id="job-abc",
84 claimed_at="2026-07-31T10:00:00Z",
85 heartbeat_at="2026-07-31T11:59:00Z")
86 assert disposition(old, max_run=3600) == "review"
87
88
89def test_a_live_claim_is_left_alone():
90 assert disposition(run_row(heartbeat_at="2026-07-31T11:59:00Z")) is None
91
92
93def test_terminal_and_reviewed_runs_are_never_reconsidered():
94 for status in ("succeeded", "failed", "abandoned", "needs_review"):
95 assert disposition(run_row(status=status,
96 provider_job_id="job-abc")) is None, status
97
98
99def test_a_job_id_outranks_a_status_a_crash_prevented():
100 """A run still marked `claimed` but carrying a job id was submitted;
101 the status write simply never landed. The id is the evidence, and
102 evidence outranks a field a crash may have stopped."""
103 assert disposition(run_row(status="claimed",
104 provider_job_id="job-abc")) == "poll"
105
106
107# --- the sweeper, end to end ----------------------------------------------------
108
109def setup_env(tmp_path, monkeypatch, runs, entries=()):
110 data_dir = tmp_path / "data"
111 for pkg, name in (("app-runner", "template_runs"),
112 ("app-templates", "templates"),
113 ("app-billing", "wallets"),
114 ("app-billing", "wallet_entries"),
115 ("app-settings", "app_settings")):
116 stage_collection(data_dir, pkg, name)
117 monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir))
118 object_records.create_collection_record(
119 "templates", {"id": "tpl-image", "name": "Picture", "handler": "ai_image",
120 "body": "a picture of {thing}", "owner_id": "dan"},
121 base_dir=data_dir)
122 object_records.create_collection_record(
123 "wallets", {"id": "w1", "owner_id": "dan", "is_active": "true"},
124 base_dir=data_dir)
125 for row in runs:
126 object_records.create_collection_record("template_runs", row,
127 base_dir=data_dir)
128 for entry in entries:
129 object_records.create_collection_record("wallet_entries", entry,
130 base_dir=data_dir)
131 return data_dir
132
133
134def hold_for(run_id, amount=600):
135 return {"wallet_id": "w1", "amount_minor": str(-amount), "kind": "hold",
136 "description": f"Hold: {run_id}",
137 "generated_from": f"template_run/{run_id}/hold", "owner_id": "dan"}
138
139
140def sweep():
141 return object_execution.execute_object(
142 RUNTIME,
143 object_execution.ObjectExecutionRequest(
144 "system_run_sweeper", method="EVENT", payload={"now": NOW}),
145 roots=[RUNNER_OBJECTS]).result
146
147
148def entries_for(data_dir, run_id):
149 return [e for e in object_records.read_collection_records(
150 "wallet_entries", base_dir=data_dir)
151 if f"template_run/{run_id}/" in str(e.get("generated_from"))]
152
153
154def test_the_sweeper_releases_a_hold_for_a_run_that_never_submitted(
155 tmp_path, monkeypatch):
156 data_dir = setup_env(tmp_path, monkeypatch, [run_row()],
157 entries=[hold_for("r1")])
158 result = sweep()
159 assert result["runs"] == ["r1"]
160 kinds = [e["kind"] for e in entries_for(data_dir, "r1")]
161 assert "release" in kinds
162
163
164def test_the_sweeper_does_not_touch_the_hold_of_a_submitted_run(
165 tmp_path, monkeypatch):
166 """THE test. The worker died, the heartbeat stopped, and the money
167 stays exactly where it is because a provider is holding the job."""
168 data_dir = setup_env(tmp_path, monkeypatch,
169 [run_row(provider_job_id="job-abc")],
170 entries=[hold_for("r1")])
171 result = sweep()
172 assert result["runs"] == []
173 assert result["needs_review"] == []
174
175 kinds = [e["kind"] for e in entries_for(data_dir, "r1")]
176 assert kinds == ["hold"] # no release, no debit
177 run = object_records.get_collection_record("template_runs", "r1",
178 base_dir=data_dir)
179 assert run["status"] == "claimed" # untouched, waiting to be polled
180
181
182def test_an_unresolved_submitted_run_escalates_with_the_hold_intact(
183 tmp_path, monkeypatch):
184 data_dir = setup_env(tmp_path, monkeypatch,
185 [run_row(status="running", provider_job_id="job-abc",
186 claimed_at="2026-07-31T10:00:00Z")],
187 entries=[hold_for("r1")])
188 result = sweep()
189 assert result["needs_review"] == ["r1"]
190
191 run = object_records.get_collection_record("template_runs", "r1",
192 base_dir=data_dir)
193 assert run["status"] == "needs_review"
194 assert "may already have been charged" in run["error"]
195
196 kinds = [e["kind"] for e in entries_for(data_dir, "r1")]
197 assert kinds == ["hold"] # still held: a person decides
198
199
200def test_needs_review_is_not_terminal_so_nothing_auto_releases_it():
201 """The property the whole design rests on. TERMINAL_STATUSES drives
202 the sweeper's automatic hold release; needs_review must never join
203 it, or the escalation quietly becomes the release it exists to
204 prevent."""
205 assert object_template_runs.NEEDS_REVIEW_STATUS \
206 not in object_template_runs.TERMINAL_STATUSES
207
208
209# --- the engine: submit, poll, settle -------------------------------------------
210
211@pytest.fixture
212def async_handler(monkeypatch):
213 """A stand-in provider, so the money rule is proven before a real one
214 rides on it. `state` lets a test decide what the provider says."""
215 import importlib.util
216 spec = importlib.util.spec_from_file_location(
217 "template_runner_mod",
218 RUNNER_OBJECTS / "system" / "template_runner.py")
219 module = importlib.util.module_from_spec(spec)
220 spec.loader.exec_module(module)
221
222 state = {"submitted": [], "done": False, "ok": True, "fail_submit": False}
223
224 def submit(run, base, timeout):
225 if state["fail_submit"]:
226 return {"provider_job_id": "", "error": "provider refused"}
227 state["submitted"].append(run["id"])
228 return {"provider_job_id": f"job-{run['id']}", "error": ""}
229
230 def poll(run, base, timeout):
231 if not state["done"]:
232 return {"done": False}
233 return {"done": True, "ok": state["ok"], "output": "a picture",
234 "error": "" if state["ok"] else "provider failed",
235 "provider_cost_cents": 12}
236
237 module.ASYNC_HANDLERS["ai_image"] = (submit, poll)
238 return module, state
239
240
241def pass_once(module):
242 return module.EVENT({})
243
244
245def test_a_submit_leaves_the_run_running_and_the_money_held(
246 tmp_path, monkeypatch, async_handler):
247 module, state = async_handler
248 data_dir = setup_env(tmp_path, monkeypatch,
249 [run_row(status="queued", claimed_at="",
250 heartbeat_at="")],
251 entries=[hold_for("r1")])
252 result = pass_once(module)
253 assert result["results"][0]["status"] == "running"
254
255 run = object_records.get_collection_record("template_runs", "r1",
256 base_dir=data_dir)
257 assert run["status"] == "running"
258 assert run["provider_job_id"] == "job-r1"
259 assert [e["kind"] for e in entries_for(data_dir, "r1")] == ["hold"]
260
261
262def test_polling_an_unfinished_job_only_touches_the_heartbeat(
263 tmp_path, monkeypatch, async_handler):
264 """What stops the sweeper writing off a five-minute job."""
265 module, state = async_handler
266 data_dir = setup_env(tmp_path, monkeypatch,
267 [run_row(status="queued", claimed_at="",
268 heartbeat_at="")],
269 entries=[hold_for("r1")])
270 pass_once(module)
271 before = object_records.get_collection_record(
272 "template_runs", "r1", base_dir=data_dir)["heartbeat_at"]
273
274 second = pass_once(module)
275 assert second["polled"] == 1
276 after = object_records.get_collection_record("template_runs", "r1",
277 base_dir=data_dir)
278 assert after["status"] == "running"
279 assert after["heartbeat_at"] >= before
280 assert [e["kind"] for e in entries_for(data_dir, "r1")] == ["hold"]
281
282
283def test_a_finished_job_settles_on_a_later_pass(
284 tmp_path, monkeypatch, async_handler):
285 module, state = async_handler
286 data_dir = setup_env(tmp_path, monkeypatch,
287 [run_row(status="queued", claimed_at="",
288 heartbeat_at="")],
289 entries=[hold_for("r1")])
290 pass_once(module)
291 state["done"] = True
292 pass_once(module)
293
294 run = object_records.get_collection_record("template_runs", "r1",
295 base_dir=data_dir)
296 assert run["status"] == "succeeded"
297 assert run["output"] == "a picture"
298 assert run["provider_cost_cents"] == "12"
299
300 kinds = [e["kind"] for e in entries_for(data_dir, "r1")]
301 assert kinds == ["hold", "release", "debit"]
302
303
304def test_a_job_that_fails_at_the_provider_costs_the_user_nothing(
305 tmp_path, monkeypatch, async_handler):
306 module, state = async_handler
307 data_dir = setup_env(tmp_path, monkeypatch,
308 [run_row(status="queued", claimed_at="",
309 heartbeat_at="")],
310 entries=[hold_for("r1")])
311 pass_once(module)
312 state["done"], state["ok"] = True, False
313 pass_once(module)
314
315 kinds = [e["kind"] for e in entries_for(data_dir, "r1")]
316 assert kinds == ["hold", "release"] # released, never debited
317
318
319def test_a_submission_that_never_reached_the_provider_refunds_in_full(
320 tmp_path, monkeypatch, async_handler):
321 """No job id means nothing was spent, so this is an ordinary failure
322 and the hold comes straight back."""
323 module, state = async_handler
324 state["fail_submit"] = True
325 data_dir = setup_env(tmp_path, monkeypatch,
326 [run_row(status="queued", claimed_at="",
327 heartbeat_at="")],
328 entries=[hold_for("r1")])
329 pass_once(module)
330
331 run = object_records.get_collection_record("template_runs", "r1",
332 base_dir=data_dir)
333 assert run["status"] == "failed"
334 assert [e["kind"] for e in entries_for(data_dir, "r1")] == ["hold", "release"]
335
336
337def test_a_failing_poll_is_not_a_failing_job(
338 tmp_path, monkeypatch, async_handler):
339 """A network blip while asking must not fail a job the provider is
340 still working on -- and must not settle anything."""
341 module, state = async_handler
342 data_dir = setup_env(tmp_path, monkeypatch,
343 [run_row(status="queued", claimed_at="",
344 heartbeat_at="")],
345 entries=[hold_for("r1")])
346 pass_once(module)
347
348 def exploding_poll(run, base, timeout):
349 raise RuntimeError("connection reset")
350
351 module.ASYNC_HANDLERS["ai_image"] = (
352 module.ASYNC_HANDLERS["ai_image"][0], exploding_poll)
353 result = pass_once(module)
354
355 assert result["poll_results"][0]["status"] == "running"
356 run = object_records.get_collection_record("template_runs", "r1",
357 base_dir=data_dir)
358 assert run["status"] == "running"
359 assert [e["kind"] for e in entries_for(data_dir, "r1")] == ["hold"]
360
361
362def test_a_sync_handler_still_runs_inline(tmp_path, monkeypatch, async_handler):
363 """Both shapes coexist: echo has no async pair and must not be
364 dragged through submit/poll."""
365 module, state = async_handler
366 data_dir = setup_env(tmp_path, monkeypatch,
367 [run_row(id="r2", status="queued", handler="echo",
368 price_cents="0", claimed_at="",
369 heartbeat_at="", rendered_body="hello")])
370 result = pass_once(module)
371 assert result["results"][0]["status"] == "succeeded"
372 assert object_records.get_collection_record(
373 "template_runs", "r2", base_dir=data_dir)["output"] == "hello"