Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_object_worker_pool.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_object_worker_pool.py · 17.0 KB · 524 lines
1import asyncio
2import time
3from pathlib import Path
4
5import pytest
6
7import object_execution
8import object_worker_pool
9from object_execution import ObjectExecutionRequest
10
11
12def write_object(path: Path, source: str) -> Path:
13 path.parent.mkdir(parents=True, exist_ok=True)
14 path.write_text(source)
15 return path
16
17
18# ---------------------------------------------------------------------------
19# Round-trip parity with execute_python_object_subprocess
20# ---------------------------------------------------------------------------
21
22
23def test_pool_execute_returns_success_result(tmp_path):
24 root = tmp_path / "objects"
25 write_object(
26 root / "basics" / "worker.py",
27 "def GET(request):\n return {'value': request['value']}\n",
28 )
29
30 async def scenario():
31 pool = object_worker_pool.WorkerPool(size=1)
32 try:
33 return await pool.execute(
34 ObjectExecutionRequest("basics_worker", payload={"value": 7}),
35 timeout_seconds=5,
36 roots=[root],
37 )
38 finally:
39 await pool.shutdown()
40
41 result = asyncio.run(scenario())
42
43 assert result.ok
44 assert result.object_id == "basics_worker"
45 assert result.method == "GET"
46 assert result.path == root / "basics" / "worker.py"
47 assert result.result == {"value": 7}
48 assert result.error is None
49 assert result.started_at.endswith("Z")
50 assert result.finished_at.endswith("Z")
51
52
53def test_pool_execute_matches_subprocess_success_shape(tmp_path):
54 root = tmp_path / "objects"
55 write_object(
56 root / "basics" / "worker.py",
57 "def GET(request):\n return {'value': request['value']}\n",
58 )
59
60 async def scenario():
61 pool = object_worker_pool.WorkerPool(size=1)
62 try:
63 return await pool.execute(
64 ObjectExecutionRequest("basics_worker", payload={"value": 7}),
65 timeout_seconds=5,
66 roots=[root],
67 )
68 finally:
69 await pool.shutdown()
70
71 pool_result = asyncio.run(scenario())
72 subprocess_result = object_execution.execute_python_object_subprocess(
73 ObjectExecutionRequest("basics_worker", payload={"value": 7}),
74 roots=[root],
75 timeout_seconds=5,
76 )
77
78 assert pool_result.ok == subprocess_result.ok
79 assert pool_result.result == subprocess_result.result
80 assert pool_result.object_id == subprocess_result.object_id
81 assert pool_result.method == subprocess_result.method
82 assert pool_result.path == subprocess_result.path
83
84
85def test_pool_execute_object_raises_preserves_error_type_and_message(tmp_path):
86 root = tmp_path / "objects"
87 write_object(
88 root / "basics" / "boom.py",
89 "def GET(request):\n raise ValueError('kaboom')\n",
90 )
91
92 async def scenario():
93 pool = object_worker_pool.WorkerPool(size=1)
94 try:
95 return await pool.execute(
96 ObjectExecutionRequest("basics_boom"),
97 timeout_seconds=5,
98 roots=[root],
99 )
100 finally:
101 await pool.shutdown()
102
103 result = asyncio.run(scenario())
104
105 assert not result.ok
106 assert result.error is not None
107 # PythonObject.execute wraps the underlying exception in
108 # ObjectMethodExecutionError -- same behavior as the in-process and
109 # spawn-per-request paths, so match that shape here too.
110 assert result.error.type == "ObjectMethodExecutionError"
111 assert "kaboom" in result.error.message
112 assert "ValueError" in result.error.message
113
114
115def test_pool_execute_missing_method_returns_method_not_supported(tmp_path):
116 root = tmp_path / "objects"
117 write_object(
118 root / "basics" / "readonly.py",
119 "def GET(request):\n return {'ok': True}\n",
120 )
121
122 async def scenario():
123 pool = object_worker_pool.WorkerPool(size=1)
124 try:
125 return await pool.execute(
126 ObjectExecutionRequest("basics_readonly", method="POST"),
127 timeout_seconds=5,
128 roots=[root],
129 )
130 finally:
131 await pool.shutdown()
132
133 result = asyncio.run(scenario())
134
135 assert not result.ok
136 assert result.error is not None
137 assert result.error.type == "MethodNotSupportedError"
138 assert "POST" in result.error.message
139
140
141# ---------------------------------------------------------------------------
142# Live edit: same warm worker must observe source rewrites between requests
143# ---------------------------------------------------------------------------
144
145
146def test_pool_live_edit_same_worker_sees_new_source(tmp_path):
147 root = tmp_path / "objects"
148 source = write_object(
149 root / "basics" / "live.py",
150 "def GET(request):\n return {'version': 1}\n",
151 )
152
153 async def scenario():
154 pool = object_worker_pool.WorkerPool(size=1) # single worker: guarantees same process
155 try:
156 first = await pool.execute(
157 ObjectExecutionRequest("basics_live"),
158 timeout_seconds=5,
159 roots=[root],
160 )
161 source.write_text("def GET(request):\n return {'version': 2}\n")
162 second = await pool.execute(
163 ObjectExecutionRequest("basics_live"),
164 timeout_seconds=5,
165 roots=[root],
166 )
167 return first, second
168 finally:
169 await pool.shutdown()
170
171 first, second = asyncio.run(scenario())
172
173 assert first.ok and first.result == {"version": 1}
174 assert second.ok and second.result == {"version": 2}
175
176
177# ---------------------------------------------------------------------------
178# Timeout
179# ---------------------------------------------------------------------------
180
181
182def test_pool_execute_timeout_matches_subprocess_error_shape(tmp_path):
183 root = tmp_path / "objects"
184 write_object(
185 root / "basics" / "slow.py",
186 "import time\n\ndef GET(request):\n time.sleep(5)\n return {'ok': True}\n",
187 )
188
189 async def scenario():
190 pool = object_worker_pool.WorkerPool(size=1)
191 try:
192 timed_out = await pool.execute(
193 ObjectExecutionRequest("basics_slow"),
194 timeout_seconds=0.5,
195 roots=[root],
196 )
197 # Pool should self-heal: a fresh request on the same pool works.
198 write_object(
199 root / "basics" / "fast.py",
200 "def GET(request):\n return {'ok': True}\n",
201 )
202 recovered = await pool.execute(
203 ObjectExecutionRequest("basics_fast"),
204 timeout_seconds=5,
205 roots=[root],
206 )
207 return timed_out, recovered
208 finally:
209 await pool.shutdown()
210
211 timed_out, recovered = asyncio.run(scenario())
212
213 assert not timed_out.ok
214 assert timed_out.error is not None
215 assert timed_out.error.type == object_execution.TIMEOUT_ERROR_TYPE
216 assert timed_out.error.message == "GET timed out for object basics_slow after 0.5 seconds"
217
218 assert recovered.ok
219 assert recovered.result == {"ok": True}
220
221
222# ---------------------------------------------------------------------------
223# Crash
224# ---------------------------------------------------------------------------
225
226
227def test_pool_execute_worker_crash_self_heals(tmp_path):
228 root = tmp_path / "objects"
229 write_object(
230 root / "basics" / "crash.py",
231 "import os\n\ndef GET(request):\n os._exit(1)\n",
232 )
233 write_object(
234 root / "basics" / "fine.py",
235 "def GET(request):\n return {'fine': True}\n",
236 )
237
238 async def scenario():
239 pool = object_worker_pool.WorkerPool(size=1)
240 try:
241 crashed = await pool.execute(
242 ObjectExecutionRequest("basics_crash"),
243 timeout_seconds=5,
244 roots=[root],
245 )
246 recovered = await pool.execute(
247 ObjectExecutionRequest("basics_fine"),
248 timeout_seconds=5,
249 roots=[root],
250 )
251 return crashed, recovered
252 finally:
253 await pool.shutdown()
254
255 crashed, recovered = asyncio.run(scenario())
256
257 assert not crashed.ok
258 assert crashed.error is not None
259 assert crashed.error.type == object_execution.WORKER_ERROR_TYPE
260 assert "exit code" in crashed.error.message
261
262 assert recovered.ok
263 assert recovered.result == {"fine": True}
264
265
266# ---------------------------------------------------------------------------
267# Concurrency: loop must not serialize concurrent executions
268#
269# These measure OVERLAP directly rather than inferring it from wall clock.
270# The earlier version asserted a total elapsed time 0.02s below the
271# fully-serialized floor, which is not a test of concurrency so much as a
272# test of how busy the machine is: one real run took 1.09s -- longer than
273# serialized execution would have needed -- purely from scheduling
274# pressure, and failed while the pool was working perfectly. Each
275# execution now records when it started and stopped, and the assertion is
276# the actual claim: two of them were running at the same moment.
277# ---------------------------------------------------------------------------
278
279_OVERLAP_OBJECT = """
280import os
281import time
282
283
284def GET(request):
285 started = time.time()
286 time.sleep(0.3)
287 with open(os.environ["DBBASIC_OVERLAP_LOG"], "a") as handle:
288 handle.write(f"{started},{time.time()}\\n")
289 return {"ok": True}
290"""
291
292
293def _max_concurrent(log_path):
294 """The most executions that were in flight at once, from their own
295 recorded intervals. Deterministic: no clock budget to blow."""
296 spans = []
297 for line in log_path.read_text().splitlines():
298 if not line.strip():
299 continue
300 start, end = (float(part) for part in line.split(","))
301 spans.append((start, 1))
302 spans.append((end, -1))
303 spans.sort()
304 running = peak = 0
305 for _stamp, delta in spans:
306 running += delta
307 peak = max(peak, running)
308 return peak
309
310
311def test_pool_size_two_runs_two_requests_concurrently(tmp_path, monkeypatch):
312 root = tmp_path / "objects"
313 log_path = tmp_path / "overlap.csv"
314 monkeypatch.setenv("DBBASIC_OVERLAP_LOG", str(log_path))
315 write_object(root / "basics" / "sleepy.py", _OVERLAP_OBJECT)
316
317 async def scenario():
318 pool = object_worker_pool.WorkerPool(size=2)
319 try:
320 started = time.perf_counter()
321 results = await asyncio.gather(
322 pool.execute(
323 ObjectExecutionRequest("basics_sleepy"),
324 timeout_seconds=5,
325 roots=[root],
326 ),
327 pool.execute(
328 ObjectExecutionRequest("basics_sleepy"),
329 timeout_seconds=5,
330 roots=[root],
331 ),
332 )
333 elapsed = time.perf_counter() - started
334 return results, elapsed
335 finally:
336 await pool.shutdown()
337
338 results, _elapsed = asyncio.run(scenario())
339
340 assert all(r.ok for r in results)
341 assert _max_concurrent(log_path) == 2, "the two requests never overlapped"
342
343
344def test_pool_three_requests_through_size_two_pool_all_complete(tmp_path, monkeypatch):
345 root = tmp_path / "objects"
346 log_path = tmp_path / "overlap.csv"
347 monkeypatch.setenv("DBBASIC_OVERLAP_LOG", str(log_path))
348 write_object(root / "basics" / "sleepy.py", _OVERLAP_OBJECT)
349
350 async def scenario():
351 pool = object_worker_pool.WorkerPool(size=2)
352 try:
353 started = time.perf_counter()
354 results = await asyncio.gather(
355 *[
356 pool.execute(
357 ObjectExecutionRequest("basics_sleepy"),
358 timeout_seconds=5,
359 roots=[root],
360 )
361 for _ in range(3)
362 ]
363 )
364 elapsed = time.perf_counter() - started
365 return results, elapsed
366 finally:
367 await pool.shutdown()
368
369 results, _elapsed = asyncio.run(scenario())
370
371 assert all(r.ok for r in results)
372 assert len(results) == 3
373 # Two at a time, never three: the pool is size two, so the third has to
374 # wait for a worker. Both halves of that are the claim, and both are
375 # now measured rather than timed.
376 assert _max_concurrent(log_path) == 2
377
378
379# ---------------------------------------------------------------------------
380# Object-not-found parity
381# ---------------------------------------------------------------------------
382
383
384def test_pool_execute_missing_object_returns_not_found(tmp_path):
385 root = tmp_path / "objects"
386 root.mkdir(parents=True, exist_ok=True)
387
388 async def scenario():
389 pool = object_worker_pool.WorkerPool(size=1)
390 try:
391 return await pool.execute(
392 ObjectExecutionRequest("basics_missing"),
393 timeout_seconds=5,
394 roots=[root],
395 )
396 finally:
397 await pool.shutdown()
398
399 result = asyncio.run(scenario())
400
401 assert not result.ok
402 assert result.error is not None
403 assert result.error.type == "ObjectNotFoundError"
404
405
406def test_worker_pool_rejects_nonpositive_size():
407 with pytest.raises(ValueError):
408 object_worker_pool.WorkerPool(size=0)
409 with pytest.raises(ValueError):
410 object_worker_pool.WorkerPool(size=-1)
411
412
413# ---------------------------------------------------------------------------
414# The pool never shrinks: 2026-10-24 two requests with bytes in the payload
415# raised out of execute() holding both workers, and every request after
416# that waited on the empty queue forever (dbbasic.com down for two days).
417# ---------------------------------------------------------------------------
418
419
420def test_pool_unencodable_payload_errors_and_keeps_the_worker(tmp_path):
421 root = tmp_path / "objects"
422 write_object(
423 root / "basics" / "fine.py",
424 "def GET(request):\n return {'fine': True}\n",
425 )
426
427 async def scenario():
428 pool = object_worker_pool.WorkerPool(size=1)
429 try:
430 results = []
431 for _ in range(3):
432 results.append(
433 await pool.execute(
434 ObjectExecutionRequest("basics_fine", payload={"blob": b"\x00\x01"}),
435 timeout_seconds=5,
436 roots=[root],
437 )
438 )
439 recovered = await asyncio.wait_for(
440 pool.execute(
441 ObjectExecutionRequest("basics_fine"),
442 timeout_seconds=5,
443 roots=[root],
444 ),
445 timeout=10,
446 )
447 return results, recovered
448 finally:
449 await pool.shutdown()
450
451 results, recovered = asyncio.run(scenario())
452
453 for result in results:
454 assert not result.ok
455 assert result.error.type == object_execution.WORKER_ERROR_TYPE
456 assert "TypeError" in result.error.message
457 assert recovered.ok
458 assert recovered.result == {"fine": True}
459
460
461def test_pool_waiting_for_a_busy_worker_times_out(tmp_path):
462 root = tmp_path / "objects"
463 write_object(
464 root / "basics" / "slow.py",
465 "import time\n\ndef GET(request):\n time.sleep(1.0)\n return {'slow': True}\n",
466 )
467
468 async def scenario():
469 pool = object_worker_pool.WorkerPool(size=1)
470 try:
471 slow = asyncio.ensure_future(
472 pool.execute(ObjectExecutionRequest("basics_slow"), timeout_seconds=5, roots=[root])
473 )
474 await asyncio.sleep(0.2)
475 started = time.perf_counter()
476 waited = await pool.execute(
477 ObjectExecutionRequest("basics_slow"), timeout_seconds=0.3, roots=[root]
478 )
479 elapsed = time.perf_counter() - started
480 return await slow, waited, elapsed
481 finally:
482 await pool.shutdown()
483
484 slow, waited, elapsed = asyncio.run(scenario())
485
486 assert slow.ok
487 assert not waited.ok
488 assert waited.error.type == object_execution.TIMEOUT_ERROR_TYPE
489 assert "No object worker came free" in waited.error.message
490 assert elapsed < 0.9
491
492
493def test_pool_cancelled_request_gives_the_slot_back(tmp_path):
494 root = tmp_path / "objects"
495 write_object(
496 root / "basics" / "slow.py",
497 "import time\n\ndef GET(request):\n time.sleep(2.0)\n return {'slow': True}\n",
498 )
499 write_object(
500 root / "basics" / "fine.py",
501 "def GET(request):\n return {'fine': True}\n",
502 )
503
504 async def scenario():
505 pool = object_worker_pool.WorkerPool(size=1)
506 try:
507 slow = asyncio.ensure_future(
508 pool.execute(ObjectExecutionRequest("basics_slow"), timeout_seconds=5, roots=[root])
509 )
510 await asyncio.sleep(0.3)
511 slow.cancel()
512 with pytest.raises(asyncio.CancelledError):
513 await slow
514 return await asyncio.wait_for(
515 pool.execute(ObjectExecutionRequest("basics_fine"), timeout_seconds=5, roots=[root]),
516 timeout=10,
517 )
518 finally:
519 await pool.shutdown()
520
521 recovered = asyncio.run(scenario())
522
523 assert recovered.ok
524 assert recovered.result == {"fine": True}