askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_object_worker_pool.py · 17.0 KB · 524 lines
| 1 | import asyncio |
| 2 | import time |
| 3 | from pathlib import Path |
| 4 | |
| 5 | import pytest |
| 6 | |
| 7 | import object_execution |
| 8 | import object_worker_pool |
| 9 | from object_execution import ObjectExecutionRequest |
| 10 | |
| 11 | |
| 12 | def write_object(path: Path, source: str) -> Path: |
| 13 | path.parent.mkdir(parents=True, exist_ok=True) |
| 14 | path.write_text(source) |
| 15 | return path |
| 16 | |
| 17 | |
| 18 | # --------------------------------------------------------------------------- |
| 19 | # Round-trip parity with execute_python_object_subprocess |
| 20 | # --------------------------------------------------------------------------- |
| 21 | |
| 22 | |
| 23 | def test_pool_execute_returns_success_result(tmp_path): |
| 24 | root = tmp_path / "objects" |
| 25 | write_object( |
| 26 | root / "basics" / "worker.py", |
| 27 | "def GET(request):\n return {'value': request['value']}\n", |
| 28 | ) |
| 29 | |
| 30 | async def scenario(): |
| 31 | pool = object_worker_pool.WorkerPool(size=1) |
| 32 | try: |
| 33 | return await pool.execute( |
| 34 | ObjectExecutionRequest("basics_worker", payload={"value": 7}), |
| 35 | timeout_seconds=5, |
| 36 | roots=[root], |
| 37 | ) |
| 38 | finally: |
| 39 | await pool.shutdown() |
| 40 | |
| 41 | result = asyncio.run(scenario()) |
| 42 | |
| 43 | assert result.ok |
| 44 | assert result.object_id == "basics_worker" |
| 45 | assert result.method == "GET" |
| 46 | assert result.path == root / "basics" / "worker.py" |
| 47 | assert result.result == {"value": 7} |
| 48 | assert result.error is None |
| 49 | assert result.started_at.endswith("Z") |
| 50 | assert result.finished_at.endswith("Z") |
| 51 | |
| 52 | |
| 53 | def test_pool_execute_matches_subprocess_success_shape(tmp_path): |
| 54 | root = tmp_path / "objects" |
| 55 | write_object( |
| 56 | root / "basics" / "worker.py", |
| 57 | "def GET(request):\n return {'value': request['value']}\n", |
| 58 | ) |
| 59 | |
| 60 | async def scenario(): |
| 61 | pool = object_worker_pool.WorkerPool(size=1) |
| 62 | try: |
| 63 | return await pool.execute( |
| 64 | ObjectExecutionRequest("basics_worker", payload={"value": 7}), |
| 65 | timeout_seconds=5, |
| 66 | roots=[root], |
| 67 | ) |
| 68 | finally: |
| 69 | await pool.shutdown() |
| 70 | |
| 71 | pool_result = asyncio.run(scenario()) |
| 72 | subprocess_result = object_execution.execute_python_object_subprocess( |
| 73 | ObjectExecutionRequest("basics_worker", payload={"value": 7}), |
| 74 | roots=[root], |
| 75 | timeout_seconds=5, |
| 76 | ) |
| 77 | |
| 78 | assert pool_result.ok == subprocess_result.ok |
| 79 | assert pool_result.result == subprocess_result.result |
| 80 | assert pool_result.object_id == subprocess_result.object_id |
| 81 | assert pool_result.method == subprocess_result.method |
| 82 | assert pool_result.path == subprocess_result.path |
| 83 | |
| 84 | |
| 85 | def test_pool_execute_object_raises_preserves_error_type_and_message(tmp_path): |
| 86 | root = tmp_path / "objects" |
| 87 | write_object( |
| 88 | root / "basics" / "boom.py", |
| 89 | "def GET(request):\n raise ValueError('kaboom')\n", |
| 90 | ) |
| 91 | |
| 92 | async def scenario(): |
| 93 | pool = object_worker_pool.WorkerPool(size=1) |
| 94 | try: |
| 95 | return await pool.execute( |
| 96 | ObjectExecutionRequest("basics_boom"), |
| 97 | timeout_seconds=5, |
| 98 | roots=[root], |
| 99 | ) |
| 100 | finally: |
| 101 | await pool.shutdown() |
| 102 | |
| 103 | result = asyncio.run(scenario()) |
| 104 | |
| 105 | assert not result.ok |
| 106 | assert result.error is not None |
| 107 | # PythonObject.execute wraps the underlying exception in |
| 108 | # ObjectMethodExecutionError -- same behavior as the in-process and |
| 109 | # spawn-per-request paths, so match that shape here too. |
| 110 | assert result.error.type == "ObjectMethodExecutionError" |
| 111 | assert "kaboom" in result.error.message |
| 112 | assert "ValueError" in result.error.message |
| 113 | |
| 114 | |
| 115 | def test_pool_execute_missing_method_returns_method_not_supported(tmp_path): |
| 116 | root = tmp_path / "objects" |
| 117 | write_object( |
| 118 | root / "basics" / "readonly.py", |
| 119 | "def GET(request):\n return {'ok': True}\n", |
| 120 | ) |
| 121 | |
| 122 | async def scenario(): |
| 123 | pool = object_worker_pool.WorkerPool(size=1) |
| 124 | try: |
| 125 | return await pool.execute( |
| 126 | ObjectExecutionRequest("basics_readonly", method="POST"), |
| 127 | timeout_seconds=5, |
| 128 | roots=[root], |
| 129 | ) |
| 130 | finally: |
| 131 | await pool.shutdown() |
| 132 | |
| 133 | result = asyncio.run(scenario()) |
| 134 | |
| 135 | assert not result.ok |
| 136 | assert result.error is not None |
| 137 | assert result.error.type == "MethodNotSupportedError" |
| 138 | assert "POST" in result.error.message |
| 139 | |
| 140 | |
| 141 | # --------------------------------------------------------------------------- |
| 142 | # Live edit: same warm worker must observe source rewrites between requests |
| 143 | # --------------------------------------------------------------------------- |
| 144 | |
| 145 | |
| 146 | def test_pool_live_edit_same_worker_sees_new_source(tmp_path): |
| 147 | root = tmp_path / "objects" |
| 148 | source = write_object( |
| 149 | root / "basics" / "live.py", |
| 150 | "def GET(request):\n return {'version': 1}\n", |
| 151 | ) |
| 152 | |
| 153 | async def scenario(): |
| 154 | pool = object_worker_pool.WorkerPool(size=1) # single worker: guarantees same process |
| 155 | try: |
| 156 | first = await pool.execute( |
| 157 | ObjectExecutionRequest("basics_live"), |
| 158 | timeout_seconds=5, |
| 159 | roots=[root], |
| 160 | ) |
| 161 | source.write_text("def GET(request):\n return {'version': 2}\n") |
| 162 | second = await pool.execute( |
| 163 | ObjectExecutionRequest("basics_live"), |
| 164 | timeout_seconds=5, |
| 165 | roots=[root], |
| 166 | ) |
| 167 | return first, second |
| 168 | finally: |
| 169 | await pool.shutdown() |
| 170 | |
| 171 | first, second = asyncio.run(scenario()) |
| 172 | |
| 173 | assert first.ok and first.result == {"version": 1} |
| 174 | assert second.ok and second.result == {"version": 2} |
| 175 | |
| 176 | |
| 177 | # --------------------------------------------------------------------------- |
| 178 | # Timeout |
| 179 | # --------------------------------------------------------------------------- |
| 180 | |
| 181 | |
| 182 | def test_pool_execute_timeout_matches_subprocess_error_shape(tmp_path): |
| 183 | root = tmp_path / "objects" |
| 184 | write_object( |
| 185 | root / "basics" / "slow.py", |
| 186 | "import time\n\ndef GET(request):\n time.sleep(5)\n return {'ok': True}\n", |
| 187 | ) |
| 188 | |
| 189 | async def scenario(): |
| 190 | pool = object_worker_pool.WorkerPool(size=1) |
| 191 | try: |
| 192 | timed_out = await pool.execute( |
| 193 | ObjectExecutionRequest("basics_slow"), |
| 194 | timeout_seconds=0.5, |
| 195 | roots=[root], |
| 196 | ) |
| 197 | # Pool should self-heal: a fresh request on the same pool works. |
| 198 | write_object( |
| 199 | root / "basics" / "fast.py", |
| 200 | "def GET(request):\n return {'ok': True}\n", |
| 201 | ) |
| 202 | recovered = await pool.execute( |
| 203 | ObjectExecutionRequest("basics_fast"), |
| 204 | timeout_seconds=5, |
| 205 | roots=[root], |
| 206 | ) |
| 207 | return timed_out, recovered |
| 208 | finally: |
| 209 | await pool.shutdown() |
| 210 | |
| 211 | timed_out, recovered = asyncio.run(scenario()) |
| 212 | |
| 213 | assert not timed_out.ok |
| 214 | assert timed_out.error is not None |
| 215 | assert timed_out.error.type == object_execution.TIMEOUT_ERROR_TYPE |
| 216 | assert timed_out.error.message == "GET timed out for object basics_slow after 0.5 seconds" |
| 217 | |
| 218 | assert recovered.ok |
| 219 | assert recovered.result == {"ok": True} |
| 220 | |
| 221 | |
| 222 | # --------------------------------------------------------------------------- |
| 223 | # Crash |
| 224 | # --------------------------------------------------------------------------- |
| 225 | |
| 226 | |
| 227 | def test_pool_execute_worker_crash_self_heals(tmp_path): |
| 228 | root = tmp_path / "objects" |
| 229 | write_object( |
| 230 | root / "basics" / "crash.py", |
| 231 | "import os\n\ndef GET(request):\n os._exit(1)\n", |
| 232 | ) |
| 233 | write_object( |
| 234 | root / "basics" / "fine.py", |
| 235 | "def GET(request):\n return {'fine': True}\n", |
| 236 | ) |
| 237 | |
| 238 | async def scenario(): |
| 239 | pool = object_worker_pool.WorkerPool(size=1) |
| 240 | try: |
| 241 | crashed = await pool.execute( |
| 242 | ObjectExecutionRequest("basics_crash"), |
| 243 | timeout_seconds=5, |
| 244 | roots=[root], |
| 245 | ) |
| 246 | recovered = await pool.execute( |
| 247 | ObjectExecutionRequest("basics_fine"), |
| 248 | timeout_seconds=5, |
| 249 | roots=[root], |
| 250 | ) |
| 251 | return crashed, recovered |
| 252 | finally: |
| 253 | await pool.shutdown() |
| 254 | |
| 255 | crashed, recovered = asyncio.run(scenario()) |
| 256 | |
| 257 | assert not crashed.ok |
| 258 | assert crashed.error is not None |
| 259 | assert crashed.error.type == object_execution.WORKER_ERROR_TYPE |
| 260 | assert "exit code" in crashed.error.message |
| 261 | |
| 262 | assert recovered.ok |
| 263 | assert recovered.result == {"fine": True} |
| 264 | |
| 265 | |
| 266 | # --------------------------------------------------------------------------- |
| 267 | # Concurrency: loop must not serialize concurrent executions |
| 268 | # |
| 269 | # These measure OVERLAP directly rather than inferring it from wall clock. |
| 270 | # The earlier version asserted a total elapsed time 0.02s below the |
| 271 | # fully-serialized floor, which is not a test of concurrency so much as a |
| 272 | # test of how busy the machine is: one real run took 1.09s -- longer than |
| 273 | # serialized execution would have needed -- purely from scheduling |
| 274 | # pressure, and failed while the pool was working perfectly. Each |
| 275 | # execution now records when it started and stopped, and the assertion is |
| 276 | # the actual claim: two of them were running at the same moment. |
| 277 | # --------------------------------------------------------------------------- |
| 278 | |
| 279 | _OVERLAP_OBJECT = """ |
| 280 | import os |
| 281 | import time |
| 282 | |
| 283 | |
| 284 | def GET(request): |
| 285 | started = time.time() |
| 286 | time.sleep(0.3) |
| 287 | with open(os.environ["DBBASIC_OVERLAP_LOG"], "a") as handle: |
| 288 | handle.write(f"{started},{time.time()}\\n") |
| 289 | return {"ok": True} |
| 290 | """ |
| 291 | |
| 292 | |
| 293 | def _max_concurrent(log_path): |
| 294 | """The most executions that were in flight at once, from their own |
| 295 | recorded intervals. Deterministic: no clock budget to blow.""" |
| 296 | spans = [] |
| 297 | for line in log_path.read_text().splitlines(): |
| 298 | if not line.strip(): |
| 299 | continue |
| 300 | start, end = (float(part) for part in line.split(",")) |
| 301 | spans.append((start, 1)) |
| 302 | spans.append((end, -1)) |
| 303 | spans.sort() |
| 304 | running = peak = 0 |
| 305 | for _stamp, delta in spans: |
| 306 | running += delta |
| 307 | peak = max(peak, running) |
| 308 | return peak |
| 309 | |
| 310 | |
| 311 | def test_pool_size_two_runs_two_requests_concurrently(tmp_path, monkeypatch): |
| 312 | root = tmp_path / "objects" |
| 313 | log_path = tmp_path / "overlap.csv" |
| 314 | monkeypatch.setenv("DBBASIC_OVERLAP_LOG", str(log_path)) |
| 315 | write_object(root / "basics" / "sleepy.py", _OVERLAP_OBJECT) |
| 316 | |
| 317 | async def scenario(): |
| 318 | pool = object_worker_pool.WorkerPool(size=2) |
| 319 | try: |
| 320 | started = time.perf_counter() |
| 321 | results = await asyncio.gather( |
| 322 | pool.execute( |
| 323 | ObjectExecutionRequest("basics_sleepy"), |
| 324 | timeout_seconds=5, |
| 325 | roots=[root], |
| 326 | ), |
| 327 | pool.execute( |
| 328 | ObjectExecutionRequest("basics_sleepy"), |
| 329 | timeout_seconds=5, |
| 330 | roots=[root], |
| 331 | ), |
| 332 | ) |
| 333 | elapsed = time.perf_counter() - started |
| 334 | return results, elapsed |
| 335 | finally: |
| 336 | await pool.shutdown() |
| 337 | |
| 338 | results, _elapsed = asyncio.run(scenario()) |
| 339 | |
| 340 | assert all(r.ok for r in results) |
| 341 | assert _max_concurrent(log_path) == 2, "the two requests never overlapped" |
| 342 | |
| 343 | |
| 344 | def test_pool_three_requests_through_size_two_pool_all_complete(tmp_path, monkeypatch): |
| 345 | root = tmp_path / "objects" |
| 346 | log_path = tmp_path / "overlap.csv" |
| 347 | monkeypatch.setenv("DBBASIC_OVERLAP_LOG", str(log_path)) |
| 348 | write_object(root / "basics" / "sleepy.py", _OVERLAP_OBJECT) |
| 349 | |
| 350 | async def scenario(): |
| 351 | pool = object_worker_pool.WorkerPool(size=2) |
| 352 | try: |
| 353 | started = time.perf_counter() |
| 354 | results = await asyncio.gather( |
| 355 | *[ |
| 356 | pool.execute( |
| 357 | ObjectExecutionRequest("basics_sleepy"), |
| 358 | timeout_seconds=5, |
| 359 | roots=[root], |
| 360 | ) |
| 361 | for _ in range(3) |
| 362 | ] |
| 363 | ) |
| 364 | elapsed = time.perf_counter() - started |
| 365 | return results, elapsed |
| 366 | finally: |
| 367 | await pool.shutdown() |
| 368 | |
| 369 | results, _elapsed = asyncio.run(scenario()) |
| 370 | |
| 371 | assert all(r.ok for r in results) |
| 372 | assert len(results) == 3 |
| 373 | # Two at a time, never three: the pool is size two, so the third has to |
| 374 | # wait for a worker. Both halves of that are the claim, and both are |
| 375 | # now measured rather than timed. |
| 376 | assert _max_concurrent(log_path) == 2 |
| 377 | |
| 378 | |
| 379 | # --------------------------------------------------------------------------- |
| 380 | # Object-not-found parity |
| 381 | # --------------------------------------------------------------------------- |
| 382 | |
| 383 | |
| 384 | def test_pool_execute_missing_object_returns_not_found(tmp_path): |
| 385 | root = tmp_path / "objects" |
| 386 | root.mkdir(parents=True, exist_ok=True) |
| 387 | |
| 388 | async def scenario(): |
| 389 | pool = object_worker_pool.WorkerPool(size=1) |
| 390 | try: |
| 391 | return await pool.execute( |
| 392 | ObjectExecutionRequest("basics_missing"), |
| 393 | timeout_seconds=5, |
| 394 | roots=[root], |
| 395 | ) |
| 396 | finally: |
| 397 | await pool.shutdown() |
| 398 | |
| 399 | result = asyncio.run(scenario()) |
| 400 | |
| 401 | assert not result.ok |
| 402 | assert result.error is not None |
| 403 | assert result.error.type == "ObjectNotFoundError" |
| 404 | |
| 405 | |
| 406 | def test_worker_pool_rejects_nonpositive_size(): |
| 407 | with pytest.raises(ValueError): |
| 408 | object_worker_pool.WorkerPool(size=0) |
| 409 | with pytest.raises(ValueError): |
| 410 | object_worker_pool.WorkerPool(size=-1) |
| 411 | |
| 412 | |
| 413 | # --------------------------------------------------------------------------- |
| 414 | # The pool never shrinks: 2026-10-24 two requests with bytes in the payload |
| 415 | # raised out of execute() holding both workers, and every request after |
| 416 | # that waited on the empty queue forever (dbbasic.com down for two days). |
| 417 | # --------------------------------------------------------------------------- |
| 418 | |
| 419 | |
| 420 | def test_pool_unencodable_payload_errors_and_keeps_the_worker(tmp_path): |
| 421 | root = tmp_path / "objects" |
| 422 | write_object( |
| 423 | root / "basics" / "fine.py", |
| 424 | "def GET(request):\n return {'fine': True}\n", |
| 425 | ) |
| 426 | |
| 427 | async def scenario(): |
| 428 | pool = object_worker_pool.WorkerPool(size=1) |
| 429 | try: |
| 430 | results = [] |
| 431 | for _ in range(3): |
| 432 | results.append( |
| 433 | await pool.execute( |
| 434 | ObjectExecutionRequest("basics_fine", payload={"blob": b"\x00\x01"}), |
| 435 | timeout_seconds=5, |
| 436 | roots=[root], |
| 437 | ) |
| 438 | ) |
| 439 | recovered = await asyncio.wait_for( |
| 440 | pool.execute( |
| 441 | ObjectExecutionRequest("basics_fine"), |
| 442 | timeout_seconds=5, |
| 443 | roots=[root], |
| 444 | ), |
| 445 | timeout=10, |
| 446 | ) |
| 447 | return results, recovered |
| 448 | finally: |
| 449 | await pool.shutdown() |
| 450 | |
| 451 | results, recovered = asyncio.run(scenario()) |
| 452 | |
| 453 | for result in results: |
| 454 | assert not result.ok |
| 455 | assert result.error.type == object_execution.WORKER_ERROR_TYPE |
| 456 | assert "TypeError" in result.error.message |
| 457 | assert recovered.ok |
| 458 | assert recovered.result == {"fine": True} |
| 459 | |
| 460 | |
| 461 | def test_pool_waiting_for_a_busy_worker_times_out(tmp_path): |
| 462 | root = tmp_path / "objects" |
| 463 | write_object( |
| 464 | root / "basics" / "slow.py", |
| 465 | "import time\n\ndef GET(request):\n time.sleep(1.0)\n return {'slow': True}\n", |
| 466 | ) |
| 467 | |
| 468 | async def scenario(): |
| 469 | pool = object_worker_pool.WorkerPool(size=1) |
| 470 | try: |
| 471 | slow = asyncio.ensure_future( |
| 472 | pool.execute(ObjectExecutionRequest("basics_slow"), timeout_seconds=5, roots=[root]) |
| 473 | ) |
| 474 | await asyncio.sleep(0.2) |
| 475 | started = time.perf_counter() |
| 476 | waited = await pool.execute( |
| 477 | ObjectExecutionRequest("basics_slow"), timeout_seconds=0.3, roots=[root] |
| 478 | ) |
| 479 | elapsed = time.perf_counter() - started |
| 480 | return await slow, waited, elapsed |
| 481 | finally: |
| 482 | await pool.shutdown() |
| 483 | |
| 484 | slow, waited, elapsed = asyncio.run(scenario()) |
| 485 | |
| 486 | assert slow.ok |
| 487 | assert not waited.ok |
| 488 | assert waited.error.type == object_execution.TIMEOUT_ERROR_TYPE |
| 489 | assert "No object worker came free" in waited.error.message |
| 490 | assert elapsed < 0.9 |
| 491 | |
| 492 | |
| 493 | def test_pool_cancelled_request_gives_the_slot_back(tmp_path): |
| 494 | root = tmp_path / "objects" |
| 495 | write_object( |
| 496 | root / "basics" / "slow.py", |
| 497 | "import time\n\ndef GET(request):\n time.sleep(2.0)\n return {'slow': True}\n", |
| 498 | ) |
| 499 | write_object( |
| 500 | root / "basics" / "fine.py", |
| 501 | "def GET(request):\n return {'fine': True}\n", |
| 502 | ) |
| 503 | |
| 504 | async def scenario(): |
| 505 | pool = object_worker_pool.WorkerPool(size=1) |
| 506 | try: |
| 507 | slow = asyncio.ensure_future( |
| 508 | pool.execute(ObjectExecutionRequest("basics_slow"), timeout_seconds=5, roots=[root]) |
| 509 | ) |
| 510 | await asyncio.sleep(0.3) |
| 511 | slow.cancel() |
| 512 | with pytest.raises(asyncio.CancelledError): |
| 513 | await slow |
| 514 | return await asyncio.wait_for( |
| 515 | pool.execute(ObjectExecutionRequest("basics_fine"), timeout_seconds=5, roots=[root]), |
| 516 | timeout=10, |
| 517 | ) |
| 518 | finally: |
| 519 | await pool.shutdown() |
| 520 | |
| 521 | recovered = asyncio.run(scenario()) |
| 522 | |
| 523 | assert recovered.ok |
| 524 | assert recovered.result == {"fine": True} |