askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_object_server_worker_pool.py · 11.6 KB · 314 lines
| 1 | """Server-level integration tests for the DBBASIC_WORKER_POOL_SIZE path. |
| 2 | |
| 3 | object_worker_pool.WorkerPool itself is covered end-to-end in |
| 4 | tests/test_object_worker_pool.py. These tests exercise the object_server.py |
| 5 | wiring: the _execute_object_method call site, the _get_worker_pool / |
| 6 | _shutdown_worker_pool lifecycle helpers, and the admin capabilities payload. |
| 7 | |
| 8 | Note on event loops: asyncio subprocess transports are bound to the loop |
| 9 | that created them, and every test here that actually spawns pool workers |
| 10 | runs its HTTP calls *and* the final object_server._shutdown_worker_pool() |
| 11 | inside one asyncio.run() (see run_scenario below), rather than opening a |
| 12 | fresh loop per call the way tests/test_object_server.py's request() helper |
| 13 | does. That mirrors how a real server actually uses the pool -- uvicorn runs |
| 14 | one event loop for the process's whole lifetime -- and it lets shutdown() |
| 15 | gracefully await each worker's exit instead of falling back to its |
| 16 | best-effort cross-loop pid kill. |
| 17 | """ |
| 18 | import asyncio |
| 19 | import json |
| 20 | import time as time_module |
| 21 | |
| 22 | import object_correlation |
| 23 | import object_execution |
| 24 | import object_server |
| 25 | import object_worker_pool |
| 26 | |
| 27 | TEST_ADMIN_TOKEN = "unit-test-only-admin-token" |
| 28 | |
| 29 | |
| 30 | def write_source(path, content): |
| 31 | path.parent.mkdir(parents=True, exist_ok=True) |
| 32 | path.write_text(content) |
| 33 | return path |
| 34 | |
| 35 | |
| 36 | def auth_headers(): |
| 37 | return [("authorization", f"Token {TEST_ADMIN_TOKEN}")] |
| 38 | |
| 39 | |
| 40 | def enable_admin_token(monkeypatch): |
| 41 | monkeypatch.setenv("DBBASIC_ADMIN_TOKEN", TEST_ADMIN_TOKEN) |
| 42 | |
| 43 | |
| 44 | async def asgi_request(path, method="GET", query_string="", body=b"", headers=None): |
| 45 | messages = [] |
| 46 | |
| 47 | async def receive(): |
| 48 | return {"type": "http.request", "body": body, "more_body": False} |
| 49 | |
| 50 | async def send(message): |
| 51 | messages.append(message) |
| 52 | |
| 53 | scope_headers = [(b"accept", b"application/json")] |
| 54 | for name, value in headers or []: |
| 55 | if isinstance(name, str): |
| 56 | name = name.encode("latin-1") |
| 57 | if isinstance(value, str): |
| 58 | value = value.encode("latin-1") |
| 59 | scope_headers.append((name, value)) |
| 60 | |
| 61 | scope = { |
| 62 | "type": "http", |
| 63 | "method": method, |
| 64 | "path": path, |
| 65 | "query_string": query_string.encode("utf-8"), |
| 66 | "headers": scope_headers, |
| 67 | "client": ("127.0.0.1", 12345), |
| 68 | } |
| 69 | await object_server.app(scope, receive, send) |
| 70 | |
| 71 | start = next(m for m in messages if m["type"] == "http.response.start") |
| 72 | body_parts = [m.get("body", b"") for m in messages if m["type"] == "http.response.body"] |
| 73 | payload = b"".join(body_parts) |
| 74 | return start["status"], dict(start["headers"]), json.loads(payload.decode("utf-8")) |
| 75 | |
| 76 | |
| 77 | def request(path, method="GET", query_string="", body=b"", headers=None): |
| 78 | """Plain single-call helper for tests that never touch the pool.""" |
| 79 | return asyncio.run( |
| 80 | asgi_request(path, method=method, query_string=query_string, body=body, headers=headers) |
| 81 | ) |
| 82 | |
| 83 | |
| 84 | def run_scenario(coro_factory): |
| 85 | """Run an async test body and shut the pool down on the SAME loop. |
| 86 | |
| 87 | coro_factory is a zero-arg callable returning the coroutine to run (not |
| 88 | the coroutine itself, so it's only created inside the new event loop). |
| 89 | """ |
| 90 | |
| 91 | async def wrapper(): |
| 92 | try: |
| 93 | return await coro_factory() |
| 94 | finally: |
| 95 | await object_server._shutdown_worker_pool() |
| 96 | |
| 97 | return asyncio.run(wrapper()) |
| 98 | |
| 99 | |
| 100 | # --------------------------------------------------------------------------- |
| 101 | # Disabled by default |
| 102 | # --------------------------------------------------------------------------- |
| 103 | |
| 104 | |
| 105 | def test_worker_pool_disabled_by_default_uses_subprocess_path(tmp_path, monkeypatch): |
| 106 | root = tmp_path / "objects" |
| 107 | write_source( |
| 108 | root / "basics" / "fast.py", |
| 109 | "def GET(request):\n return {'mode': 'object_source'}\n", |
| 110 | ) |
| 111 | monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(root)) |
| 112 | monkeypatch.delenv(object_worker_pool.WORKER_POOL_SIZE_ENV, raising=False) |
| 113 | monkeypatch.setenv(object_server.OBJECT_TIMEOUT_SECONDS_ENV, "5") |
| 114 | |
| 115 | calls = [] |
| 116 | |
| 117 | def fake_subprocess(exec_request, roots=None, *, timeout_seconds, raise_on_error=False): |
| 118 | calls.append((exec_request.object_id, timeout_seconds)) |
| 119 | return object_execution.ObjectExecutionResult( |
| 120 | object_id=exec_request.object_id, |
| 121 | method=exec_request.normalized_method(), |
| 122 | path=exec_request.path, |
| 123 | ok=True, |
| 124 | result={"mode": "subprocess"}, |
| 125 | error=None, |
| 126 | started_at="2026-01-01T00:00:00Z", |
| 127 | finished_at="2026-01-01T00:00:00Z", |
| 128 | duration_ms=1.0, |
| 129 | ) |
| 130 | |
| 131 | monkeypatch.setattr(object_execution, "execute_python_object_subprocess", fake_subprocess) |
| 132 | |
| 133 | status, _, payload = request("/objects/basics_fast") |
| 134 | |
| 135 | assert status == 200 |
| 136 | assert payload == {"mode": "subprocess"} |
| 137 | assert calls == [("basics_fast", 5.0)] |
| 138 | # The pool must never have been constructed at all. |
| 139 | assert object_server._worker_pool is None |
| 140 | |
| 141 | |
| 142 | def test_trusted_in_process_object_still_bypasses_pool(tmp_path, monkeypatch): |
| 143 | root = tmp_path / "objects" |
| 144 | write_source( |
| 145 | root / "basics" / "fast.py", |
| 146 | "def GET(request):\n return {'mode': 'in_process'}\n", |
| 147 | ) |
| 148 | monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(root)) |
| 149 | monkeypatch.setenv(object_server.OBJECT_TIMEOUT_SECONDS_ENV, "5") |
| 150 | monkeypatch.setenv(object_server.TRUSTED_IN_PROCESS_OBJECTS_ENV, "basics_fast") |
| 151 | monkeypatch.setenv(object_worker_pool.WORKER_POOL_SIZE_ENV, "1") |
| 152 | |
| 153 | status, _, payload = request("/objects/basics_fast") |
| 154 | |
| 155 | assert status == 200 |
| 156 | assert payload == {"mode": "in_process"} |
| 157 | # Trusted objects never touch the pool, so it should never spin up. |
| 158 | assert object_server._worker_pool is None |
| 159 | |
| 160 | |
| 161 | # --------------------------------------------------------------------------- |
| 162 | # Capabilities payload (no pool workers spawned either way) |
| 163 | # --------------------------------------------------------------------------- |
| 164 | |
| 165 | |
| 166 | def test_admin_status_reports_worker_pool_disabled_by_default(monkeypatch): |
| 167 | monkeypatch.delenv(object_worker_pool.WORKER_POOL_SIZE_ENV, raising=False) |
| 168 | enable_admin_token(monkeypatch) |
| 169 | |
| 170 | status, _, payload = request("/admin/status", headers=auth_headers()) |
| 171 | |
| 172 | assert status == 200 |
| 173 | assert payload["capabilities"]["worker_pool"] == { |
| 174 | "enabled": False, |
| 175 | "size": 0, |
| 176 | "env": "DBBASIC_WORKER_POOL_SIZE", |
| 177 | } |
| 178 | |
| 179 | |
| 180 | def test_admin_status_reports_worker_pool_enabled_with_size(monkeypatch): |
| 181 | monkeypatch.setenv(object_worker_pool.WORKER_POOL_SIZE_ENV, "4") |
| 182 | enable_admin_token(monkeypatch) |
| 183 | |
| 184 | status, _, payload = request("/admin/status", headers=auth_headers()) |
| 185 | |
| 186 | assert status == 200 |
| 187 | assert payload["capabilities"]["worker_pool"] == { |
| 188 | "enabled": True, |
| 189 | "size": 4, |
| 190 | "env": "DBBASIC_WORKER_POOL_SIZE", |
| 191 | } |
| 192 | |
| 193 | |
| 194 | # --------------------------------------------------------------------------- |
| 195 | # Enabled: pool path taken instead of subprocess (spawns real workers) |
| 196 | # --------------------------------------------------------------------------- |
| 197 | |
| 198 | |
| 199 | def test_worker_pool_enabled_executes_via_pool_not_subprocess(tmp_path, monkeypatch): |
| 200 | root = tmp_path / "objects" |
| 201 | write_source( |
| 202 | root / "basics" / "fast.py", |
| 203 | "def GET(request):\n return {'mode': 'pool'}\n", |
| 204 | ) |
| 205 | monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(root)) |
| 206 | monkeypatch.setenv(object_server.OBJECT_TIMEOUT_SECONDS_ENV, "5") |
| 207 | monkeypatch.setenv(object_worker_pool.WORKER_POOL_SIZE_ENV, "1") |
| 208 | |
| 209 | def fail_subprocess(*args, **kwargs): |
| 210 | raise AssertionError("pool is enabled -- subprocess-per-request must not run") |
| 211 | |
| 212 | monkeypatch.setattr(object_execution, "execute_python_object_subprocess", fail_subprocess) |
| 213 | |
| 214 | status, _, payload = run_scenario(lambda: asgi_request("/objects/basics_fast")) |
| 215 | |
| 216 | assert status == 200 |
| 217 | assert payload == {"mode": "pool"} |
| 218 | |
| 219 | |
| 220 | def test_worker_pool_execution_error_reports_through_normal_error_path(tmp_path, monkeypatch): |
| 221 | root = tmp_path / "objects" |
| 222 | write_source( |
| 223 | root / "basics" / "broken.py", |
| 224 | "def GET(request):\n raise RuntimeError('boom')\n", |
| 225 | ) |
| 226 | monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(root)) |
| 227 | monkeypatch.setenv(object_server.OBJECT_TIMEOUT_SECONDS_ENV, "5") |
| 228 | monkeypatch.setenv(object_worker_pool.WORKER_POOL_SIZE_ENV, "1") |
| 229 | |
| 230 | status, _, payload = run_scenario(lambda: asgi_request("/objects/basics_broken")) |
| 231 | |
| 232 | assert status == 500 |
| 233 | assert payload["status"] == "error" |
| 234 | assert "RuntimeError: boom" in payload["error"] |
| 235 | |
| 236 | |
| 237 | def test_worker_pool_timeout_returns_504_matching_subprocess_message(tmp_path, monkeypatch): |
| 238 | root = tmp_path / "objects" |
| 239 | write_source( |
| 240 | root / "basics" / "slow.py", |
| 241 | "import time\n\ndef GET(request):\n time.sleep(5)\n return {'ok': True}\n", |
| 242 | ) |
| 243 | monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(root)) |
| 244 | monkeypatch.setenv(object_server.OBJECT_TIMEOUT_SECONDS_ENV, "0.5") |
| 245 | monkeypatch.setenv(object_worker_pool.WORKER_POOL_SIZE_ENV, "1") |
| 246 | |
| 247 | status, _, payload = run_scenario(lambda: asgi_request("/objects/basics_slow")) |
| 248 | |
| 249 | assert status == 504 |
| 250 | assert payload["status"] == "error" |
| 251 | assert payload["error"] == "Execution failed: GET timed out for object basics_slow after 0.5 seconds" |
| 252 | assert object_correlation.normalize_correlation_id(payload["correlation_id"]) == payload[ |
| 253 | "correlation_id" |
| 254 | ] |
| 255 | |
| 256 | |
| 257 | # --------------------------------------------------------------------------- |
| 258 | # Live edit through the real server call site, same warm worker |
| 259 | # --------------------------------------------------------------------------- |
| 260 | |
| 261 | |
| 262 | def test_worker_pool_live_edit_survives_across_requests_on_same_worker(tmp_path, monkeypatch): |
| 263 | root = tmp_path / "objects" |
| 264 | source = write_source( |
| 265 | root / "basics" / "live.py", |
| 266 | "def GET(request):\n return {'version': 1}\n", |
| 267 | ) |
| 268 | monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(root)) |
| 269 | monkeypatch.setenv(object_server.OBJECT_TIMEOUT_SECONDS_ENV, "5") |
| 270 | monkeypatch.setenv(object_worker_pool.WORKER_POOL_SIZE_ENV, "1") # single worker: same process both times |
| 271 | |
| 272 | async def scenario(): |
| 273 | first = await asgi_request("/objects/basics_live") |
| 274 | source.write_text("def GET(request):\n return {'version': 2}\n") |
| 275 | second = await asgi_request("/objects/basics_live") |
| 276 | return first, second |
| 277 | |
| 278 | first, second = run_scenario(scenario) |
| 279 | status1, _, payload1 = first |
| 280 | status2, _, payload2 = second |
| 281 | |
| 282 | assert status1 == 200 and payload1 == {"version": 1} |
| 283 | assert status2 == 200 and payload2 == {"version": 2} |
| 284 | |
| 285 | |
| 286 | # --------------------------------------------------------------------------- |
| 287 | # Concurrency: pool path must not serialize the event loop |
| 288 | # --------------------------------------------------------------------------- |
| 289 | |
| 290 | |
| 291 | def test_worker_pool_concurrent_requests_do_not_block_event_loop(tmp_path, monkeypatch): |
| 292 | root = tmp_path / "objects" |
| 293 | write_source( |
| 294 | root / "basics" / "sleepy.py", |
| 295 | "import time\n\ndef GET(request):\n time.sleep(0.3)\n return {'ok': True}\n", |
| 296 | ) |
| 297 | monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(root)) |
| 298 | monkeypatch.setenv(object_server.OBJECT_TIMEOUT_SECONDS_ENV, "5") |
| 299 | monkeypatch.setenv(object_worker_pool.WORKER_POOL_SIZE_ENV, "2") |
| 300 | monkeypatch.setenv(object_server.MAX_CONCURRENT_EXECUTIONS_ENV, "2") |
| 301 | |
| 302 | async def scenario(): |
| 303 | started = time_module.perf_counter() |
| 304 | results = await asyncio.gather( |
| 305 | asgi_request("/objects/basics_sleepy"), |
| 306 | asgi_request("/objects/basics_sleepy"), |
| 307 | ) |
| 308 | elapsed = time_module.perf_counter() - started |
| 309 | return results, elapsed |
| 310 | |
| 311 | results, elapsed = run_scenario(scenario) |
| 312 | |
| 313 | assert all(status == 200 and payload == {"ok": True} for status, _, payload in results) |
| 314 | assert elapsed < 0.55, f"expected overlap, took {elapsed:.3f}s" |