Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_object_server_worker_pool.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_object_server_worker_pool.py · 11.6 KB · 314 lines
1"""Server-level integration tests for the DBBASIC_WORKER_POOL_SIZE path.
2
3object_worker_pool.WorkerPool itself is covered end-to-end in
4tests/test_object_worker_pool.py. These tests exercise the object_server.py
5wiring: the _execute_object_method call site, the _get_worker_pool /
6_shutdown_worker_pool lifecycle helpers, and the admin capabilities payload.
7
8Note on event loops: asyncio subprocess transports are bound to the loop
9that created them, and every test here that actually spawns pool workers
10runs its HTTP calls *and* the final object_server._shutdown_worker_pool()
11inside one asyncio.run() (see run_scenario below), rather than opening a
12fresh loop per call the way tests/test_object_server.py's request() helper
13does. That mirrors how a real server actually uses the pool -- uvicorn runs
14one event loop for the process's whole lifetime -- and it lets shutdown()
15gracefully await each worker's exit instead of falling back to its
16best-effort cross-loop pid kill.
17"""
18import asyncio
19import json
20import time as time_module
21
22import object_correlation
23import object_execution
24import object_server
25import object_worker_pool
26
27TEST_ADMIN_TOKEN = "unit-test-only-admin-token"
28
29
30def write_source(path, content):
31 path.parent.mkdir(parents=True, exist_ok=True)
32 path.write_text(content)
33 return path
34
35
36def auth_headers():
37 return [("authorization", f"Token {TEST_ADMIN_TOKEN}")]
38
39
40def enable_admin_token(monkeypatch):
41 monkeypatch.setenv("DBBASIC_ADMIN_TOKEN", TEST_ADMIN_TOKEN)
42
43
44async def asgi_request(path, method="GET", query_string="", body=b"", headers=None):
45 messages = []
46
47 async def receive():
48 return {"type": "http.request", "body": body, "more_body": False}
49
50 async def send(message):
51 messages.append(message)
52
53 scope_headers = [(b"accept", b"application/json")]
54 for name, value in headers or []:
55 if isinstance(name, str):
56 name = name.encode("latin-1")
57 if isinstance(value, str):
58 value = value.encode("latin-1")
59 scope_headers.append((name, value))
60
61 scope = {
62 "type": "http",
63 "method": method,
64 "path": path,
65 "query_string": query_string.encode("utf-8"),
66 "headers": scope_headers,
67 "client": ("127.0.0.1", 12345),
68 }
69 await object_server.app(scope, receive, send)
70
71 start = next(m for m in messages if m["type"] == "http.response.start")
72 body_parts = [m.get("body", b"") for m in messages if m["type"] == "http.response.body"]
73 payload = b"".join(body_parts)
74 return start["status"], dict(start["headers"]), json.loads(payload.decode("utf-8"))
75
76
77def request(path, method="GET", query_string="", body=b"", headers=None):
78 """Plain single-call helper for tests that never touch the pool."""
79 return asyncio.run(
80 asgi_request(path, method=method, query_string=query_string, body=body, headers=headers)
81 )
82
83
84def run_scenario(coro_factory):
85 """Run an async test body and shut the pool down on the SAME loop.
86
87 coro_factory is a zero-arg callable returning the coroutine to run (not
88 the coroutine itself, so it's only created inside the new event loop).
89 """
90
91 async def wrapper():
92 try:
93 return await coro_factory()
94 finally:
95 await object_server._shutdown_worker_pool()
96
97 return asyncio.run(wrapper())
98
99
100# ---------------------------------------------------------------------------
101# Disabled by default
102# ---------------------------------------------------------------------------
103
104
105def test_worker_pool_disabled_by_default_uses_subprocess_path(tmp_path, monkeypatch):
106 root = tmp_path / "objects"
107 write_source(
108 root / "basics" / "fast.py",
109 "def GET(request):\n return {'mode': 'object_source'}\n",
110 )
111 monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(root))
112 monkeypatch.delenv(object_worker_pool.WORKER_POOL_SIZE_ENV, raising=False)
113 monkeypatch.setenv(object_server.OBJECT_TIMEOUT_SECONDS_ENV, "5")
114
115 calls = []
116
117 def fake_subprocess(exec_request, roots=None, *, timeout_seconds, raise_on_error=False):
118 calls.append((exec_request.object_id, timeout_seconds))
119 return object_execution.ObjectExecutionResult(
120 object_id=exec_request.object_id,
121 method=exec_request.normalized_method(),
122 path=exec_request.path,
123 ok=True,
124 result={"mode": "subprocess"},
125 error=None,
126 started_at="2026-01-01T00:00:00Z",
127 finished_at="2026-01-01T00:00:00Z",
128 duration_ms=1.0,
129 )
130
131 monkeypatch.setattr(object_execution, "execute_python_object_subprocess", fake_subprocess)
132
133 status, _, payload = request("/objects/basics_fast")
134
135 assert status == 200
136 assert payload == {"mode": "subprocess"}
137 assert calls == [("basics_fast", 5.0)]
138 # The pool must never have been constructed at all.
139 assert object_server._worker_pool is None
140
141
142def test_trusted_in_process_object_still_bypasses_pool(tmp_path, monkeypatch):
143 root = tmp_path / "objects"
144 write_source(
145 root / "basics" / "fast.py",
146 "def GET(request):\n return {'mode': 'in_process'}\n",
147 )
148 monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(root))
149 monkeypatch.setenv(object_server.OBJECT_TIMEOUT_SECONDS_ENV, "5")
150 monkeypatch.setenv(object_server.TRUSTED_IN_PROCESS_OBJECTS_ENV, "basics_fast")
151 monkeypatch.setenv(object_worker_pool.WORKER_POOL_SIZE_ENV, "1")
152
153 status, _, payload = request("/objects/basics_fast")
154
155 assert status == 200
156 assert payload == {"mode": "in_process"}
157 # Trusted objects never touch the pool, so it should never spin up.
158 assert object_server._worker_pool is None
159
160
161# ---------------------------------------------------------------------------
162# Capabilities payload (no pool workers spawned either way)
163# ---------------------------------------------------------------------------
164
165
166def test_admin_status_reports_worker_pool_disabled_by_default(monkeypatch):
167 monkeypatch.delenv(object_worker_pool.WORKER_POOL_SIZE_ENV, raising=False)
168 enable_admin_token(monkeypatch)
169
170 status, _, payload = request("/admin/status", headers=auth_headers())
171
172 assert status == 200
173 assert payload["capabilities"]["worker_pool"] == {
174 "enabled": False,
175 "size": 0,
176 "env": "DBBASIC_WORKER_POOL_SIZE",
177 }
178
179
180def test_admin_status_reports_worker_pool_enabled_with_size(monkeypatch):
181 monkeypatch.setenv(object_worker_pool.WORKER_POOL_SIZE_ENV, "4")
182 enable_admin_token(monkeypatch)
183
184 status, _, payload = request("/admin/status", headers=auth_headers())
185
186 assert status == 200
187 assert payload["capabilities"]["worker_pool"] == {
188 "enabled": True,
189 "size": 4,
190 "env": "DBBASIC_WORKER_POOL_SIZE",
191 }
192
193
194# ---------------------------------------------------------------------------
195# Enabled: pool path taken instead of subprocess (spawns real workers)
196# ---------------------------------------------------------------------------
197
198
199def test_worker_pool_enabled_executes_via_pool_not_subprocess(tmp_path, monkeypatch):
200 root = tmp_path / "objects"
201 write_source(
202 root / "basics" / "fast.py",
203 "def GET(request):\n return {'mode': 'pool'}\n",
204 )
205 monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(root))
206 monkeypatch.setenv(object_server.OBJECT_TIMEOUT_SECONDS_ENV, "5")
207 monkeypatch.setenv(object_worker_pool.WORKER_POOL_SIZE_ENV, "1")
208
209 def fail_subprocess(*args, **kwargs):
210 raise AssertionError("pool is enabled -- subprocess-per-request must not run")
211
212 monkeypatch.setattr(object_execution, "execute_python_object_subprocess", fail_subprocess)
213
214 status, _, payload = run_scenario(lambda: asgi_request("/objects/basics_fast"))
215
216 assert status == 200
217 assert payload == {"mode": "pool"}
218
219
220def test_worker_pool_execution_error_reports_through_normal_error_path(tmp_path, monkeypatch):
221 root = tmp_path / "objects"
222 write_source(
223 root / "basics" / "broken.py",
224 "def GET(request):\n raise RuntimeError('boom')\n",
225 )
226 monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(root))
227 monkeypatch.setenv(object_server.OBJECT_TIMEOUT_SECONDS_ENV, "5")
228 monkeypatch.setenv(object_worker_pool.WORKER_POOL_SIZE_ENV, "1")
229
230 status, _, payload = run_scenario(lambda: asgi_request("/objects/basics_broken"))
231
232 assert status == 500
233 assert payload["status"] == "error"
234 assert "RuntimeError: boom" in payload["error"]
235
236
237def test_worker_pool_timeout_returns_504_matching_subprocess_message(tmp_path, monkeypatch):
238 root = tmp_path / "objects"
239 write_source(
240 root / "basics" / "slow.py",
241 "import time\n\ndef GET(request):\n time.sleep(5)\n return {'ok': True}\n",
242 )
243 monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(root))
244 monkeypatch.setenv(object_server.OBJECT_TIMEOUT_SECONDS_ENV, "0.5")
245 monkeypatch.setenv(object_worker_pool.WORKER_POOL_SIZE_ENV, "1")
246
247 status, _, payload = run_scenario(lambda: asgi_request("/objects/basics_slow"))
248
249 assert status == 504
250 assert payload["status"] == "error"
251 assert payload["error"] == "Execution failed: GET timed out for object basics_slow after 0.5 seconds"
252 assert object_correlation.normalize_correlation_id(payload["correlation_id"]) == payload[
253 "correlation_id"
254 ]
255
256
257# ---------------------------------------------------------------------------
258# Live edit through the real server call site, same warm worker
259# ---------------------------------------------------------------------------
260
261
262def test_worker_pool_live_edit_survives_across_requests_on_same_worker(tmp_path, monkeypatch):
263 root = tmp_path / "objects"
264 source = write_source(
265 root / "basics" / "live.py",
266 "def GET(request):\n return {'version': 1}\n",
267 )
268 monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(root))
269 monkeypatch.setenv(object_server.OBJECT_TIMEOUT_SECONDS_ENV, "5")
270 monkeypatch.setenv(object_worker_pool.WORKER_POOL_SIZE_ENV, "1") # single worker: same process both times
271
272 async def scenario():
273 first = await asgi_request("/objects/basics_live")
274 source.write_text("def GET(request):\n return {'version': 2}\n")
275 second = await asgi_request("/objects/basics_live")
276 return first, second
277
278 first, second = run_scenario(scenario)
279 status1, _, payload1 = first
280 status2, _, payload2 = second
281
282 assert status1 == 200 and payload1 == {"version": 1}
283 assert status2 == 200 and payload2 == {"version": 2}
284
285
286# ---------------------------------------------------------------------------
287# Concurrency: pool path must not serialize the event loop
288# ---------------------------------------------------------------------------
289
290
291def test_worker_pool_concurrent_requests_do_not_block_event_loop(tmp_path, monkeypatch):
292 root = tmp_path / "objects"
293 write_source(
294 root / "basics" / "sleepy.py",
295 "import time\n\ndef GET(request):\n time.sleep(0.3)\n return {'ok': True}\n",
296 )
297 monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(root))
298 monkeypatch.setenv(object_server.OBJECT_TIMEOUT_SECONDS_ENV, "5")
299 monkeypatch.setenv(object_worker_pool.WORKER_POOL_SIZE_ENV, "2")
300 monkeypatch.setenv(object_server.MAX_CONCURRENT_EXECUTIONS_ENV, "2")
301
302 async def scenario():
303 started = time_module.perf_counter()
304 results = await asyncio.gather(
305 asgi_request("/objects/basics_sleepy"),
306 asgi_request("/objects/basics_sleepy"),
307 )
308 elapsed = time_module.perf_counter() - started
309 return results, elapsed
310
311 results, elapsed = run_scenario(scenario)
312
313 assert all(status == 200 and payload == {"ok": True} for status, _, payload in results)
314 assert elapsed < 0.55, f"expected overlap, took {elapsed:.3f}s"