Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_worker_pool.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_worker_pool.py · 19.5 KB · 567 lines
1"""Persistent worker-process pool for near-in-process object execution.
2
3Objects that are not listed in ``DBBASIC_TRUSTED_IN_PROCESS_OBJECTS`` run
4through ``object_execution.execute_python_object_subprocess`` today, which
5spawns a brand new interpreter for every single request. That keeps process
6isolation and wall-clock timeout enforcement, but a cold interpreter boot
7dominates latency (roughly 100-200ms) for objects that could otherwise run
8in a couple of milliseconds.
9
10This module keeps a small pool of long-lived worker subprocesses around
11instead. Each worker runs a tiny read-request/write-response loop
12(``_worker_main`` below, reachable as ``python -m object_worker_pool
13--worker``) and re-loads the target object's source fresh on every request
14via ``python_object_runtime.PythonObjectRuntime`` -- so a warm worker still
15picks up source edits made between requests (the "live edit" guarantee
16objects depend on). Isolation and timeout enforcement are preserved because
17every request still runs in its own OS process; only the process-spawn cost
18is amortized away by keeping the process around between requests.
19
20Disabled by default: ``DBBASIC_WORKER_POOL_SIZE`` unset or ``0`` means the
21pool is never constructed and callers should keep using the existing
22spawn-per-request path untouched.
23
24Wire protocol
25--------------
26Parent and worker exchange one JSON message per request/response, each
27framed as a 4-byte big-endian length prefix followed by that many bytes of
28UTF-8 JSON (``_encode_message`` / ``_read_message_async`` /
29``_read_message_sync``).
30
31Request message::
32
33 {
34 "object_id": str,
35 "method": str,
36 "payload": dict,
37 "correlation_id": str | None,
38 "path": str | None, # pre-resolved by the pool manager
39 "base_dir": str | None, # optional PythonObjectRuntime base_dir override
40 }
41
42Response message is ``object_execution.ObjectExecutionResult.to_dict()``.
43
44Worker stderr is left connected to the parent's stderr (not piped), so
45worker-side tracebacks/log noise land in the normal server logs.
46"""
47from __future__ import annotations
48
49import asyncio
50import atexit
51import json
52import os
53import signal
54import struct
55import sys
56import time
57from pathlib import Path
58from typing import Any, Iterable
59
60import object_execution
61from object_execution import (
62 ObjectExecutionError,
63 ObjectExecutionRequest,
64 ObjectExecutionResult,
65)
66from object_namespace import resolve_object_id
67
68WORKER_POOL_SIZE_ENV = "DBBASIC_WORKER_POOL_SIZE"
69
70_LENGTH_PREFIX = struct.Struct(">I")
71_MAX_MESSAGE_BYTES = 64 * 1024 * 1024 # guard against a corrupt/desynced stream
72
73# Reuse object_execution's private timestamp/duration/format helpers so the
74# results this module builds (timeouts, crashes, worker responses) are
75# byte-for-byte identical in shape to the ones execute_python_object_subprocess
76# produces. These are considered part of the internal contract between the
77# two modules, not object_execution's public API.
78_utc_timestamp = object_execution._utc_timestamp
79_duration_ms = object_execution._duration_ms
80_format_seconds = object_execution._format_seconds
81
82
83# ---------------------------------------------------------------------------
84# Wire protocol helpers
85# ---------------------------------------------------------------------------
86
87
88def _encode_message(payload: dict[str, Any]) -> bytes:
89 body = json.dumps(payload).encode("utf-8")
90 return _LENGTH_PREFIX.pack(len(body)) + body
91
92
93async def _read_message_async(reader: asyncio.StreamReader) -> dict[str, Any]:
94 header = await reader.readexactly(4)
95 (length,) = _LENGTH_PREFIX.unpack(header)
96 if length > _MAX_MESSAGE_BYTES:
97 raise ValueError(f"worker message too large ({length} bytes)")
98 body = await reader.readexactly(length)
99 return json.loads(body.decode("utf-8"))
100
101
102def _read_message_sync(stream: Any) -> dict[str, Any] | None:
103 header = _read_exact_sync(stream, 4)
104 if header is None:
105 return None
106 (length,) = _LENGTH_PREFIX.unpack(header)
107 body = _read_exact_sync(stream, length)
108 if body is None:
109 return None
110 return json.loads(body.decode("utf-8"))
111
112
113def _read_exact_sync(stream: Any, size: int) -> bytes | None:
114 chunks: list[bytes] = []
115 remaining = size
116 while remaining > 0:
117 chunk = stream.read(remaining)
118 if not chunk:
119 return None
120 chunks.append(chunk)
121 remaining -= len(chunk)
122 return b"".join(chunks)
123
124
125# ---------------------------------------------------------------------------
126# Worker process (child side)
127# ---------------------------------------------------------------------------
128
129
130def _worker_main() -> None:
131 """Entry point for a worker subprocess (``python -m object_worker_pool --worker``)."""
132 from python_object_runtime import PythonObjectRuntime
133
134 stdin = sys.stdin.buffer
135 stdout = sys.stdout.buffer
136
137 while True:
138 try:
139 request_message = _read_message_sync(stdin)
140 except Exception:
141 return
142 if request_message is None:
143 return # parent closed stdin (pool shutdown) -- exit cleanly
144
145 response = _handle_worker_request(request_message, PythonObjectRuntime())
146 try:
147 stdout.write(_encode_message(response))
148 stdout.flush()
149 except Exception:
150 return
151
152
153def _handle_worker_request(message: dict[str, Any], runtime: Any) -> dict[str, Any]:
154 object_id = message.get("object_id") or ""
155 method = message.get("method") or "GET"
156 payload = message.get("payload") or {}
157 correlation_id = message.get("correlation_id")
158 path_value = message.get("path")
159
160 request = ObjectExecutionRequest(
161 object_id=object_id,
162 method=method,
163 payload=payload,
164 path=Path(path_value) if path_value else None,
165 correlation_id=correlation_id,
166 )
167
168 try:
169 result = object_execution.execute_object(runtime, request, roots=None)
170 except BaseException as exc: # keep the worker loop alive no matter what happened
171 result = _worker_error_result(request, exc)
172
173 return result.to_dict()
174
175
176def _worker_error_result(request: ObjectExecutionRequest, exc: BaseException) -> ObjectExecutionResult:
177 import traceback
178
179 now = _utc_timestamp()
180 return ObjectExecutionResult(
181 object_id=request.object_id,
182 method=request.normalized_method(),
183 path=request.path,
184 ok=False,
185 result=None,
186 error=ObjectExecutionError(
187 type=object_execution.WORKER_ERROR_TYPE,
188 message=f"Object worker failed: {type(exc).__name__}: {exc}",
189 traceback="".join(traceback.format_exception(type(exc), exc, exc.__traceback__)),
190 ),
191 started_at=now,
192 finished_at=now,
193 duration_ms=0.0,
194 correlation_id=request.correlation_id,
195 )
196
197
198# ---------------------------------------------------------------------------
199# Pool manager (parent side)
200# ---------------------------------------------------------------------------
201
202_LAZY = object() # sentinel occupying a queue slot: "spawn a worker on demand"
203
204
205class _Worker:
206 __slots__ = ("process", "pid")
207
208 def __init__(self, process: "asyncio.subprocess.Process"):
209 self.process = process
210 self.pid = process.pid
211
212
213def _error_result(
214 request: ObjectExecutionRequest,
215 method: str,
216 error_type: str,
217 message: str,
218 started_at: str,
219 started_perf: float,
220) -> ObjectExecutionResult:
221 return ObjectExecutionResult(
222 object_id=request.object_id,
223 method=method,
224 path=request.path,
225 ok=False,
226 result=None,
227 error=ObjectExecutionError(type=error_type, message=message),
228 started_at=started_at,
229 finished_at=_utc_timestamp(),
230 duration_ms=_duration_ms(started_perf),
231 correlation_id=request.correlation_id,
232 )
233
234
235def _result_from_dict(data: dict[str, Any]) -> ObjectExecutionResult:
236 error = None
237 error_data = data.get("error")
238 if error_data:
239 error = ObjectExecutionError(
240 type=error_data["type"],
241 message=error_data["message"],
242 traceback=error_data.get("traceback", ""),
243 )
244
245 path_value = data.get("path")
246 return ObjectExecutionResult(
247 object_id=data["object_id"],
248 method=data["method"],
249 path=Path(path_value) if path_value else None,
250 ok=bool(data["ok"]),
251 result=data.get("result"),
252 error=error,
253 started_at=data["started_at"],
254 finished_at=data["finished_at"],
255 duration_ms=data["duration_ms"],
256 correlation_id=data.get("correlation_id"),
257 )
258
259
260def _request_with_resolved_path(
261 request: ObjectExecutionRequest,
262 roots: Iterable[Path] | None,
263) -> ObjectExecutionRequest:
264 if request.path is not None:
265 return request
266 return ObjectExecutionRequest(
267 object_id=request.object_id,
268 method=request.method,
269 payload=request.payload,
270 path=resolve_object_id(request.object_id, roots),
271 correlation_id=request.correlation_id,
272 )
273
274
275class WorkerPool:
276 """Async pool of persistent worker subprocesses.
277
278 Workers are spawned lazily -- nothing is started until the first
279 ``execute()`` call needs one, and at most ``size`` workers ever run at
280 once. Each worker handles one request at a time; ``execute()`` acquires
281 a free worker (or spawns one if a lazy slot is available), ships it one
282 request over the wire protocol above, and returns it to the pool when
283 done. A worker that times out or crashes is killed/discarded and its
284 slot is put back as "lazy" so a replacement spawns on next use.
285 """
286
287 def __init__(
288 self,
289 size: int,
290 *,
291 env: dict[str, str] | None = None,
292 python_executable: str | None = None,
293 ) -> None:
294 if size <= 0:
295 raise ValueError("WorkerPool size must be positive")
296 self.size = size
297 self._env = dict(env) if env is not None else None
298 self._python_executable = python_executable or sys.executable
299 self._loop: asyncio.AbstractEventLoop | None = None
300 self._queue: asyncio.Queue | None = None
301 self._known_pids: set[int] = set()
302 self._closed = False
303 atexit.register(self._atexit_kill)
304
305 # -- queue / lifecycle -------------------------------------------------
306
307 def _queue_for_current_loop(self) -> asyncio.Queue:
308 loop = asyncio.get_running_loop()
309 if self._loop is not loop:
310 # A pool must not straddle event loops: asyncio subprocess
311 # transports are bound to the loop that created them. If we're
312 # being used from a new loop (process restart of the loop, or a
313 # test harness that opens a fresh loop per call), hard-kill any
314 # previously known workers by pid (loop-independent) and start
315 # over with fresh lazy slots.
316 self._reap_stale_workers()
317 self._loop = loop
318 self._queue = asyncio.Queue()
319 for _ in range(self.size):
320 self._queue.put_nowait(_LAZY)
321 assert self._queue is not None
322 return self._queue
323
324 def _reap_stale_workers(self) -> None:
325 for pid in self._known_pids:
326 _hard_kill(pid)
327 self._known_pids.clear()
328
329 def _atexit_kill(self) -> None:
330 for pid in list(self._known_pids):
331 _hard_kill(pid)
332
333 async def _spawn_worker(self) -> _Worker:
334 kwargs: dict[str, Any] = {
335 "stdin": asyncio.subprocess.PIPE,
336 "stdout": asyncio.subprocess.PIPE,
337 "stderr": None, # inherit -- worker stderr flows to server logs
338 }
339 if self._env is not None:
340 kwargs["env"] = self._env
341 process = await asyncio.create_subprocess_exec(
342 self._python_executable,
343 "-m",
344 "object_worker_pool",
345 "--worker",
346 **kwargs,
347 )
348 worker = _Worker(process)
349 self._known_pids.add(worker.pid)
350 return worker
351
352 async def _kill_worker(self, worker: _Worker) -> None:
353 process = worker.process
354 self._known_pids.discard(worker.pid)
355 if process.returncode is not None:
356 return
357 try:
358 process.kill()
359 except ProcessLookupError:
360 return
361 try:
362 await asyncio.wait_for(process.wait(), timeout=1.0)
363 except asyncio.TimeoutError:
364 pass
365
366 async def _exit_code(self, worker: _Worker) -> int | None:
367 process = worker.process
368 if process.returncode is not None:
369 return process.returncode
370 try:
371 await asyncio.wait_for(process.wait(), timeout=0.5)
372 except asyncio.TimeoutError:
373 return None
374 return process.returncode
375
376 # -- execution -----------------------------------------------------
377
378 async def execute(
379 self,
380 request: ObjectExecutionRequest,
381 *,
382 timeout_seconds: float,
383 roots: Iterable[Path] | None = None,
384 ) -> ObjectExecutionResult:
385 """Execute one object method on a warm worker, enforcing timeout_seconds."""
386 if self._closed:
387 raise RuntimeError("WorkerPool is closed")
388
389 started_perf = time.perf_counter()
390 started_at = _utc_timestamp()
391 method = request.normalized_method()
392 resolved_request = _request_with_resolved_path(request, roots)
393
394 message = {
395 "object_id": resolved_request.object_id,
396 "method": method,
397 "payload": resolved_request.normalized_payload(),
398 "correlation_id": resolved_request.correlation_id,
399 "path": str(resolved_request.path) if resolved_request.path is not None else None,
400 }
401 # Encode before taking a worker: a payload JSON can't carry (bytes,
402 # say) used to raise out of here holding the worker, and once every
403 # worker had leaked that way each request waited forever.
404 try:
405 encoded = _encode_message(message)
406 except (TypeError, ValueError) as exc:
407 return _error_result(
408 resolved_request,
409 method,
410 object_execution.WORKER_ERROR_TYPE,
411 f"Request can't be sent to an object worker: {type(exc).__name__}: {exc}",
412 started_at,
413 started_perf,
414 )
415
416 queue = self._queue_for_current_loop()
417 try:
418 slot = await asyncio.wait_for(queue.get(), timeout=timeout_seconds)
419 except asyncio.TimeoutError:
420 return _error_result(
421 resolved_request,
422 method,
423 object_execution.TIMEOUT_ERROR_TYPE,
424 (
425 f"No object worker came free for {resolved_request.object_id} "
426 f"within {_format_seconds(timeout_seconds)} seconds"
427 ),
428 started_at,
429 started_perf,
430 )
431 worker = slot if isinstance(slot, _Worker) else None
432
433 if worker is None:
434 try:
435 worker = await self._spawn_worker()
436 except BaseException as exc:
437 queue.put_nowait(_LAZY)
438 if not isinstance(exc, Exception):
439 raise
440 return _error_result(
441 resolved_request,
442 method,
443 object_execution.WORKER_ERROR_TYPE,
444 f"Failed to start object worker: {type(exc).__name__}: {exc}",
445 started_at,
446 started_perf,
447 )
448
449 worker_died = False
450 try:
451 response = await asyncio.wait_for(
452 _talk_to_worker(worker, encoded),
453 timeout=timeout_seconds,
454 )
455 except asyncio.TimeoutError:
456 worker_died = True
457 await self._kill_worker(worker)
458 result = _error_result(
459 resolved_request,
460 method,
461 object_execution.TIMEOUT_ERROR_TYPE,
462 (
463 f"{method} timed out for object {resolved_request.object_id} "
464 f"after {_format_seconds(timeout_seconds)} seconds"
465 ),
466 started_at,
467 started_perf,
468 )
469 except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError, OSError):
470 worker_died = True
471 exit_code = await self._exit_code(worker)
472 self._known_pids.discard(worker.pid)
473 result = _error_result(
474 resolved_request,
475 method,
476 object_execution.WORKER_ERROR_TYPE,
477 f"Object worker exited without a result (exit code {exit_code})",
478 started_at,
479 started_perf,
480 )
481 except Exception as exc:
482 # A reply we can't read (too large, not JSON) leaves the worker's
483 # stream in an unknown state: replace the worker.
484 worker_died = True
485 self._known_pids.discard(worker.pid)
486 _hard_kill(worker.pid)
487 result = _error_result(
488 resolved_request,
489 method,
490 object_execution.WORKER_ERROR_TYPE,
491 f"Object worker sent a reply that couldn't be read: {type(exc).__name__}: {exc}",
492 started_at,
493 started_perf,
494 )
495 except BaseException:
496 # Cancelled mid-conversation: same, and the slot still goes back
497 # (finally) so the pool never shrinks.
498 worker_died = True
499 self._known_pids.discard(worker.pid)
500 _hard_kill(worker.pid)
501 raise
502 else:
503 result = _result_from_dict(response)
504 finally:
505 queue.put_nowait(_LAZY if worker_died else worker)
506
507 return result
508
509 async def shutdown(self) -> None:
510 """Best-effort termination of every known worker. Never blocks long."""
511 self._closed = True
512 atexit.unregister(self._atexit_kill)
513
514 try:
515 loop = asyncio.get_running_loop()
516 except RuntimeError:
517 loop = None
518
519 if self._queue is None or loop is not self._loop:
520 # Either never used, or being shut down from a different event
521 # loop than the one that spawned the current workers (asyncio
522 # subprocess transports can't be awaited from another loop).
523 # Hard-kill by pid instead of the graceful kill()+wait() path.
524 self._reap_stale_workers()
525 return
526
527 workers: list[_Worker] = []
528 while not self._queue.empty():
529 slot = self._queue.get_nowait()
530 if isinstance(slot, _Worker):
531 workers.append(slot)
532 for worker in workers:
533 await self._kill_worker(worker)
534 self._known_pids.clear()
535
536
537async def _talk_to_worker(worker: _Worker, encoded: bytes) -> dict[str, Any]:
538 process = worker.process
539 assert process.stdin is not None and process.stdout is not None
540 process.stdin.write(encoded)
541 await process.stdin.drain()
542 return await _read_message_async(process.stdout)
543
544
545def _hard_kill(pid: int) -> None:
546 try:
547 os.kill(pid, signal.SIGKILL)
548 except (ProcessLookupError, PermissionError, OSError):
549 pass
550
551
552# ---------------------------------------------------------------------------
553# CLI entry point
554# ---------------------------------------------------------------------------
555
556
557def main(argv: list[str] | None = None) -> int:
558 args = sys.argv[1:] if argv is None else argv
559 if "--worker" in args:
560 _worker_main()
561 return 0
562 print("usage: python -m object_worker_pool --worker", file=sys.stderr)
563 return 2
564
565
566if __name__ == "__main__":
567 raise SystemExit(main())