Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_daemon_control.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_daemon_control.py · 19.7 KB · 549 lines
1"""Admin controls for daemon-compatible scheduler and queue state."""
2
3from __future__ import annotations
4
5import json
6import re
7import time
8from collections.abc import Mapping
9from datetime import datetime, timezone
10from pathlib import Path
11from typing import Any
12
13import object_ids
14import object_state
15from object_namespace import validate_object_id
16from object_versions import DEFAULT_DATA_DIR, InvalidObjectIdError
17
18DEFAULT_LIMIT = 100
19MAX_LIMIT = 1000
20SCHEDULER_OBJECT_ID = "scheduler"
21QUEUE_OBJECT_ID = "queue"
22SCHEDULER_PREFIX = "task_"
23QUEUE_PREFIX = "msg_"
24VALID_TASK_STATUSES = {"active", "paused", "completed", "cancelled"}
25VALID_TASK_TYPES = {"cron", "onetime"}
26VALID_QUEUE_STATUSES = {"pending", "processing", "completed", "failed", "expired", "cancelled"}
27VALID_QUEUE_ACTIONS = {"cancel", "retry"}
28VALID_HTTP_METHODS = {"GET", "POST", "PUT", "PATCH", "DELETE"}
29_SAFE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$")
30_SAFE_QUEUE_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,63}$")
31
32
33class DaemonControlError(ValueError):
34 """Raised when daemon control input is invalid."""
35
36
37class DaemonItemNotFoundError(LookupError):
38 """Raised when a scheduler task or queue message cannot be found."""
39
40
41def list_scheduler_tasks(
42 *,
43 base_dir: Path | str = DEFAULT_DATA_DIR,
44 status: str | None = None,
45 limit: int = DEFAULT_LIMIT,
46 offset: int = 0,
47 include_payload: bool = False,
48) -> dict[str, Any]:
49 """Return scheduler tasks newest/nearest first."""
50 clean_status = _clean_status(status, VALID_TASK_STATUSES) if status is not None else None
51 clean_limit, clean_offset = _clean_page(limit, offset)
52 rows, invalid = _state_rows(
53 SCHEDULER_OBJECT_ID,
54 SCHEDULER_PREFIX,
55 base_dir=base_dir,
56 )
57 tasks = [row for _key, row in rows if clean_status is None or row.get("status") == clean_status]
58 tasks.sort(key=_task_sort_key)
59 total = len(tasks)
60 window = tasks[clean_offset:clean_offset + clean_limit]
61
62 return {
63 "tasks": [_public_task(task, include_payload=include_payload) for task in window],
64 "count": len(window),
65 "total": total,
66 "invalid": invalid,
67 "limit": clean_limit,
68 "offset": clean_offset,
69 "has_more": clean_offset + len(window) < total,
70 }
71
72
73def create_scheduler_task(
74 payload: Mapping[str, Any],
75 *,
76 actor: str = "api",
77 base_dir: Path | str = DEFAULT_DATA_DIR,
78 now: int | None = None,
79 include_payload: bool = False,
80) -> dict[str, Any]:
81 """Create one daemon-compatible scheduler task."""
82 current_time = _now(now)
83 object_id = _clean_object_id(payload.get("object_id"))
84 method = _clean_method(payload.get("method", "POST"))
85 task_type = _clean_choice(payload.get("type", "onetime"), VALID_TASK_TYPES, "type")
86 schedule = _clean_schedule(payload.get("schedule"))
87 next_run = _optional_epoch(payload.get("next_run"), "next_run")
88 if next_run is None and task_type == "onetime":
89 next_run = _optional_epoch(schedule, "schedule")
90
91 status = _clean_status(payload.get("status", "active"), VALID_TASK_STATUSES)
92 task_id = _clean_existing_id(payload.get("id")) if payload.get("id") else object_ids.new_uuid4()
93 task = {
94 "id": task_id,
95 "object_id": object_id,
96 "method": method,
97 "payload": _json_safe_payload(payload.get("payload", {})),
98 "type": task_type,
99 "schedule": schedule,
100 "status": status,
101 "next_run": next_run,
102 "last_run": None,
103 "run_count": 0,
104 "created_at": current_time,
105 "created_at_iso": _utc_timestamp(current_time),
106 "created_by": _clean_actor(actor),
107 "updated_at": current_time,
108 "updated_at_iso": _utc_timestamp(current_time),
109 "updated_by": _clean_actor(actor),
110 }
111
112 manager = _state_manager(SCHEDULER_OBJECT_ID, base_dir)
113 manager.set(_scheduler_key(task_id), _json_dumps(task))
114 return _public_task(task, include_payload=include_payload)
115
116
117def update_scheduler_task(
118 task_id: str,
119 payload: Mapping[str, Any],
120 *,
121 actor: str = "api",
122 base_dir: Path | str = DEFAULT_DATA_DIR,
123 now: int | None = None,
124 include_payload: bool = False,
125) -> dict[str, Any]:
126 """Patch one scheduler task without changing its state key."""
127 clean_task_id = _clean_existing_id(task_id)
128 manager = _state_manager(SCHEDULER_OBJECT_ID, base_dir)
129 key, task = _find_row(manager, SCHEDULER_PREFIX, clean_task_id, "Scheduler task")
130 current_time = _now(now)
131
132 if "status" in payload:
133 task["status"] = _clean_status(payload["status"], VALID_TASK_STATUSES)
134 if "next_run" in payload:
135 task["next_run"] = _optional_epoch(payload["next_run"], "next_run")
136 if "schedule" in payload:
137 schedule = _clean_schedule(payload["schedule"])
138 task["schedule"] = schedule
139 if task.get("type") == "onetime" and task.get("next_run") is None:
140 task["next_run"] = _optional_epoch(schedule, "schedule")
141 if "method" in payload:
142 task["method"] = _clean_method(payload["method"])
143 if "payload" in payload:
144 task["payload"] = _json_safe_payload(payload["payload"])
145 if "type" in payload:
146 task["type"] = _clean_choice(payload["type"], VALID_TASK_TYPES, "type")
147 if "object_id" in payload:
148 task["object_id"] = _clean_object_id(payload["object_id"])
149
150 task["updated_at"] = current_time
151 task["updated_at_iso"] = _utc_timestamp(current_time)
152 task["updated_by"] = _clean_actor(actor)
153 manager.set(key, _json_dumps(task))
154 return _public_task(task, include_payload=include_payload)
155
156
157def delete_scheduler_task(
158 task_id: str,
159 *,
160 base_dir: Path | str = DEFAULT_DATA_DIR,
161) -> dict[str, Any]:
162 """Delete one scheduler task."""
163 clean_task_id = _clean_existing_id(task_id)
164 manager = _state_manager(SCHEDULER_OBJECT_ID, base_dir)
165 key, task = _find_row(manager, SCHEDULER_PREFIX, clean_task_id, "Scheduler task")
166 manager.delete(key)
167 return _public_task(task, include_payload=False)
168
169
170def list_queue_messages(
171 *,
172 base_dir: Path | str = DEFAULT_DATA_DIR,
173 status: str | None = None,
174 queue_name: str | None = None,
175 limit: int = DEFAULT_LIMIT,
176 offset: int = 0,
177 include_payload: bool = False,
178) -> dict[str, Any]:
179 """Return queue messages newest/most actionable first."""
180 clean_status = _clean_status(status, VALID_QUEUE_STATUSES) if status is not None else None
181 clean_queue_name = _clean_queue_name(queue_name) if queue_name is not None else None
182 clean_limit, clean_offset = _clean_page(limit, offset)
183 rows, invalid = _state_rows(QUEUE_OBJECT_ID, QUEUE_PREFIX, base_dir=base_dir)
184 messages = [
185 row
186 for _key, row in rows
187 if (clean_status is None or row.get("status") == clean_status)
188 and (clean_queue_name is None or row.get("queue_name") == clean_queue_name)
189 ]
190 messages.sort(key=_message_sort_key)
191 total = len(messages)
192 window = messages[clean_offset:clean_offset + clean_limit]
193
194 return {
195 "messages": [_public_message(message, include_payload=include_payload) for message in window],
196 "count": len(window),
197 "total": total,
198 "invalid": invalid,
199 "limit": clean_limit,
200 "offset": clean_offset,
201 "has_more": clean_offset + len(window) < total,
202 }
203
204
205def enqueue_message(
206 payload: Mapping[str, Any],
207 *,
208 actor: str = "api",
209 base_dir: Path | str = DEFAULT_DATA_DIR,
210 now: int | None = None,
211 include_payload: bool = False,
212) -> dict[str, Any]:
213 """Create one daemon-compatible pending queue message."""
214 current_time = _now(now)
215 object_id = _clean_object_id(payload.get("object_id"))
216 method = _clean_method(payload.get("method", "POST"))
217 queue_name = _clean_queue_name(payload.get("queue_name", "default"))
218 priority_level = _int_value(payload.get("priority_level", 2), "priority_level", minimum=0)
219 message_id = _clean_existing_id(payload.get("id")) if payload.get("id") else object_ids.new_uuid4()
220 visible_after = _optional_epoch(payload.get("visible_after"), "visible_after")
221 expires_at = _optional_epoch(payload.get("expires_at"), "expires_at")
222 max_attempts = _int_value(payload.get("max_attempts", 3), "max_attempts", minimum=1)
223
224 message = {
225 "id": message_id,
226 "queue_name": queue_name,
227 "message": {
228 "object_id": object_id,
229 "method": method,
230 "payload": _json_safe_payload(payload.get("payload", {})),
231 },
232 "priority_level": priority_level,
233 "status": "pending",
234 "created_at": current_time,
235 "created_at_iso": _utc_timestamp(current_time),
236 "created_by": _clean_actor(actor),
237 "visible_after": current_time if visible_after is None else visible_after,
238 "expires_at": expires_at,
239 "attempts": 0,
240 "max_attempts": max_attempts,
241 "updated_at": current_time,
242 "updated_at_iso": _utc_timestamp(current_time),
243 "updated_by": _clean_actor(actor),
244 }
245
246 manager = _state_manager(QUEUE_OBJECT_ID, base_dir)
247 manager.set(_queue_key(message), _json_dumps(message))
248 return _public_message(message, include_payload=include_payload)
249
250
251def update_queue_message(
252 message_id: str,
253 payload: Mapping[str, Any],
254 *,
255 actor: str = "api",
256 base_dir: Path | str = DEFAULT_DATA_DIR,
257 now: int | None = None,
258 include_payload: bool = False,
259) -> dict[str, Any]:
260 """Patch one queue message, including admin retry/cancel actions."""
261 clean_message_id = _clean_existing_id(message_id)
262 manager = _state_manager(QUEUE_OBJECT_ID, base_dir)
263 key, message = _find_row(manager, QUEUE_PREFIX, clean_message_id, "Queue message")
264 current_time = _now(now)
265
266 action = payload.get("action")
267 if action is not None:
268 clean_action = _clean_choice(action, VALID_QUEUE_ACTIONS, "action")
269 if clean_action == "cancel":
270 message["status"] = "cancelled"
271 message["cancelled_at"] = current_time
272 message["cancelled_at_iso"] = _utc_timestamp(current_time)
273 message["cancelled_by"] = _clean_actor(actor)
274 elif clean_action == "retry":
275 message["status"] = "pending"
276 message["visible_after"] = current_time
277 message["retried_at"] = current_time
278 message["retried_at_iso"] = _utc_timestamp(current_time)
279 message["retried_by"] = _clean_actor(actor)
280
281 if "status" in payload:
282 message["status"] = _clean_status(payload["status"], VALID_QUEUE_STATUSES)
283 if "visible_after" in payload:
284 visible_after = _optional_epoch(payload["visible_after"], "visible_after")
285 message["visible_after"] = current_time if visible_after is None else visible_after
286 if "expires_at" in payload:
287 message["expires_at"] = _optional_epoch(payload["expires_at"], "expires_at")
288 if "priority_level" in payload:
289 message["priority_level"] = _int_value(payload["priority_level"], "priority_level", minimum=0)
290 if "max_attempts" in payload:
291 message["max_attempts"] = _int_value(payload["max_attempts"], "max_attempts", minimum=1)
292
293 message["updated_at"] = current_time
294 message["updated_at_iso"] = _utc_timestamp(current_time)
295 message["updated_by"] = _clean_actor(actor)
296 manager.set(key, _json_dumps(message))
297 return _public_message(message, include_payload=include_payload)
298
299
300def delete_queue_message(
301 message_id: str,
302 *,
303 base_dir: Path | str = DEFAULT_DATA_DIR,
304) -> dict[str, Any]:
305 """Delete one queue message."""
306 clean_message_id = _clean_existing_id(message_id)
307 manager = _state_manager(QUEUE_OBJECT_ID, base_dir)
308 key, message = _find_row(manager, QUEUE_PREFIX, clean_message_id, "Queue message")
309 manager.delete(key)
310 return _public_message(message, include_payload=False)
311
312
313def _state_manager(object_id: str, base_dir: Path | str) -> object_state.ObjectStateManager:
314 return object_state.ObjectStateManager(object_id, base_dir=base_dir)
315
316
317def _state_rows(
318 object_id: str,
319 prefix: str,
320 *,
321 base_dir: Path | str,
322) -> tuple[list[tuple[str, dict[str, Any]]], int]:
323 rows: list[tuple[str, dict[str, Any]]] = []
324 invalid = 0
325 for key, value in _state_manager(object_id, base_dir).get_all().items():
326 if not key.startswith(prefix):
327 continue
328 row = _load_json_object(value)
329 if row is None:
330 invalid += 1
331 continue
332 row.setdefault("id", key.removeprefix(prefix))
333 rows.append((key, row))
334 return rows, invalid
335
336
337def _find_row(
338 manager: object_state.ObjectStateManager,
339 prefix: str,
340 item_id: str,
341 label: str,
342) -> tuple[str, dict[str, Any]]:
343 for key, value in manager.get_all().items():
344 if not key.startswith(prefix):
345 continue
346 row = _load_json_object(value)
347 if row is None:
348 continue
349 if str(row.get("id", "")) == item_id or key == f"{prefix}{item_id}":
350 return key, row
351 raise DaemonItemNotFoundError(f"{label} not found: {item_id}")
352
353
354def _scheduler_key(task_id: str) -> str:
355 return f"{SCHEDULER_PREFIX}{task_id}"
356
357
358def _queue_key(message: Mapping[str, Any]) -> str:
359 queue_name = _clean_queue_name(message.get("queue_name", "default"))
360 priority = _int_value(message.get("priority_level", 2), "priority_level", minimum=0)
361 created_at = _int_value(message.get("created_at", 0), "created_at", minimum=0)
362 message_id = _clean_existing_id(message.get("id"))
363 return f"{QUEUE_PREFIX}{queue_name}_{priority}_{created_at}_{message_id}"
364
365
366def _public_task(task: Mapping[str, Any], *, include_payload: bool) -> dict[str, Any]:
367 public = dict(task)
368 payload = public.pop("payload", None)
369 public["payload_present"] = payload not in ({}, None, "")
370 if include_payload:
371 public["payload"] = payload
372 return public
373
374
375def _public_message(message: Mapping[str, Any], *, include_payload: bool) -> dict[str, Any]:
376 public = dict(message)
377 body = public.get("message")
378 if isinstance(body, Mapping):
379 body_public = dict(body)
380 payload = body_public.pop("payload", None)
381 body_public["payload_present"] = payload not in ({}, None, "")
382 if include_payload:
383 body_public["payload"] = payload
384 public["message"] = body_public
385 return public
386
387
388def _task_sort_key(task: Mapping[str, Any]) -> tuple[int, int, str]:
389 next_run = _coerce_int(task.get("next_run"))
390 created_at = _coerce_int(task.get("created_at"))
391 return (next_run if next_run is not None else 2**63 - 1, created_at or 0, str(task.get("id", "")))
392
393
394def _message_sort_key(message: Mapping[str, Any]) -> tuple[int, int, int, str]:
395 status_rank = 0 if message.get("status") == "pending" else 1
396 visible_after = _coerce_int(message.get("visible_after")) or 0
397 priority = _coerce_int(message.get("priority_level")) or 0
398 return (status_rank, visible_after, -priority, str(message.get("id", "")))
399
400
401def _load_json_object(value: Any) -> dict[str, Any] | None:
402 if isinstance(value, Mapping):
403 return dict(value)
404 if not isinstance(value, str):
405 return None
406 try:
407 loaded = json.loads(value)
408 except (TypeError, json.JSONDecodeError):
409 return None
410 if not isinstance(loaded, Mapping):
411 return None
412 return dict(loaded)
413
414
415def _json_safe_payload(payload: Any) -> Any:
416 try:
417 json.dumps(payload)
418 except (TypeError, ValueError) as exc:
419 raise DaemonControlError("payload must be JSON serializable") from exc
420 return payload
421
422
423def _json_dumps(value: Mapping[str, Any]) -> str:
424 return json.dumps(value, sort_keys=True, separators=(",", ":"))
425
426
427def _clean_object_id(value: Any) -> str:
428 if not isinstance(value, str) or not value.strip():
429 raise DaemonControlError("object_id is required")
430 clean = value.strip()
431 if not validate_object_id(clean):
432 raise InvalidObjectIdError(f"Invalid object ID: {clean}")
433 return clean
434
435
436def _clean_method(value: Any) -> str:
437 if not isinstance(value, str) or not value.strip():
438 raise DaemonControlError("method must be a non-empty string")
439 clean = value.strip().upper()
440 if clean not in VALID_HTTP_METHODS:
441 raise DaemonControlError(f"method must be one of: {', '.join(sorted(VALID_HTTP_METHODS))}")
442 return clean
443
444
445def _clean_schedule(value: Any) -> str:
446 if not isinstance(value, str) or not value.strip():
447 raise DaemonControlError("schedule is required")
448 return value.strip()
449
450
451def _clean_status(value: Any, allowed: set[str]) -> str:
452 return _clean_choice(value, allowed, "status")
453
454
455def _clean_choice(value: Any, allowed: set[str], field: str) -> str:
456 if not isinstance(value, str) or not value.strip():
457 raise DaemonControlError(f"{field} must be a non-empty string")
458 clean = value.strip().lower()
459 if clean not in allowed:
460 raise DaemonControlError(f"{field} must be one of: {', '.join(sorted(allowed))}")
461 return clean
462
463
464def _clean_existing_id(value: Any) -> str:
465 if not isinstance(value, str) or not value.strip():
466 raise DaemonControlError("id must be a non-empty string")
467 clean = value.strip()
468 if not _SAFE_ID_RE.fullmatch(clean):
469 raise DaemonControlError(f"Invalid id: {clean}")
470 return clean
471
472
473def _clean_queue_name(value: Any) -> str:
474 if not isinstance(value, str) or not value.strip():
475 raise DaemonControlError("queue_name must be a non-empty string")
476 clean = value.strip()
477 if not _SAFE_QUEUE_RE.fullmatch(clean):
478 raise DaemonControlError(f"Invalid queue_name: {clean}")
479 return clean
480
481
482def _clean_actor(actor: str) -> str:
483 return str(actor or "api").strip()[:128] or "api"
484
485
486def _clean_page(limit: int, offset: int) -> tuple[int, int]:
487 clean_limit = _int_value(limit, "limit", minimum=1)
488 if clean_limit > MAX_LIMIT:
489 raise DaemonControlError(f"limit must be at most {MAX_LIMIT}")
490 clean_offset = _int_value(offset, "offset", minimum=0)
491 return clean_limit, clean_offset
492
493
494def _int_value(value: Any, field: str, *, minimum: int | None = None) -> int:
495 if isinstance(value, bool):
496 raise DaemonControlError(f"{field} must be an integer")
497 try:
498 parsed = int(value)
499 except (TypeError, ValueError) as exc:
500 raise DaemonControlError(f"{field} must be an integer") from exc
501 if minimum is not None and parsed < minimum:
502 raise DaemonControlError(f"{field} must be at least {minimum}")
503 return parsed
504
505
506def _optional_epoch(value: Any, field: str) -> int | None:
507 if value is None or value == "":
508 return None
509 if isinstance(value, bool):
510 raise DaemonControlError(f"{field} must be an epoch integer or ISO timestamp")
511 if isinstance(value, (int, float)):
512 if int(value) < 0:
513 raise DaemonControlError(f"{field} must be at least 0")
514 return int(value)
515 if isinstance(value, str):
516 clean = value.strip()
517 if not clean:
518 return None
519 try:
520 parsed = int(clean)
521 except ValueError:
522 try:
523 dt = datetime.fromisoformat(clean.replace("Z", "+00:00"))
524 except ValueError as exc:
525 raise DaemonControlError(
526 f"{field} must be an epoch integer or ISO timestamp"
527 ) from exc
528 if dt.tzinfo is None:
529 dt = dt.replace(tzinfo=timezone.utc)
530 return int(dt.timestamp())
531 if parsed < 0:
532 raise DaemonControlError(f"{field} must be at least 0")
533 return parsed
534 raise DaemonControlError(f"{field} must be an epoch integer or ISO timestamp")
535
536
537def _coerce_int(value: Any) -> int | None:
538 try:
539 return int(value)
540 except (TypeError, ValueError):
541 return None
542
543
544def _now(now: int | None) -> int:
545 return int(time.time() if now is None else now)
546
547
548def _utc_timestamp(timestamp: int) -> str:
549 return datetime.fromtimestamp(timestamp, timezone.utc).isoformat().replace("+00:00", "Z")