askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_daemon.py · 89.5 KB · 2161 lines
| 1 | #!/usr/bin/env python3 |
| 2 | """ |
| 3 | Object Primitive Daemon — Scheduler, Queue, Events |
| 4 | |
| 5 | Background worker that polls trigger objects and executes target objects. |
| 6 | |
| 7 | Runs alongside the HTTP server, sharing file-based state (TSV). |
| 8 | Does not require HTTP or auth — executes objects directly via ObjectRuntime. |
| 9 | |
| 10 | Usage: |
| 11 | python object_daemon.py |
| 12 | python object_daemon.py --interval 5 # poll every 5 seconds |
| 13 | """ |
| 14 | from __future__ import annotations |
| 15 | |
| 16 | import json |
| 17 | import os |
| 18 | import signal |
| 19 | import time |
| 20 | import urllib.error |
| 21 | import urllib.request |
| 22 | from datetime import datetime, timedelta, timezone |
| 23 | from pathlib import Path |
| 24 | from typing import TYPE_CHECKING, Any |
| 25 | |
| 26 | if TYPE_CHECKING: |
| 27 | from dbbasic_object_core.runtime.object_runtime import ObjectRuntime |
| 28 | |
| 29 | try: |
| 30 | from croniter import croniter |
| 31 | except ImportError: |
| 32 | croniter = None |
| 33 | |
| 34 | import object_analytics |
| 35 | import object_backup_index |
| 36 | import object_books_status |
| 37 | import object_change_dispatch |
| 38 | import object_collections |
| 39 | import object_connectors |
| 40 | import object_email |
| 41 | import object_events |
| 42 | import object_identity |
| 43 | import object_ids |
| 44 | import object_materialize |
| 45 | import object_notify |
| 46 | import object_packages |
| 47 | import object_record_changes |
| 48 | import object_records |
| 49 | import object_rollups |
| 50 | from object_execution import ObjectExecutionFailure, ObjectExecutionRequest, execute_object |
| 51 | from object_namespace import find_trigger_file, get_object_roots, resolve_object_id |
| 52 | |
| 53 | # Daemon state |
| 54 | _running = True |
| 55 | EVENT_KEEP_COUNT_ENV = "DBBASIC_EVENT_KEEP_COUNT" |
| 56 | EVENT_KEEP_SECONDS_ENV = "DBBASIC_EVENT_KEEP_SECONDS" |
| 57 | EVENT_CLEANUP_INTERVAL_SECONDS = 60.0 |
| 58 | |
| 59 | # Compaction pass (docs/append-only-storage-design.md Compaction: "run on |
| 60 | # a schedule or when the superseded-row ratio passes a threshold -- never |
| 61 | # inline in a request"). See process_compactions. |
| 62 | COMPACTION_INTERVAL_SECONDS_ENV = "DBBASIC_COMPACTION_INTERVAL_SECONDS" |
| 63 | COMPACTION_BLOAT_RATIO_ENV = "DBBASIC_COMPACTION_BLOAT_RATIO" |
| 64 | _DEFAULT_COMPACTION_INTERVAL_SECONDS = 3600 |
| 65 | _DEFAULT_COMPACTION_BLOAT_RATIO = 1.0 |
| 66 | COMPACTION_MARKER_NAME = ".compaction_last_run" |
| 67 | |
| 68 | # Stale-state auto-transition pass. The predecessor system's "48-hour |
| 69 | # auto-approve" job (waiting_on_client -> approved on tasks) was written |
| 70 | # but never scheduled anywhere, silently stranding 200+ records in a |
| 71 | # waiting state forever. To make that class of mistake impossible here, |
| 72 | # this pass ships ON by default -- see _DEFAULT_AUTO_TRANSITION_RULES -- |
| 73 | # and requires no setup beyond running the daemon. See |
| 74 | # process_stale_transitions. |
| 75 | AUTO_TRANSITION_RULES_ENV = "DBBASIC_AUTO_TRANSITION_RULES" |
| 76 | AUTO_TRANSITION_INTERVAL_SECONDS_ENV = "DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS" |
| 77 | _DEFAULT_AUTO_TRANSITION_INTERVAL_SECONDS = 3600 |
| 78 | AUTO_TRANSITION_MARKER_NAME = ".auto_transition_last_run" |
| 79 | AUTO_TRANSITION_ACTOR = "daemon:auto-transition" |
| 80 | _DEFAULT_AUTO_TRANSITION_RULES = json.dumps([ |
| 81 | {"collection": "tasks", "field": "status", "from": "waiting_on_client", |
| 82 | "to": "approved", "after_hours": 48}, |
| 83 | ]) |
| 84 | |
| 85 | # Collections named in a rule but not found on disk (the owning package |
| 86 | # isn't installed) are logged once per process, not once per poll -- |
| 87 | # see _apply_auto_transition_rule. |
| 88 | _WARNED_UNKNOWN_AUTO_TRANSITION_COLLECTIONS: set[str] = set() |
| 89 | |
| 90 | # Materialize pass (plan/vocabulary/61-materialize-spec.md, object_materialize.py). |
| 91 | # Unlike process_rollups (whose due-gate lives entirely on each |
| 92 | # rollup_definitions row -- see process_rollups' own docstring), |
| 93 | # materialize's Events section explicitly asks for process_stale_ |
| 94 | # transitions' marker-file-gated shape instead: "riding the exact |
| 95 | # marker-file-gated, try/except-per-item pattern process_stale_ |
| 96 | # transitions already establishes." See process_materializations. |
| 97 | MATERIALIZE_INTERVAL_SECONDS_ENV = "DBBASIC_MATERIALIZE_INTERVAL_SECONDS" |
| 98 | _DEFAULT_MATERIALIZE_INTERVAL_SECONDS = 3600 |
| 99 | MATERIALIZE_MARKER_NAME = ".materialize_last_run" |
| 100 | # 12 notify: the cursor file holds the ISO timestamp of the last record-change |
| 101 | # entry process_notifications has already turned into notifications. Unlike the |
| 102 | # other passes' interval markers, this is a POSITION (content), not a clock |
| 103 | # (mtime) -- the pass runs every poll for near-instant delivery and advances |
| 104 | # the cursor past what it processed. First run stamps it at "now" so a fresh |
| 105 | # install never backfills notifications for the entire change history. |
| 106 | NOTIFY_CURSOR_NAME = ".notify_cursor" |
| 107 | |
| 108 | # Definitions referencing a source/output/child collection that doesn't |
| 109 | # exist yet (the owning package isn't installed) are logged once per |
| 110 | # process, not once per poll -- mirrors _WARNED_UNKNOWN_AUTO_TRANSITION_ |
| 111 | # COLLECTIONS exactly (61's Degradation section names this precedent). |
| 112 | _WARNED_UNKNOWN_MATERIALIZE_COLLECTIONS: set[str] = set() |
| 113 | |
| 114 | |
| 115 | def log(msg, level='INFO'): |
| 116 | ts = time.strftime('%Y-%m-%d %H:%M:%S') |
| 117 | print(f"[{ts}] [{level}] {msg}") |
| 118 | |
| 119 | |
| 120 | def _object_roots() -> list[Path]: |
| 121 | """Return object source roots in lookup order.""" |
| 122 | return get_object_roots() |
| 123 | |
| 124 | |
| 125 | def _find_trigger_file(trigger_name: str) -> Path | None: |
| 126 | """Find a trigger object in the configured object roots.""" |
| 127 | return find_trigger_file(trigger_name) |
| 128 | |
| 129 | |
| 130 | # --- Scheduler --- |
| 131 | |
| 132 | def process_scheduler(runtime: ObjectRuntime): |
| 133 | """Check scheduler for due tasks and execute them.""" |
| 134 | scheduler_file = _find_trigger_file('scheduler') |
| 135 | if scheduler_file is None: |
| 136 | return |
| 137 | |
| 138 | obj = runtime.load_object(scheduler_file) # ID = 'scheduler' (filename stem) |
| 139 | obj.state_manager.reload() # Re-read from disk (server may have written new tasks) |
| 140 | state = obj.state_manager.get_all() |
| 141 | now = int(time.time()) |
| 142 | |
| 143 | for key, value in list(state.items()): |
| 144 | if not key.startswith('task_'): |
| 145 | continue |
| 146 | |
| 147 | try: |
| 148 | task = json.loads(value) |
| 149 | except (json.JSONDecodeError, TypeError): |
| 150 | continue |
| 151 | |
| 152 | if task.get('status') != 'active': |
| 153 | continue |
| 154 | |
| 155 | # Calculate next_run if not set |
| 156 | next_run = task.get('next_run') |
| 157 | if next_run is None: |
| 158 | next_run = _calculate_next_run(task) |
| 159 | if next_run is None: |
| 160 | continue |
| 161 | task['next_run'] = next_run |
| 162 | obj.state_manager.set(key, json.dumps(task)) |
| 163 | |
| 164 | if next_run > now: |
| 165 | continue |
| 166 | |
| 167 | # Task is due — execute target |
| 168 | target_id = task.get('object_id') |
| 169 | method = task.get('method', 'POST') |
| 170 | payload = task.get('payload', {}) |
| 171 | |
| 172 | if not target_id: |
| 173 | continue |
| 174 | |
| 175 | log(f"Scheduler: executing {target_id}.{method} (task {task['id']})") |
| 176 | |
| 177 | run_started = time.time() |
| 178 | run_result = None |
| 179 | run_error: Exception | None = None |
| 180 | try: |
| 181 | run_result = _execute_target(runtime, target_id, method, payload) |
| 182 | log(f"Scheduler: {target_id}.{method} completed") |
| 183 | except Exception as e: |
| 184 | run_error = e |
| 185 | log(f"Scheduler: {target_id}.{method} failed: {e}", 'ERROR') |
| 186 | _record_scheduler_run( |
| 187 | task, target_id, method, |
| 188 | started=run_started, |
| 189 | duration_ms=int((time.time() - run_started) * 1000), |
| 190 | result=run_result, |
| 191 | error=run_error, |
| 192 | ) |
| 193 | |
| 194 | # Update task |
| 195 | task['last_run'] = now |
| 196 | task['run_count'] = task.get('run_count', 0) + 1 |
| 197 | |
| 198 | if task.get('type') == 'onetime': |
| 199 | task['status'] = 'completed' |
| 200 | task['next_run'] = None |
| 201 | else: |
| 202 | task['next_run'] = _calculate_next_run(task, after=now) |
| 203 | |
| 204 | obj.state_manager.set(key, json.dumps(task)) |
| 205 | |
| 206 | |
| 207 | def _record_scheduler_run(task, target_id, method, *, started, duration_ms, |
| 208 | result, error): |
| 209 | """Append one scheduler_runs row -- run history an operator can query. |
| 210 | |
| 211 | A run's outcome must be a RECORD, not a stdout line only ssh can reach |
| 212 | (the observability hole that let q9's auto-approve and this server's own |
| 213 | scheduler passes sit dead unnoticed). Best-effort: skips silently when |
| 214 | the scheduler_runs collection is not installed (system-dashboard ships |
| 215 | it) and never breaks the pass. |
| 216 | """ |
| 217 | try: |
| 218 | base_dir = os.environ.get("DBBASIC_DATA_DIR", "data") |
| 219 | row = { |
| 220 | "id": object_ids.new_uuid4(), |
| 221 | "task_id": str(task.get("id") or ""), |
| 222 | "object_id": str(target_id), |
| 223 | "method": str(method), |
| 224 | "ok": "false" if error is not None else "true", |
| 225 | "started_at": datetime.fromtimestamp(started, tz=timezone.utc).isoformat(), |
| 226 | "duration_ms": str(int(duration_ms)), |
| 227 | "result": "" if error is not None or result is None else json.dumps(result)[:2000], |
| 228 | "error": str(error)[:2000] if error is not None else "", |
| 229 | "error_type": type(error).__name__ if error is not None else "", |
| 230 | } |
| 231 | object_records.create_collection_record( |
| 232 | "scheduler_runs", row, base_dir=base_dir, actor="daemon:scheduler" |
| 233 | ) |
| 234 | except Exception: |
| 235 | pass # observability must never break the scheduling pass |
| 236 | |
| 237 | |
| 238 | def _calculate_next_run(task, after=None): |
| 239 | """Calculate next run time for a task.""" |
| 240 | schedule = task.get('schedule', '') |
| 241 | task_type = task.get('type', '') |
| 242 | |
| 243 | if after is None: |
| 244 | after = int(time.time()) |
| 245 | |
| 246 | if task_type == 'cron' and croniter: |
| 247 | try: |
| 248 | cron = croniter(schedule, after) |
| 249 | return int(cron.get_next(float)) |
| 250 | except (ValueError, KeyError): |
| 251 | return None |
| 252 | |
| 253 | elif task_type == 'onetime': |
| 254 | try: |
| 255 | dt = datetime.fromisoformat(schedule.replace('Z', '+00:00')) |
| 256 | return int(dt.timestamp()) |
| 257 | except (ValueError, TypeError): |
| 258 | return None |
| 259 | |
| 260 | return None |
| 261 | |
| 262 | |
| 263 | # --- Queue --- |
| 264 | |
| 265 | def process_queue(runtime: ObjectRuntime, max_messages=10): |
| 266 | """Dequeue messages and execute target objects.""" |
| 267 | queue_file = _find_trigger_file('queue') |
| 268 | if queue_file is None: |
| 269 | return |
| 270 | |
| 271 | obj = runtime.load_object(queue_file) # ID = 'queue' (filename stem) |
| 272 | obj.state_manager.reload() # Re-read from disk |
| 273 | state = obj.state_manager.get_all() |
| 274 | now = int(time.time()) |
| 275 | |
| 276 | # Find pending, visible messages |
| 277 | messages = [] |
| 278 | for key, value in state.items(): |
| 279 | if not key.startswith('msg_'): |
| 280 | continue |
| 281 | |
| 282 | try: |
| 283 | msg = json.loads(value) |
| 284 | except (json.JSONDecodeError, TypeError): |
| 285 | continue |
| 286 | |
| 287 | if msg.get('status') != 'pending': |
| 288 | continue |
| 289 | |
| 290 | # Check expiration. `or`-style defaulting rather than .get(key, |
| 291 | # default): object_daemon_control.enqueue_message always WRITES |
| 292 | # expires_at, using None to mean "never expires", so the key exists |
| 293 | # and .get's default never applied -- every message enqueued through |
| 294 | # the official API raised TypeError here and took the whole pass |
| 295 | # down with it. The queue was fully built and unusable for exactly |
| 296 | # that reason. |
| 297 | expires_at = msg.get('expires_at') |
| 298 | if expires_at is not None and expires_at < now: |
| 299 | msg['status'] = 'expired' |
| 300 | obj.state_manager.set(key, json.dumps(msg)) |
| 301 | continue |
| 302 | |
| 303 | # Check visibility (same defaulting hazard: visible_after is always |
| 304 | # written, and a null there means "runnable now"). |
| 305 | visible_after = msg.get('visible_after') |
| 306 | if visible_after is not None and visible_after > now: |
| 307 | continue |
| 308 | |
| 309 | messages.append((key, msg)) |
| 310 | |
| 311 | # Sort by priority (highest first), then timestamp (oldest first) |
| 312 | messages.sort(key=lambda m: (-m[1].get('priority_level', 2), m[1].get('created_at', 0))) |
| 313 | |
| 314 | # Process up to max_messages |
| 315 | for key, msg in messages[:max_messages]: |
| 316 | body = msg.get('message', {}) |
| 317 | if not isinstance(body, dict): |
| 318 | continue |
| 319 | |
| 320 | target_id = body.get('object_id') |
| 321 | if not target_id: |
| 322 | continue |
| 323 | |
| 324 | method = body.get('method', 'POST') |
| 325 | payload = body.get('payload', {}) |
| 326 | |
| 327 | log(f"Queue: executing {target_id}.{method} (msg {msg['id']}, queue {msg.get('queue_name')})") |
| 328 | |
| 329 | # Mark as processing |
| 330 | msg['status'] = 'processing' |
| 331 | msg['dequeued_at'] = now |
| 332 | obj.state_manager.set(key, json.dumps(msg)) |
| 333 | |
| 334 | try: |
| 335 | _execute_target(runtime, target_id, method, payload) |
| 336 | msg['status'] = 'completed' |
| 337 | msg['completed_at'] = int(time.time()) |
| 338 | log(f"Queue: {target_id}.{method} completed") |
| 339 | except Exception as e: |
| 340 | log(f"Queue: {target_id}.{method} failed: {e}", 'ERROR') |
| 341 | msg['attempts'] = msg.get('attempts', 0) + 1 |
| 342 | if msg['attempts'] >= msg.get('max_attempts', 3): |
| 343 | msg['status'] = 'failed' |
| 344 | msg['failed_at'] = int(time.time()) |
| 345 | else: |
| 346 | msg['status'] = 'pending' |
| 347 | msg['visible_after'] = int(time.time()) + (2 ** msg['attempts']) |
| 348 | |
| 349 | obj.state_manager.set(key, json.dumps(msg)) |
| 350 | |
| 351 | |
| 352 | # --- Events --- |
| 353 | |
| 354 | def process_events(runtime: ObjectRuntime): |
| 355 | """Deliver events to subscribers via callback URLs.""" |
| 356 | events_file = _find_trigger_file('events') |
| 357 | if events_file is None: |
| 358 | return |
| 359 | |
| 360 | obj = runtime.load_object(events_file) # ID = 'events' (filename stem) |
| 361 | obj.state_manager.reload() # Re-read from disk |
| 362 | state = obj.state_manager.get_all() |
| 363 | |
| 364 | # Collect subscriptions and events |
| 365 | subscriptions = {} |
| 366 | events = {} |
| 367 | |
| 368 | for key, value in state.items(): |
| 369 | try: |
| 370 | data = json.loads(value) |
| 371 | except (json.JSONDecodeError, TypeError): |
| 372 | continue |
| 373 | |
| 374 | if key.startswith('sub_'): |
| 375 | callback = data.get('callback_url') |
| 376 | if callback: |
| 377 | subscriptions[key] = data |
| 378 | elif key.startswith('event_'): |
| 379 | events[key] = data |
| 380 | |
| 381 | if not subscriptions or not events: |
| 382 | return |
| 383 | |
| 384 | # Sort events by timestamp |
| 385 | sorted_events = sorted(events.values(), key=lambda e: e.get('timestamp', 0)) |
| 386 | |
| 387 | for sub_key, sub in subscriptions.items(): |
| 388 | event_type = sub.get('event_type') |
| 389 | last_event_id = sub.get('last_event_id') |
| 390 | callback_url = sub.get('callback_url') |
| 391 | |
| 392 | # Find events for this subscription |
| 393 | matching = [e for e in sorted_events if e.get('event_type') == event_type] |
| 394 | |
| 395 | # Skip events already delivered |
| 396 | if last_event_id: |
| 397 | found = False |
| 398 | pending = [] |
| 399 | for e in matching: |
| 400 | if found: |
| 401 | pending.append(e) |
| 402 | elif e.get('id') == last_event_id: |
| 403 | found = True |
| 404 | if not found: |
| 405 | # last_event_id not found, deliver all |
| 406 | pending = matching |
| 407 | else: |
| 408 | pending = matching |
| 409 | |
| 410 | if not pending: |
| 411 | continue |
| 412 | |
| 413 | # Deliver events |
| 414 | for event in pending: |
| 415 | status_code = None |
| 416 | try: |
| 417 | req = urllib.request.Request( |
| 418 | callback_url, |
| 419 | data=json.dumps(event).encode('utf-8'), |
| 420 | headers={'Content-Type': 'application/json'}, |
| 421 | method='POST', |
| 422 | ) |
| 423 | with urllib.request.urlopen(req, timeout=5) as resp: |
| 424 | if hasattr(resp, 'getcode'): |
| 425 | status_code = resp.getcode() |
| 426 | else: |
| 427 | status_code = getattr(resp, 'status', None) |
| 428 | log(f"Events: delivered {event['event_type']} ({event['id']}) to {sub['id']}") |
| 429 | sub = object_events.record_subscription_delivery( |
| 430 | sub, |
| 431 | event, |
| 432 | success=True, |
| 433 | status_code=status_code, |
| 434 | ) |
| 435 | obj.state_manager.set(sub_key, json.dumps(sub)) |
| 436 | except (urllib.error.URLError, OSError) as e: |
| 437 | status_code = getattr(e, 'code', None) |
| 438 | log(f"Events: failed to deliver to {callback_url}: {e}", 'WARN') |
| 439 | sub = object_events.record_subscription_delivery( |
| 440 | sub, |
| 441 | event, |
| 442 | success=False, |
| 443 | status_code=status_code, |
| 444 | error=str(e), |
| 445 | ) |
| 446 | obj.state_manager.set(sub_key, json.dumps(sub)) |
| 447 | break |
| 448 | |
| 449 | |
| 450 | # --- Rate Limit Cleanup --- |
| 451 | |
| 452 | def cleanup_ratelimit(max_age=120, *, base_dir: Path | str = "data"): |
| 453 | """Delete rate limit files older than max_age seconds.""" |
| 454 | ratelimit_dir = Path(base_dir) / 'ratelimit' |
| 455 | if not ratelimit_dir.exists(): |
| 456 | return |
| 457 | |
| 458 | now = time.time() |
| 459 | cutoff = now - max_age |
| 460 | cleaned = 0 |
| 461 | |
| 462 | for f in ratelimit_dir.iterdir(): |
| 463 | if not f.is_file() or not f.name.endswith('.txt'): |
| 464 | continue |
| 465 | try: |
| 466 | # Check if file has any recent timestamps |
| 467 | has_recent = False |
| 468 | for line in f.read_text().strip().split('\n'): |
| 469 | if line: |
| 470 | ts = float(line) |
| 471 | if ts > cutoff: |
| 472 | has_recent = True |
| 473 | break |
| 474 | if not has_recent: |
| 475 | f.unlink() |
| 476 | cleaned += 1 |
| 477 | except (ValueError, IOError, OSError): |
| 478 | # Corrupt or unreadable — safe to remove |
| 479 | try: |
| 480 | f.unlink() |
| 481 | cleaned += 1 |
| 482 | except OSError: |
| 483 | pass |
| 484 | |
| 485 | if cleaned: |
| 486 | log(f"Cleanup: removed {cleaned} expired rate limit file(s)") |
| 487 | |
| 488 | |
| 489 | def cleanup_events( |
| 490 | *, |
| 491 | base_dir: Path | str = "data", |
| 492 | keep_count: int | None = None, |
| 493 | keep_seconds: int | None = None, |
| 494 | ): |
| 495 | """Prune old event queue rows while preserving subscriptions.""" |
| 496 | if keep_count is None: |
| 497 | keep_count = _env_int(EVENT_KEEP_COUNT_ENV, object_events.DEFAULT_EVENT_KEEP_COUNT) |
| 498 | if keep_seconds is None: |
| 499 | keep_seconds = _env_int(EVENT_KEEP_SECONDS_ENV, object_events.DEFAULT_EVENT_KEEP_SECONDS) |
| 500 | |
| 501 | result = object_events.prune_events( |
| 502 | base_dir=base_dir, |
| 503 | keep_count=keep_count, |
| 504 | keep_seconds=keep_seconds, |
| 505 | ) |
| 506 | if result["deleted"]: |
| 507 | log(f"Cleanup: removed {result['deleted']} expired event row(s)") |
| 508 | return result |
| 509 | |
| 510 | |
| 511 | def _env_int(name: str, default: int) -> int: |
| 512 | value = os.environ.get(name, "").strip() |
| 513 | if not value: |
| 514 | return default |
| 515 | try: |
| 516 | return int(value) |
| 517 | except ValueError: |
| 518 | return default |
| 519 | |
| 520 | |
| 521 | def _env_float(name: str, default: float) -> float: |
| 522 | value = os.environ.get(name, "").strip() |
| 523 | if not value: |
| 524 | return default |
| 525 | try: |
| 526 | return float(value) |
| 527 | except ValueError: |
| 528 | return default |
| 529 | |
| 530 | |
| 531 | # --- Compaction --- |
| 532 | |
| 533 | def process_compactions(*, base_dir: Path | str = "data") -> dict | None: |
| 534 | """Compact over-threshold append-mode collections on a timer. |
| 535 | |
| 536 | Runs at most once per DBBASIC_COMPACTION_INTERVAL_SECONDS (default |
| 537 | 3600 -- an hour), gated by a marker FILE (`.compaction_last_run` under |
| 538 | `base_dir`) rather than an in-process variable like the event-cleanup |
| 539 | pass above uses: a file marker survives a daemon restart, and if more |
| 540 | than one daemon process ever points at the same data dir, whichever |
| 541 | one's write lands last simply wins that interval's marker -- there is |
| 542 | no harm in an occasional double-run of a compaction that is itself |
| 543 | idempotent-per-collection (compact_collection on an already-compacted |
| 544 | file is a correctly-reported no-op). |
| 545 | |
| 546 | Every collection whose schema currently declares "storage": "append" |
| 547 | (object_records.list_append_collection_stats) is compacted when BOTH: |
| 548 | - its physical row count is at or above DBBASIC_APPEND_COMPACT_ |
| 549 | MIN_ROWS (object_records.append_compact_min_rows -- the same |
| 550 | floor object_records._maybe_flag_auto_compact uses for its own |
| 551 | inline auto-compact trigger on ordinary writes), AND |
| 552 | - its bloat_ratio is at or above DBBASIC_COMPACTION_BLOAT_RATIO |
| 553 | (default 1.0 -- dead rows at least matching live rows). |
| 554 | A stats entry this call can't get real numbers for (list_append_ |
| 555 | collection_stats defaults to allow_fold=True, so this is not expected |
| 556 | in practice, but a None/estimated physical_rows or bloat_ratio is |
| 557 | skipped rather than guessed at) is left alone; it will be reconsidered |
| 558 | on the next interval. |
| 559 | |
| 560 | Each collection's compaction is wrapped in its own try/except: one |
| 561 | collection failing (a lock contention, a mid-flight schema change, a |
| 562 | permissions error) is logged and skipped, never allowed to stop the |
| 563 | rest of the pass or propagate out of this function -- the daemon is |
| 564 | an optional process (nothing else may depend on it running), and a |
| 565 | single bad collection must not take an entire poll interval's worth |
| 566 | of compaction down with it. The daemon's own main loop wraps this |
| 567 | call in a try/except too, belt and suspenders. |
| 568 | |
| 569 | Returns None when the interval hasn't elapsed yet (no work attempted |
| 570 | this call). Otherwise returns {"checked": <int>, "compacted": [{ |
| 571 | "collection", "rows_before", "rows_after", "bytes_before", |
| 572 | "bytes_after"}, ...]} -- mainly for tests and manual/CLI invocation; |
| 573 | the daemon's own loop only logs. |
| 574 | """ |
| 575 | interval = _env_int(COMPACTION_INTERVAL_SECONDS_ENV, _DEFAULT_COMPACTION_INTERVAL_SECONDS) |
| 576 | marker_path = Path(base_dir) / COMPACTION_MARKER_NAME |
| 577 | now = time.time() |
| 578 | try: |
| 579 | last_run = marker_path.stat().st_mtime |
| 580 | except OSError: |
| 581 | last_run = 0.0 |
| 582 | |
| 583 | if now - last_run < interval: |
| 584 | return None |
| 585 | |
| 586 | marker_path.parent.mkdir(parents=True, exist_ok=True) |
| 587 | marker_path.write_text(str(now)) |
| 588 | |
| 589 | bloat_threshold = _env_float(COMPACTION_BLOAT_RATIO_ENV, _DEFAULT_COMPACTION_BLOAT_RATIO) |
| 590 | min_rows = object_records.append_compact_min_rows() |
| 591 | |
| 592 | try: |
| 593 | stats = object_records.list_append_collection_stats(base_dir=base_dir) |
| 594 | except Exception as e: |
| 595 | log(f"Compaction: could not list append collections: {e}", "ERROR") |
| 596 | return {"checked": 0, "compacted": []} |
| 597 | |
| 598 | compacted = [] |
| 599 | for entry in stats: |
| 600 | collection = entry.get("collection") |
| 601 | try: |
| 602 | physical_rows = entry.get("physical_rows") |
| 603 | bloat_ratio = entry.get("bloat_ratio") |
| 604 | if physical_rows is None or bloat_ratio is None: |
| 605 | continue |
| 606 | if physical_rows < min_rows or bloat_ratio < bloat_threshold: |
| 607 | continue |
| 608 | |
| 609 | summary = object_records.compact_collection(collection, base_dir=base_dir) |
| 610 | log( |
| 611 | f"Compaction: {collection} rows {summary['rows_before']}->" |
| 612 | f"{summary['rows_after']}, bytes {summary['bytes_before']}->" |
| 613 | f"{summary['bytes_after']}" |
| 614 | ) |
| 615 | compacted.append({"collection": collection, **summary}) |
| 616 | except Exception as e: |
| 617 | log(f"Compaction: {collection} failed: {e}", "ERROR") |
| 618 | continue |
| 619 | |
| 620 | return {"checked": len(stats), "compacted": compacted} |
| 621 | |
| 622 | |
| 623 | # --- Stale-state auto-transition --- |
| 624 | |
| 625 | def auto_transition_rules_from_env() -> list[dict]: |
| 626 | """Parse DBBASIC_AUTO_TRANSITION_RULES into a list of clean rule dicts. |
| 627 | |
| 628 | Unset -> _DEFAULT_AUTO_TRANSITION_RULES (tasks' waiting_on_client -> |
| 629 | approved after 48h). Set to "" or "[]" -> [] (the pass runs every |
| 630 | interval, finds nothing configured, and logs a zero-work summary |
| 631 | line rather than silently vanishing -- see process_stale_transitions). |
| 632 | A malformed entry (missing string field, unparseable after_hours) is |
| 633 | dropped with a warning rather than aborting the whole list; a |
| 634 | malformed *list* (bad JSON, not a JSON array) disables every rule and |
| 635 | is logged loudly, since silently falling back to the default here |
| 636 | would hide a real configuration mistake. |
| 637 | """ |
| 638 | raw = os.environ.get(AUTO_TRANSITION_RULES_ENV) |
| 639 | if raw is None: |
| 640 | raw = _DEFAULT_AUTO_TRANSITION_RULES |
| 641 | text = raw.strip() |
| 642 | if not text: |
| 643 | return [] |
| 644 | |
| 645 | try: |
| 646 | parsed = json.loads(text) |
| 647 | except json.JSONDecodeError as e: |
| 648 | log(f"Auto-transition: {AUTO_TRANSITION_RULES_ENV} is not valid JSON, disabling: {e}", "ERROR") |
| 649 | return [] |
| 650 | if not isinstance(parsed, list): |
| 651 | log(f"Auto-transition: {AUTO_TRANSITION_RULES_ENV} must be a JSON list, disabling", "ERROR") |
| 652 | return [] |
| 653 | |
| 654 | rules = [] |
| 655 | for entry in parsed: |
| 656 | if not isinstance(entry, dict): |
| 657 | log(f"Auto-transition: skipping non-object rule: {entry!r}", "WARN") |
| 658 | continue |
| 659 | collection = entry.get("collection") |
| 660 | field = entry.get("field") |
| 661 | from_value = entry.get("from") |
| 662 | to_value = entry.get("to") |
| 663 | if not all(isinstance(v, str) and v for v in (collection, field, from_value, to_value)): |
| 664 | log(f"Auto-transition: skipping malformed rule (missing collection/field/from/to): {entry!r}", "WARN") |
| 665 | continue |
| 666 | try: |
| 667 | after_hours = float(entry.get("after_hours")) |
| 668 | except (TypeError, ValueError): |
| 669 | log(f"Auto-transition: skipping rule with invalid after_hours: {entry!r}", "WARN") |
| 670 | continue |
| 671 | rules.append({ |
| 672 | "collection": collection, |
| 673 | "field": field, |
| 674 | "from": from_value, |
| 675 | "to": to_value, |
| 676 | "after_hours": after_hours, |
| 677 | }) |
| 678 | return rules |
| 679 | |
| 680 | |
| 681 | def _parse_iso_timestamp(value: Any) -> datetime | None: |
| 682 | if not isinstance(value, str) or not value: |
| 683 | return None |
| 684 | try: |
| 685 | parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) |
| 686 | except ValueError: |
| 687 | return None |
| 688 | if parsed.tzinfo is None: |
| 689 | parsed = parsed.replace(tzinfo=timezone.utc) |
| 690 | return parsed |
| 691 | |
| 692 | |
| 693 | def _latest_field_entry_timestamps( |
| 694 | collection: str, field: str, from_value: str, *, base_dir: Path | str |
| 695 | ) -> dict[str, str]: |
| 696 | """Best-effort record id -> timestamp of the most recent change that |
| 697 | set `field` to `from_value`, built from ONE sequential read of the |
| 698 | collection's record-change log (object_record_changes.CHANGES_FILE). |
| 699 | |
| 700 | This is the most accurate signal available for "how long has this |
| 701 | record been sitting in this state": it answers exactly the question |
| 702 | the FSM cares about, unlike a record's own `created_at` (when the row |
| 703 | was first made, not when it entered THIS state) or a generic |
| 704 | `updated_at` (bumped by any field edit, not just this one). It only |
| 705 | stays cheap because it is read once per rule per pass -- not once per |
| 706 | candidate record -- and the log is append-ordered, so a later entry |
| 707 | for the same id simply overwrites an earlier one as this walks the |
| 708 | file forward, leaving the most recent one behind. |
| 709 | |
| 710 | Returns {} when the log doesn't exist yet or can't be read; callers |
| 711 | fall back to the record's own fields in that case (see |
| 712 | _record_age_seconds). |
| 713 | """ |
| 714 | try: |
| 715 | path = object_record_changes.record_changes_file(collection, base_dir=base_dir) |
| 716 | except object_collections.InvalidCollectionNameError: |
| 717 | return {} |
| 718 | if not path.exists(): |
| 719 | return {} |
| 720 | |
| 721 | try: |
| 722 | text = path.read_text(encoding="utf-8") |
| 723 | except OSError: |
| 724 | return {} |
| 725 | |
| 726 | timestamps: dict[str, str] = {} |
| 727 | for line in text.splitlines(): |
| 728 | if not line.strip(): |
| 729 | continue |
| 730 | try: |
| 731 | entry = json.loads(line) |
| 732 | except json.JSONDecodeError: |
| 733 | continue |
| 734 | if not isinstance(entry, dict): |
| 735 | continue |
| 736 | after = entry.get("after") |
| 737 | if not isinstance(after, dict) or after.get(field) != from_value: |
| 738 | continue |
| 739 | record_id = entry.get("record_id") |
| 740 | timestamp = entry.get("timestamp") |
| 741 | if isinstance(record_id, str) and isinstance(timestamp, str): |
| 742 | timestamps[record_id] = timestamp |
| 743 | return timestamps |
| 744 | |
| 745 | |
| 746 | def _record_age_seconds( |
| 747 | record: dict[str, str], |
| 748 | record_id: str, |
| 749 | change_timestamps: dict[str, str], |
| 750 | *, |
| 751 | now: datetime, |
| 752 | ) -> float | None: |
| 753 | """Return how long `record` has held its current value, in seconds, or |
| 754 | None when no usable timestamp exists at all (caller skips the record |
| 755 | rather than guessing). |
| 756 | |
| 757 | Preference order, documented honestly: the record-change log's entry |
| 758 | for the change that set this record's field to its current value |
| 759 | (most accurate -- see _latest_field_entry_timestamps), then an |
| 760 | explicit `updated_at` field on the record itself (cheap, no schema in |
| 761 | this codebase declares one yet, but a future one might), then |
| 762 | `created_at` (tasks.json's only timestamp field today -- a real but |
| 763 | honestly-worse proxy, since it marks row creation, not state entry). |
| 764 | """ |
| 765 | for source in ( |
| 766 | change_timestamps.get(record_id), |
| 767 | record.get("updated_at"), |
| 768 | record.get("created_at"), |
| 769 | ): |
| 770 | parsed = _parse_iso_timestamp(source) |
| 771 | if parsed is not None: |
| 772 | return (now - parsed).total_seconds() |
| 773 | return None |
| 774 | |
| 775 | |
| 776 | def _apply_auto_transition_rule(rule: dict, *, base_dir: Path | str) -> int: |
| 777 | """Apply one rule and return the count of records actually moved. |
| 778 | |
| 779 | Every failure mode here is a skip, never an abort: an unknown |
| 780 | collection (the owning package isn't installed) is logged once and |
| 781 | skipped; a record with no usable timestamp is skipped; a record whose |
| 782 | live value has already moved on since it was listed is skipped; and a |
| 783 | write rejected by the schema (e.g. a transitions map that no longer |
| 784 | allows this move) is logged and skipped, matching process_compactions' |
| 785 | per-item isolation posture -- one bad record must never stop the rest |
| 786 | of the rule, and one bad rule must never stop the rest of the pass. |
| 787 | """ |
| 788 | collection = rule["collection"] |
| 789 | field = rule["field"] |
| 790 | from_value = rule["from"] |
| 791 | to_value = rule["to"] |
| 792 | after_hours = rule["after_hours"] |
| 793 | |
| 794 | try: |
| 795 | records = object_records.read_collection_records(collection, base_dir=base_dir) |
| 796 | except (object_collections.CollectionNotFoundError, object_collections.InvalidCollectionNameError): |
| 797 | if collection not in _WARNED_UNKNOWN_AUTO_TRANSITION_COLLECTIONS: |
| 798 | log( |
| 799 | f"Auto-transition: collection '{collection}' not found (package not " |
| 800 | f"installed?), skipping rule -- will keep checking, logged once", |
| 801 | "WARN", |
| 802 | ) |
| 803 | _WARNED_UNKNOWN_AUTO_TRANSITION_COLLECTIONS.add(collection) |
| 804 | return 0 |
| 805 | |
| 806 | _WARNED_UNKNOWN_AUTO_TRANSITION_COLLECTIONS.discard(collection) |
| 807 | |
| 808 | candidates = [r for r in records if r.get(field) == from_value] |
| 809 | if not candidates: |
| 810 | return 0 |
| 811 | |
| 812 | change_timestamps = _latest_field_entry_timestamps(collection, field, from_value, base_dir=base_dir) |
| 813 | cutoff_seconds = after_hours * 3600.0 |
| 814 | now = datetime.now(timezone.utc) |
| 815 | |
| 816 | moved = 0 |
| 817 | for record in candidates: |
| 818 | record_id = record.get("id") |
| 819 | if not record_id: |
| 820 | continue |
| 821 | |
| 822 | age_seconds = _record_age_seconds(record, record_id, change_timestamps, now=now) |
| 823 | if age_seconds is None or age_seconds < cutoff_seconds: |
| 824 | continue |
| 825 | |
| 826 | try: |
| 827 | current = object_records.get_collection_record(collection, record_id, base_dir=base_dir) |
| 828 | except object_records.RecordNotFoundError: |
| 829 | continue # deleted since the listing above |
| 830 | if current.get(field) != from_value: |
| 831 | continue # moved on already -- someone/something got there first |
| 832 | |
| 833 | try: |
| 834 | object_records.update_collection_record( |
| 835 | collection, record_id, {field: to_value}, |
| 836 | base_dir=base_dir, actor=AUTO_TRANSITION_ACTOR, |
| 837 | ) |
| 838 | moved += 1 |
| 839 | except Exception as e: |
| 840 | log( |
| 841 | f"Auto-transition: {collection}/{record_id} {field} " |
| 842 | f"'{from_value}'->'{to_value}' failed: {e}", |
| 843 | "ERROR", |
| 844 | ) |
| 845 | continue |
| 846 | |
| 847 | return moved |
| 848 | |
| 849 | |
| 850 | def process_stale_transitions(*, base_dir: Path | str = "data") -> dict | None: |
| 851 | """Auto-transition records that have sat in one FSM state too long. |
| 852 | |
| 853 | Runs at most once per DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS |
| 854 | (default 3600) gated by a marker FILE (`.auto_transition_last_run` |
| 855 | under `base_dir`), the same pattern process_compactions above uses |
| 856 | and for the same reason: a file marker survives a daemon restart, so |
| 857 | the 48-hour clock this pass exists to enforce is never quietly reset |
| 858 | by a deploy. |
| 859 | |
| 860 | Rules come from DBBASIC_AUTO_TRANSITION_RULES (see |
| 861 | auto_transition_rules_from_env), a compact JSON list of |
| 862 | {"collection", "field", "from", "to", "after_hours"}. This pass ships |
| 863 | ON by default -- the empty-env case still resolves to a real rule |
| 864 | (tasks' waiting_on_client -> approved after 48h) -- because the |
| 865 | predecessor system's version of this job was written but never |
| 866 | scheduled, and 200+ records sat stranded as a result. Setting the env |
| 867 | to "" or "[]" is how an operator opts back out. |
| 868 | |
| 869 | Every rule is applied independently and wrapped in its own try/except |
| 870 | (_apply_auto_transition_rule already isolates per-record failures; |
| 871 | this is belt and suspenders against a rule failing before it gets |
| 872 | that far, e.g. a read_collection_records call raising something other |
| 873 | than the two "not installed" exceptions it explicitly handles) -- |
| 874 | one bad rule is logged and skipped, never allowed to stop the rest of |
| 875 | the pass. |
| 876 | |
| 877 | Always logs exactly one summary line per run (even a no-op run, with |
| 878 | zero rules configured or zero records moved) so the pass's activity |
| 879 | -- or deliberate silence -- is visible in the daemon's own log rather |
| 880 | than something that has to be inferred. Returns None when the |
| 881 | interval hasn't elapsed yet (no work attempted this call). Otherwise |
| 882 | returns {"checked": <rule count>, "moved": <total records moved>, |
| 883 | "rules": [{"collection", "field", "from", "to", "moved"}, ...]} -- |
| 884 | mainly for tests and manual/CLI invocation, matching |
| 885 | process_compactions' contract. |
| 886 | """ |
| 887 | interval = _env_int(AUTO_TRANSITION_INTERVAL_SECONDS_ENV, _DEFAULT_AUTO_TRANSITION_INTERVAL_SECONDS) |
| 888 | marker_path = Path(base_dir) / AUTO_TRANSITION_MARKER_NAME |
| 889 | now = time.time() |
| 890 | try: |
| 891 | last_run = marker_path.stat().st_mtime |
| 892 | except OSError: |
| 893 | last_run = 0.0 |
| 894 | |
| 895 | if now - last_run < interval: |
| 896 | return None |
| 897 | |
| 898 | marker_path.parent.mkdir(parents=True, exist_ok=True) |
| 899 | marker_path.write_text(str(now)) |
| 900 | |
| 901 | rules = auto_transition_rules_from_env() |
| 902 | rule_results = [] |
| 903 | total_moved = 0 |
| 904 | for rule in rules: |
| 905 | try: |
| 906 | moved = _apply_auto_transition_rule(rule, base_dir=base_dir) |
| 907 | except Exception as e: |
| 908 | log( |
| 909 | f"Auto-transition: rule {rule['collection']}.{rule['field']} " |
| 910 | f"'{rule['from']}'->'{rule['to']}' failed: {e}", |
| 911 | "ERROR", |
| 912 | ) |
| 913 | moved = 0 |
| 914 | total_moved += moved |
| 915 | rule_results.append({ |
| 916 | "collection": rule["collection"], |
| 917 | "field": rule["field"], |
| 918 | "from": rule["from"], |
| 919 | "to": rule["to"], |
| 920 | "moved": moved, |
| 921 | }) |
| 922 | |
| 923 | log(f"Auto-transition: checked {len(rules)} rule(s), moved {total_moved} record(s)") |
| 924 | return {"checked": len(rules), "moved": total_moved, "rules": rule_results} |
| 925 | |
| 926 | |
| 927 | # --- Rollups --- |
| 928 | |
| 929 | def process_rollups(*, base_dir: Path | str = "data") -> dict | None: |
| 930 | """Recompute every due, enabled rollup_definitions row. |
| 931 | |
| 932 | Rides this daemon's existing poll loop as one more pass |
| 933 | (plan/vocabulary/14-rollup-spec.md's Dependencies: "No new daemon... |
| 934 | process_rollups, alongside process_compactions"), but its due-gate is |
| 935 | NOT the marker-file pattern process_compactions/process_stale_ |
| 936 | transitions use above: 14 is explicit that a rollup_definitions row |
| 937 | already IS the natural place to hold that state (last_computed_at), |
| 938 | unlike compaction's collection-wide pass, which has no equivalent |
| 939 | per-collection record. So there's no marker file here and no fixed |
| 940 | poll interval either -- each ENABLED definition is checked against |
| 941 | its own last_computed_at/refresh_interval_seconds every call |
| 942 | (object_rollups.is_definition_due), which is cheap (a timestamp |
| 943 | comparison) for every definition that isn't due yet. |
| 944 | |
| 945 | Returns None when the block-wide `rollup_enabled` flag is off (14's |
| 946 | Degradation: "the daemon's rollup pass is skipped entirely; existing |
| 947 | target collections keep serving their last-computed data") or when |
| 948 | the rollup_definitions collection doesn't exist yet (package not |
| 949 | installed). Otherwise returns {"checked": <definitions considered>, |
| 950 | "computed": [{"definition_id", "target_collection", "groups", |
| 951 | "suppressed", ...}, ...]} -- computed only lists definitions that |
| 952 | were actually due and enabled; a skipped/disabled/not-yet-due |
| 953 | definition doesn't appear. |
| 954 | |
| 955 | Every definition's recompute is wrapped in its own try/except, same |
| 956 | isolation pattern as process_compactions: one definition with a bad |
| 957 | filter, a missing source collection, or a lock contention is logged |
| 958 | and skipped, every other definition's pass proceeds. |
| 959 | """ |
| 960 | if not object_rollups.rollup_pass_enabled(base_dir=base_dir): |
| 961 | return None |
| 962 | |
| 963 | try: |
| 964 | definitions = object_records.read_collection_records( |
| 965 | object_rollups.ROLLUP_DEFINITIONS_COLLECTION, base_dir=base_dir |
| 966 | ) |
| 967 | except (object_collections.CollectionNotFoundError, object_collections.InvalidCollectionNameError): |
| 968 | return None |
| 969 | |
| 970 | now = datetime.now(timezone.utc) |
| 971 | computed = [] |
| 972 | for definition in definitions: |
| 973 | definition_id = definition.get("id") or "<unknown>" |
| 974 | |
| 975 | if not object_rollups.is_definition_enabled(definition): |
| 976 | continue |
| 977 | if not object_rollups.is_definition_due(definition, now=now): |
| 978 | continue |
| 979 | |
| 980 | try: |
| 981 | result = object_rollups.compute_rollup(definition, base_dir=base_dir) |
| 982 | except Exception as e: |
| 983 | log(f"Rollup: {definition_id} failed: {e}", "ERROR") |
| 984 | continue |
| 985 | |
| 986 | try: |
| 987 | object_records.update_collection_record( |
| 988 | object_rollups.ROLLUP_DEFINITIONS_COLLECTION, |
| 989 | definition_id, |
| 990 | {"last_computed_at": result["computed_at"]}, |
| 991 | base_dir=base_dir, |
| 992 | actor=object_rollups.ROLLUP_ACTOR, |
| 993 | preserve_read_only=True, |
| 994 | ) |
| 995 | except Exception as e: |
| 996 | # The recompute itself already succeeded and is live in the |
| 997 | # target collection; failing to stamp last_computed_at only |
| 998 | # means this definition looks "due" again next call (a |
| 999 | # harmless, self-correcting re-run), never lost or corrupted |
| 1000 | # data -- logged, not fatal to the rest of the pass. |
| 1001 | log(f"Rollup: {definition_id} computed but failed to stamp last_computed_at: {e}", "ERROR") |
| 1002 | |
| 1003 | log( |
| 1004 | f"Rollup: {definition_id} -> {result['target_collection']} " |
| 1005 | f"({result['groups']} group(s), {result['suppressed']} suppressed)" |
| 1006 | ) |
| 1007 | computed.append(result) |
| 1008 | |
| 1009 | return {"checked": len(definitions), "computed": computed} |
| 1010 | |
| 1011 | |
| 1012 | # --- Materialize --- |
| 1013 | |
| 1014 | def process_materializations(*, base_dir: Path | str = "data") -> dict | None: |
| 1015 | """Generate every due, enabled, non-blocked scheduled/scheduled_fixed |
| 1016 | materialize_definitions row's output. |
| 1017 | |
| 1018 | Rides this daemon's existing poll loop as one more pass |
| 1019 | (plan/vocabulary/61-materialize-spec.md's Dependencies: "No new |
| 1020 | daemon... process_materializations, alongside process_compactions"). |
| 1021 | Unlike process_rollups (whose due-gate lives entirely on the |
| 1022 | definition row, no marker file needed), 61's Events section asks for |
| 1023 | process_stale_transitions' marker-file-gated shape instead -- gated by |
| 1024 | DBBASIC_MATERIALIZE_INTERVAL_SECONDS (default 3600), marker file |
| 1025 | `.materialize_last_run` under `base_dir`, surviving a daemon restart |
| 1026 | the same way every other marker-gated pass does. |
| 1027 | |
| 1028 | Event-mode definitions (trigger.mode == "event") are never driven by |
| 1029 | this pass -- in v1 they run via materialize_run's manual path only |
| 1030 | (degrade-to-manual, per 61's Degradation section). The automatic |
| 1031 | on-create dispatch (materialize_seed's HANDLES) is deliberately NOT |
| 1032 | auto-wired: this pass does not rewrite materialize_seed's source to |
| 1033 | track the definition set. Rewriting an installed object at runtime -- |
| 1034 | and doing it every poll -- is the wrong shape (expensive, surprising, |
| 1035 | self-modifying code); if auto-synced HANDLES is ever wanted it belongs |
| 1036 | in a deliberate scheduled job with its own tests, not the poll loop. |
| 1037 | materialize_seed ships inert (HANDLES == []); the event-dispatch logic |
| 1038 | it would call still exists and is tested, ready for that future |
| 1039 | mechanism or a manual wiring. |
| 1040 | |
| 1041 | For each enabled, non-blocked, due scheduled/scheduled_fixed |
| 1042 | definition: object_materialize.generate_definition reads |
| 1043 | source_collection outside row-filters (daemon posture, same as |
| 1044 | compaction/rollup/auto-transition), computes the due (source row, |
| 1045 | period) set from first principles every call (never trusting a |
| 1046 | row's own next_run/last_run as the correctness mechanism -- see that |
| 1047 | module's docstring), and for each checks the deterministic header id |
| 1048 | in output_collection: missing -> generate, present -> skip, not an |
| 1049 | error. |
| 1050 | |
| 1051 | Two-level isolation, per 61's Events section (drawn from process_ |
| 1052 | stale_transitions' _apply_auto_transition_rule, not process_rollups' |
| 1053 | per-definition-only isolation): one bad DEFINITION (a malformed |
| 1054 | mapping, a missing source/output/child collection) is caught here and |
| 1055 | never stops any other definition; one bad SOURCE ROW within an |
| 1056 | otherwise-good definition is caught inside object_materialize. |
| 1057 | generate_config and surfaces as one entry in that definition's own |
| 1058 | "errors" list, never stopping any other row in the same definition. |
| 1059 | |
| 1060 | A missing source/output/child collection is logged once per process |
| 1061 | (object_materialize.MissingCollectionError, warned-once via |
| 1062 | _WARNED_UNKNOWN_MATERIALIZE_COLLECTIONS) -- mirrors process_stale_ |
| 1063 | transitions' _WARNED_UNKNOWN_AUTO_TRANSITION_COLLECTIONS exactly, to |
| 1064 | avoid re-logging every poll interval. |
| 1065 | |
| 1066 | Always logs exactly one summary line per run, even a no-op run |
| 1067 | (`"Materialize: checked N definition(s), generated M record(s), |
| 1068 | skipped K already-generated"`) -- every sibling pass's "activity or |
| 1069 | deliberate silence is visible in the daemon log" convention. |
| 1070 | |
| 1071 | Returns None when the block-wide `materialize_enabled` flag is off, |
| 1072 | the materialize_definitions collection doesn't exist yet (package not |
| 1073 | installed), or the interval hasn't elapsed. Otherwise returns |
| 1074 | {"checked": <definitions considered>, "generated": <total records>, |
| 1075 | "skipped_already_generated": <total>, "results": [...]}. |
| 1076 | """ |
| 1077 | if not object_materialize.materialize_pass_enabled(base_dir=base_dir): |
| 1078 | return None |
| 1079 | |
| 1080 | interval = _env_int(MATERIALIZE_INTERVAL_SECONDS_ENV, _DEFAULT_MATERIALIZE_INTERVAL_SECONDS) |
| 1081 | marker_path = Path(base_dir) / MATERIALIZE_MARKER_NAME |
| 1082 | now_ts = time.time() |
| 1083 | try: |
| 1084 | last_run = marker_path.stat().st_mtime |
| 1085 | except OSError: |
| 1086 | last_run = 0.0 |
| 1087 | |
| 1088 | if now_ts - last_run < interval: |
| 1089 | return None |
| 1090 | |
| 1091 | try: |
| 1092 | definitions = object_records.read_collection_records( |
| 1093 | object_materialize.MATERIALIZE_DEFINITIONS_COLLECTION, base_dir=base_dir |
| 1094 | ) |
| 1095 | except (object_collections.CollectionNotFoundError, object_collections.InvalidCollectionNameError): |
| 1096 | return None |
| 1097 | |
| 1098 | marker_path.parent.mkdir(parents=True, exist_ok=True) |
| 1099 | marker_path.write_text(str(now_ts)) |
| 1100 | |
| 1101 | now = datetime.now(timezone.utc) |
| 1102 | total_generated = 0 |
| 1103 | total_skipped = 0 |
| 1104 | results = [] |
| 1105 | |
| 1106 | for definition in definitions: |
| 1107 | definition_id = definition.get("id") or "<unknown>" |
| 1108 | |
| 1109 | if object_materialize.is_definition_blocked(definition): |
| 1110 | continue |
| 1111 | if not object_materialize.is_definition_enabled(definition): |
| 1112 | continue |
| 1113 | if not object_materialize.is_definition_due(definition, now=now): |
| 1114 | continue |
| 1115 | |
| 1116 | try: |
| 1117 | config = object_materialize.parse_definition(definition, base_dir=base_dir) |
| 1118 | except object_materialize.MissingCollectionError as e: |
| 1119 | if definition_id not in _WARNED_UNKNOWN_MATERIALIZE_COLLECTIONS: |
| 1120 | log( |
| 1121 | f"Materialize: {definition_id} references missing {e.role} collection " |
| 1122 | f"'{e.collection}' (package not installed?), skipping -- will keep " |
| 1123 | "checking, logged once", |
| 1124 | "WARN", |
| 1125 | ) |
| 1126 | _WARNED_UNKNOWN_MATERIALIZE_COLLECTIONS.add(definition_id) |
| 1127 | continue |
| 1128 | except Exception as e: |
| 1129 | log(f"Materialize: {definition_id} failed: {e}", "ERROR") |
| 1130 | continue |
| 1131 | |
| 1132 | if config.trigger_mode == "event": |
| 1133 | continue # event-mode definitions are driven by dispatch/manual only |
| 1134 | |
| 1135 | _WARNED_UNKNOWN_MATERIALIZE_COLLECTIONS.discard(definition_id) |
| 1136 | |
| 1137 | try: |
| 1138 | result = object_materialize.generate_config(config, base_dir=base_dir, now=now) |
| 1139 | except Exception as e: |
| 1140 | log(f"Materialize: {definition_id} failed: {e}", "ERROR") |
| 1141 | continue |
| 1142 | |
| 1143 | for error in result["errors"]: |
| 1144 | log( |
| 1145 | f"Materialize: {definition_id} source={error['source_id']} " |
| 1146 | f"period={error['period_start']} failed: {error['error']}", |
| 1147 | "ERROR", |
| 1148 | ) |
| 1149 | |
| 1150 | try: |
| 1151 | object_records.update_collection_record( |
| 1152 | object_materialize.MATERIALIZE_DEFINITIONS_COLLECTION, |
| 1153 | definition_id, |
| 1154 | {"last_run_at": _now_iso_utc()}, |
| 1155 | base_dir=base_dir, |
| 1156 | actor=config.actor, |
| 1157 | preserve_read_only=True, |
| 1158 | ) |
| 1159 | except Exception as e: |
| 1160 | # The generation itself already succeeded (whatever was due is |
| 1161 | # live in output_collection); failing to stamp last_run_at only |
| 1162 | # means this definition looks "due" again next call -- a |
| 1163 | # harmless, self-correcting re-run (the deterministic header id |
| 1164 | # is still the real gate), never lost or corrupted data. |
| 1165 | log(f"Materialize: {definition_id} ran but failed to stamp last_run_at: {e}", "ERROR") |
| 1166 | |
| 1167 | total_generated += result["generated"] |
| 1168 | total_skipped += result["skipped_already_generated"] |
| 1169 | results.append(result) |
| 1170 | |
| 1171 | log( |
| 1172 | f"Materialize: checked {len(definitions)} definition(s), generated " |
| 1173 | f"{total_generated} record(s), skipped {total_skipped} already-generated" |
| 1174 | ) |
| 1175 | return { |
| 1176 | "checked": len(definitions), |
| 1177 | "generated": total_generated, |
| 1178 | "skipped_already_generated": total_skipped, |
| 1179 | "results": results, |
| 1180 | } |
| 1181 | |
| 1182 | |
| 1183 | def _now_iso_utc() -> str: |
| 1184 | return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") |
| 1185 | |
| 1186 | |
| 1187 | # --- Object Execution --- |
| 1188 | |
| 1189 | def _execute_target(runtime: ObjectRuntime, object_id: str, method: str, payload: dict): |
| 1190 | """Load and execute a target object.""" |
| 1191 | request = ObjectExecutionRequest(object_id=object_id, method=method, payload=payload) |
| 1192 | result = execute_object(runtime, request) |
| 1193 | if not result.ok: |
| 1194 | if result.error and result.error.type == "ObjectNotFoundError": |
| 1195 | raise FileNotFoundError(result.error.message) |
| 1196 | raise ObjectExecutionFailure(result) |
| 1197 | return result.result |
| 1198 | |
| 1199 | |
| 1200 | def _find_object_file(object_id: str) -> Path | None: |
| 1201 | """Find the .py file for an object ID.""" |
| 1202 | return resolve_object_id(object_id) |
| 1203 | |
| 1204 | |
| 1205 | def process_notifications(*, base_dir: Path | str = "data") -> dict | None: |
| 1206 | """12 notify: turn new record-change events into notifications. |
| 1207 | |
| 1208 | Polls the record-change log rather than riding synchronous HANDLES |
| 1209 | dispatch -- HANDLES is gated behind DBBASIC_ENABLE_EVENT_HANDLERS (off in |
| 1210 | prod) and we deliberately don't rewrite installed objects to track dynamic |
| 1211 | event sets (see object_notify's docstring). Reads each watched collection's |
| 1212 | change entries newer than the cursor, matches them against enabled |
| 1213 | notify_rules (object_notify), and appends a notifications row per resolved |
| 1214 | recipient. At-least-once: a crash between writing a notification and |
| 1215 | advancing the cursor re-notifies, the delivery bar 12/01 accept. |
| 1216 | |
| 1217 | Returns None when notify is disabled or there are no rules; otherwise |
| 1218 | {"changes": <processed>, "notifications": <written>}. |
| 1219 | """ |
| 1220 | if not object_notify.notify_pass_enabled(base_dir=base_dir): |
| 1221 | return None |
| 1222 | try: |
| 1223 | rules = object_records.read_collection_records( |
| 1224 | object_notify.NOTIFY_RULES_COLLECTION, base_dir=base_dir |
| 1225 | ) |
| 1226 | except (object_collections.CollectionNotFoundError, object_collections.InvalidCollectionNameError): |
| 1227 | return None |
| 1228 | enabled = [r for r in rules if str(r.get("enabled", "true")).strip().lower() not in {"off", "false", "0", "no"}] |
| 1229 | if not enabled: |
| 1230 | return None |
| 1231 | |
| 1232 | cursor_path = Path(base_dir) / NOTIFY_CURSOR_NAME |
| 1233 | now_iso = datetime.now(timezone.utc).isoformat() |
| 1234 | try: |
| 1235 | cursor = cursor_path.read_text().strip() |
| 1236 | except OSError: |
| 1237 | cursor = "" |
| 1238 | if not cursor: |
| 1239 | # First run: stamp at now, notify only on FUTURE changes (never |
| 1240 | # backfill the entire history). |
| 1241 | cursor_path.parent.mkdir(parents=True, exist_ok=True) |
| 1242 | cursor_path.write_text(now_iso) |
| 1243 | return None |
| 1244 | |
| 1245 | try: |
| 1246 | known = {c.get("name") for c in object_collections.list_collections(base_dir=base_dir) if c.get("name")} |
| 1247 | except (OSError, ValueError): |
| 1248 | known = set() |
| 1249 | watched = object_notify.watched_collections(enabled, known) |
| 1250 | |
| 1251 | # Gather every change newer than the cursor, across watched collections. |
| 1252 | fresh: list[dict] = [] |
| 1253 | for collection in watched: |
| 1254 | try: |
| 1255 | payload = object_record_changes.list_record_changes(collection, base_dir=base_dir, limit=1000) |
| 1256 | except (object_collections.CollectionNotFoundError, object_collections.InvalidCollectionNameError, OSError, ValueError): |
| 1257 | continue |
| 1258 | for change in payload.get("changes") or []: |
| 1259 | ts = str(change.get("timestamp") or "") |
| 1260 | if ts > cursor: |
| 1261 | fresh.append(change) |
| 1262 | fresh.sort(key=lambda c: str(c.get("timestamp") or "")) |
| 1263 | |
| 1264 | written = 0 |
| 1265 | emailed = 0 |
| 1266 | max_ts = cursor |
| 1267 | for change in fresh: |
| 1268 | # Per-change try/except: one malformed rule or change never stops the |
| 1269 | # rest, and the cursor still advances past it (never re-storming). |
| 1270 | try: |
| 1271 | for rule in enabled: |
| 1272 | for note in object_notify.notifications_for_change(rule, change, base_dir=base_dir): |
| 1273 | object_records.create_collection_record( |
| 1274 | object_notify.NOTIFICATIONS_COLLECTION, note, |
| 1275 | base_dir=base_dir, actor=object_notify.DEFAULT_ACTOR, |
| 1276 | ) |
| 1277 | written += 1 |
| 1278 | # The email channel rides the GENERIC 01 outbox (object_email), |
| 1279 | # never the private mail package -- notify depends only on the |
| 1280 | # open adapter; whatever SMTP relay is configured delivers it. |
| 1281 | for intent in object_notify.email_intents_for_change(rule, change, base_dir=base_dir): |
| 1282 | if _enqueue_notify_email(intent, base_dir=base_dir): |
| 1283 | emailed += 1 |
| 1284 | except Exception as exc: # noqa: BLE001 -- isolate one bad change |
| 1285 | log(f"Notify: change {change.get('change_id')} failed: {exc}", "ERROR") |
| 1286 | ts = str(change.get("timestamp") or "") |
| 1287 | if ts > max_ts: |
| 1288 | max_ts = ts |
| 1289 | |
| 1290 | if max_ts != cursor: |
| 1291 | cursor_path.write_text(max_ts) |
| 1292 | if fresh or written or emailed: |
| 1293 | log(f"Notify: processed {len(fresh)} change(s), wrote {written} notification(s), queued {emailed} email(s)") |
| 1294 | return {"changes": len(fresh), "notifications": written, "emails": emailed} |
| 1295 | |
| 1296 | |
| 1297 | def _enqueue_notify_email(intent: dict, *, base_dir: Path | str) -> bool: |
| 1298 | """Map a notify email intent's user_id to an address and enqueue it into the |
| 1299 | generic 01 outbox. Returns False (skips) when the user has no address on |
| 1300 | file or the email_outbox isn't installed -- email is best-effort, never |
| 1301 | blocks the in_app write.""" |
| 1302 | try: |
| 1303 | user = object_identity.get_user(intent["user_id"], base_dir=base_dir) |
| 1304 | except (object_identity.UserNotFoundError, OSError, ValueError): |
| 1305 | return False |
| 1306 | address = (user.get("email") or "").strip() |
| 1307 | if not address: |
| 1308 | return False |
| 1309 | try: |
| 1310 | object_email.enqueue( |
| 1311 | address, intent.get("subject") or "", intent.get("body") or "", |
| 1312 | source_object_id="notify", |
| 1313 | extra={"user_id": intent["user_id"], "target": intent.get("target", "")}, |
| 1314 | base_dir=base_dir, |
| 1315 | ) |
| 1316 | except (object_collections.CollectionNotFoundError, |
| 1317 | object_collections.InvalidCollectionNameError): |
| 1318 | return False # app-email not installed -> email delivery unavailable |
| 1319 | return True |
| 1320 | |
| 1321 | |
| 1322 | # 01 email adapter: rolling per-minute send-rate window, persisted as JSON |
| 1323 | # ({window_start, sent_count}) so it survives a daemon restart -- the same |
| 1324 | # "cheap state outside the collection" shape process_compactions' marker uses. |
| 1325 | EMAIL_RATE_MARKER_NAME = ".email_rate_window" |
| 1326 | # Whether we've already logged "SMTP not configured" this process. The pass |
| 1327 | # runs every tick (it's one cheap collection read) but must not repeat the |
| 1328 | # line every poll -- matching process_scheduler/process_queue when their own |
| 1329 | # trigger object is absent. |
| 1330 | _EMAIL_UNCONFIGURED_WARNED = False |
| 1331 | |
| 1332 | |
| 1333 | def _read_email_rate_window(base_dir: Path | str) -> dict | None: |
| 1334 | try: |
| 1335 | return json.loads((Path(base_dir) / EMAIL_RATE_MARKER_NAME).read_text()) |
| 1336 | except (OSError, ValueError): |
| 1337 | return None |
| 1338 | |
| 1339 | |
| 1340 | def _write_email_rate_window(base_dir: Path | str, window: dict) -> None: |
| 1341 | path = Path(base_dir) / EMAIL_RATE_MARKER_NAME |
| 1342 | path.parent.mkdir(parents=True, exist_ok=True) |
| 1343 | path.write_text(json.dumps(window)) |
| 1344 | |
| 1345 | |
| 1346 | def process_email_outbox(*, base_dir: Path | str = "data") -> dict | None: |
| 1347 | """01 email adapter: drain the email_outbox queue. |
| 1348 | |
| 1349 | Every tick, delivers the `queued` rows that are due (`next_attempt_at` |
| 1350 | arrived) up to DBBASIC_SMTP_BATCH_SIZE, bounded by a rolling per-minute |
| 1351 | rate window. Each send's outcome is written back onto its row: `sent`, or |
| 1352 | `queued` again with a backed-off `next_attempt_at` and `last_error` (a |
| 1353 | transient failure), or `dead` (a permanent SMTP error, or retries |
| 1354 | exhausted). Per-message try/except -- one bad send never blocks the batch, |
| 1355 | matching process_compactions' per-item isolation. |
| 1356 | |
| 1357 | Returns None when email is disabled (feature flag), unconfigured (no SMTP |
| 1358 | mode / host -- everything just keeps queuing, fully inspectable), or the |
| 1359 | email_outbox collection isn't installed. Otherwise a small summary dict. |
| 1360 | """ |
| 1361 | global _EMAIL_UNCONFIGURED_WARNED |
| 1362 | if not object_email.email_pass_enabled(base_dir=base_dir): |
| 1363 | return None |
| 1364 | |
| 1365 | config = object_email.smtp_config_from_env() |
| 1366 | if not config.configured: |
| 1367 | if not _EMAIL_UNCONFIGURED_WARNED: |
| 1368 | log("Email: SMTP not configured (DBBASIC_SMTP_MODE) -- outbound mail is queuing only") |
| 1369 | _EMAIL_UNCONFIGURED_WARNED = True |
| 1370 | return None |
| 1371 | _EMAIL_UNCONFIGURED_WARNED = False # configured again -> re-warn if it flips back |
| 1372 | |
| 1373 | try: |
| 1374 | records = object_records.read_collection_records( |
| 1375 | object_email.OUTBOX_COLLECTION, base_dir=base_dir |
| 1376 | ) |
| 1377 | except (object_collections.CollectionNotFoundError, |
| 1378 | object_collections.InvalidCollectionNameError, OSError, ValueError): |
| 1379 | return None |
| 1380 | |
| 1381 | now = datetime.now(timezone.utc) |
| 1382 | due = [r for r in records if object_email.is_due(r, now)] |
| 1383 | if not due: |
| 1384 | return {"attempted": 0, "sent": 0, "dead": 0} |
| 1385 | # Oldest-scheduled first, so a backlog drains fairly rather than starving |
| 1386 | # the earliest-queued behind newer arrivals. |
| 1387 | due.sort(key=lambda r: (str(r.get("next_attempt_at") or ""), str(r.get("created_at") or ""))) |
| 1388 | |
| 1389 | window = object_email.rate_window_reset(_read_email_rate_window(base_dir), time.time()) |
| 1390 | sender = object_email.sender_for(config) |
| 1391 | attempted = sent = dead = rate_limited = 0 |
| 1392 | |
| 1393 | for record in due[:config.batch_size]: |
| 1394 | if window["sent_count"] >= config.rate_limit: |
| 1395 | rate_limited += 1 |
| 1396 | continue # left `queued`; a scheduling skip is not a delivery attempt |
| 1397 | try: |
| 1398 | update = object_email.attempt_delivery(record, config, sender=sender, now=now) |
| 1399 | except Exception as exc: # noqa: BLE001 -- isolate one bad message |
| 1400 | log(f"Email: message {record.get('id')} delivery raised: {exc}", "ERROR") |
| 1401 | continue |
| 1402 | window["sent_count"] += 1 # a network attempt was made (success or not) |
| 1403 | attempted += 1 |
| 1404 | try: |
| 1405 | object_records.update_collection_record( |
| 1406 | object_email.OUTBOX_COLLECTION, record["id"], update, |
| 1407 | base_dir=base_dir, actor=object_email.DEFAULT_ACTOR, |
| 1408 | preserve_read_only=True, |
| 1409 | ) |
| 1410 | except Exception as exc: # noqa: BLE001 |
| 1411 | # The send already happened; failing to record the outcome only |
| 1412 | # risks an at-least-once re-send next tick -- the delivery bar 01 |
| 1413 | # accepts, never lost data. |
| 1414 | log(f"Email: message {record.get('id')} sent but status write failed: {exc}", "ERROR") |
| 1415 | if update.get("status") == object_email.STATUS_SENT: |
| 1416 | sent += 1 |
| 1417 | elif update.get("status") == object_email.STATUS_DEAD: |
| 1418 | dead += 1 |
| 1419 | |
| 1420 | _write_email_rate_window(base_dir, window) |
| 1421 | if attempted or rate_limited: |
| 1422 | extra = f", {rate_limited} rate-limited" if rate_limited else "" |
| 1423 | log(f"Email: attempted {attempted} ({sent} sent, {dead} dead){extra}") |
| 1424 | return {"attempted": attempted, "sent": sent, "dead": dead, "rate_limited": rate_limited} |
| 1425 | |
| 1426 | |
| 1427 | CHANGE_LOG_RETENTION_MARKER_NAME = ".change_log_retention_last_run" |
| 1428 | CHANGE_LOG_RETENTION_INTERVAL_ENV = "DBBASIC_CHANGE_LOG_RETENTION_INTERVAL_SECONDS" |
| 1429 | _DEFAULT_CHANGE_LOG_RETENTION_INTERVAL = 21600 # 6h -- rewriting every log is not cheap |
| 1430 | |
| 1431 | |
| 1432 | def process_change_log_retention(*, base_dir: Path | str = "data") -> dict | None: |
| 1433 | """Rotate, archive and (only if asked) remove record change logs. |
| 1434 | |
| 1435 | Returns None when no policy is configured or the pass is not due. |
| 1436 | Off unless an operator sets a retention window or an entry cap: a |
| 1437 | change log IS the audit trail, and deciding how much of it to keep is |
| 1438 | not a decision a daemon makes on somebody's behalf. |
| 1439 | |
| 1440 | When it IS configured, expired entries are ARCHIVED by default rather |
| 1441 | than deleted -- rotated into a timestamped gzip segment beside the log |
| 1442 | (object_record_changes.prune_all_record_changes). Somebody asking for |
| 1443 | a smaller hot file has not asked to lose the history, and the hot file |
| 1444 | is the only part that costs anything: it is what every feed render |
| 1445 | reads. On the box that motivated this, 1.1GB of logs became 22MB hot |
| 1446 | plus a 115MB archive with all 1.7 million entries still in it. |
| 1447 | |
| 1448 | Runs on a 6h marker, the same shape as the analytics retention pass, |
| 1449 | because rewriting every collection's log is not something to do on a |
| 1450 | tick -- the first live sweep took three minutes. |
| 1451 | """ |
| 1452 | policy = object_record_changes.retention_policy() |
| 1453 | if not policy["days"] and not policy["max_entries"]: |
| 1454 | return None |
| 1455 | |
| 1456 | marker_path = Path(base_dir) / CHANGE_LOG_RETENTION_MARKER_NAME |
| 1457 | interval = _env_int(CHANGE_LOG_RETENTION_INTERVAL_ENV, |
| 1458 | _DEFAULT_CHANGE_LOG_RETENTION_INTERVAL) |
| 1459 | now = time.time() |
| 1460 | try: |
| 1461 | if now - marker_path.stat().st_mtime < interval: |
| 1462 | return None |
| 1463 | except OSError: |
| 1464 | pass # no marker yet -> run now |
| 1465 | marker_path.parent.mkdir(parents=True, exist_ok=True) |
| 1466 | marker_path.write_text(str(now)) |
| 1467 | |
| 1468 | try: |
| 1469 | result = object_record_changes.prune_all_record_changes( |
| 1470 | base_dir=base_dir, |
| 1471 | days=policy["days"], |
| 1472 | max_entries=policy["max_entries"], |
| 1473 | archive=policy["archive"], |
| 1474 | keep_archives=policy["keep_archives"], |
| 1475 | ) |
| 1476 | except OSError: |
| 1477 | return None |
| 1478 | if result.get("removed"): |
| 1479 | verb = "archived" if policy["archive"] else "removed" |
| 1480 | log(f"Change logs: {verb} {result['removed']} entr(ies) across " |
| 1481 | f"{result['collections']} log(s)") |
| 1482 | return result |
| 1483 | |
| 1484 | |
| 1485 | RESTORE_POINT_RETENTION_MARKER_NAME = ".restore_point_retention_last_run" |
| 1486 | RESTORE_POINT_RETENTION_INTERVAL_ENV = "DBBASIC_RESTORE_POINT_RETENTION_INTERVAL_SECONDS" |
| 1487 | RESTORE_POINT_KEEP_LAST_ENV = "DBBASIC_RESTORE_POINT_KEEP_LAST" |
| 1488 | RESTORE_POINT_KEEP_DAYS_ENV = "DBBASIC_RESTORE_POINT_KEEP_DAYS" |
| 1489 | RESTORE_POINT_MAX_BYTES_ENV = "DBBASIC_RESTORE_POINT_MAX_BYTES" |
| 1490 | _DEFAULT_RESTORE_POINT_INTERVAL = 21600 # 6h, like the other sweeps |
| 1491 | _DEFAULT_RESTORE_POINT_KEEP_LAST = 20 |
| 1492 | _DEFAULT_RESTORE_POINT_KEEP_DAYS = 7 |
| 1493 | |
| 1494 | |
| 1495 | def process_restore_point_retention(*, base_dir: Path | str = "data") -> dict | None: |
| 1496 | """Sweep AUTOMATIC restore points. Never touches a manual backup. |
| 1497 | |
| 1498 | ON BY DEFAULT, which is the opposite of the change-log pass above, and |
| 1499 | the difference is who asked for the files. A change log IS the audit |
| 1500 | trail, so how much to keep is not a decision a daemon makes on |
| 1501 | somebody's behalf. Restore points are scaffolding the server writes for |
| 1502 | itself before every package install -- nobody asked for them, and the |
| 1503 | hundredth-oldest one from three weeks ago is worth nothing. |
| 1504 | |
| 1505 | The box that motivated this had 157 of them at ~91MB, 14.3GB against |
| 1506 | 4.0GB free, on a machine also serving production, with 93 written in |
| 1507 | the preceding two days. Left alone it fills the disk in about a day, |
| 1508 | and a full disk on that box takes production down with it. That is the |
| 1509 | fourth time this shape of bug has appeared here (page_views, restore |
| 1510 | points, change logs, restore points again). |
| 1511 | |
| 1512 | Keeps the newest DBBASIC_RESTORE_POINT_KEEP_LAST (default 20) and |
| 1513 | anything younger than DBBASIC_RESTORE_POINT_KEEP_DAYS (default 7), so |
| 1514 | a burst of installs in one afternoon cannot evict the morning's safety |
| 1515 | net. Setting KEEP_LAST to 0 does NOT disable the sweep -- the floor is |
| 1516 | object_backup_index.MIN_KEEP -- because sweeping to the bone would |
| 1517 | remove the very safety net this feature exists to provide. |
| 1518 | |
| 1519 | The BUDGET (DBBASIC_RESTORE_POINT_MAX_BYTES, default 2GB) is what |
| 1520 | actually bounds the disk, and it outranks the window. A first cut of |
| 1521 | this pass had only a count and an age window; a dry run on the real box |
| 1522 | freed 78MB of 14.3GB, because 93 archives were two days old and the |
| 1523 | window protected them all. Count and age cannot bound bytes. |
| 1524 | """ |
| 1525 | marker_path = Path(base_dir) / RESTORE_POINT_RETENTION_MARKER_NAME |
| 1526 | interval = _env_int(RESTORE_POINT_RETENTION_INTERVAL_ENV, |
| 1527 | _DEFAULT_RESTORE_POINT_INTERVAL) |
| 1528 | now = time.time() |
| 1529 | try: |
| 1530 | if now - marker_path.stat().st_mtime < interval: |
| 1531 | return None |
| 1532 | except OSError: |
| 1533 | pass # no marker yet -> run now |
| 1534 | marker_path.parent.mkdir(parents=True, exist_ok=True) |
| 1535 | marker_path.write_text(str(now)) |
| 1536 | |
| 1537 | try: |
| 1538 | result = object_backup_index.prune_backups( |
| 1539 | data_dir=base_dir, |
| 1540 | keep_last=_env_int(RESTORE_POINT_KEEP_LAST_ENV, |
| 1541 | _DEFAULT_RESTORE_POINT_KEEP_LAST), |
| 1542 | keep_newer_than_days=_env_int(RESTORE_POINT_KEEP_DAYS_ENV, |
| 1543 | _DEFAULT_RESTORE_POINT_KEEP_DAYS), |
| 1544 | max_total_bytes=_env_int( |
| 1545 | RESTORE_POINT_MAX_BYTES_ENV, |
| 1546 | object_backup_index.DEFAULT_MAX_TOTAL_BYTES), |
| 1547 | ) |
| 1548 | except OSError: |
| 1549 | return None |
| 1550 | if result.get("removed"): |
| 1551 | freed_mb = int(result["freed_bytes"]) // (1024 * 1024) |
| 1552 | log(f"Restore points: removed {result['removed']} archive(s), " |
| 1553 | f"freed {freed_mb}MB, kept {result['kept']} " |
| 1554 | f"(+{result['protected']} manual, never swept)") |
| 1555 | for problem in result.get("errors") or []: |
| 1556 | log(f"Restore points: could not remove {problem}", "ERROR") |
| 1557 | return result |
| 1558 | |
| 1559 | |
| 1560 | ANALYTICS_RETENTION_MARKER_NAME = ".analytics_retention_last_run" |
| 1561 | ANALYTICS_RETENTION_INTERVAL_ENV = "DBBASIC_ANALYTICS_RETENTION_INTERVAL_SECONDS" |
| 1562 | _DEFAULT_ANALYTICS_RETENTION_INTERVAL = 21600 # 6h -- a rewrite of a big file isn't cheap |
| 1563 | |
| 1564 | |
| 1565 | def process_analytics_retention(*, base_dir: Path | str = "data") -> dict | None: |
| 1566 | """Age page_views rows out past the retention window, bounding the file. |
| 1567 | |
| 1568 | page_views is the platform's heaviest write path and a pure event log -- |
| 1569 | id-fold compaction can't shrink it (nothing is superseded), so retention is |
| 1570 | a time-windowed rewrite (object_records.prune_collection_records). Gated by |
| 1571 | a marker file + interval (default 6h), the same shape process_compactions |
| 1572 | uses -- a rewrite of a massive file is not something to do every tick. |
| 1573 | |
| 1574 | Returns None when disabled/not-due/not-installed; else the prune summary. |
| 1575 | """ |
| 1576 | if not object_analytics.analytics_enabled(): |
| 1577 | return None |
| 1578 | |
| 1579 | marker_path = Path(base_dir) / ANALYTICS_RETENTION_MARKER_NAME |
| 1580 | interval = _env_int(ANALYTICS_RETENTION_INTERVAL_ENV, _DEFAULT_ANALYTICS_RETENTION_INTERVAL) |
| 1581 | now = time.time() |
| 1582 | try: |
| 1583 | if now - marker_path.stat().st_mtime < interval: |
| 1584 | return None |
| 1585 | except OSError: |
| 1586 | pass # no marker yet -> run now |
| 1587 | marker_path.parent.mkdir(parents=True, exist_ok=True) |
| 1588 | marker_path.write_text(str(now)) |
| 1589 | |
| 1590 | days = object_analytics.retention_days() |
| 1591 | cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat().replace("+00:00", "Z") |
| 1592 | cap = object_analytics.max_rows() |
| 1593 | try: |
| 1594 | result = object_records.prune_collection_records( |
| 1595 | object_analytics.PAGE_VIEWS_COLLECTION, |
| 1596 | keep_newer_than=cutoff, timestamp_field="created_at", |
| 1597 | # Two bounds, because a retention window says how far back to |
| 1598 | # keep and nothing at all about how much can arrive inside it. |
| 1599 | keep_last=(cap or None), |
| 1600 | base_dir=base_dir, |
| 1601 | ) |
| 1602 | except (object_collections.CollectionNotFoundError, |
| 1603 | object_collections.InvalidCollectionNameError, OSError, ValueError): |
| 1604 | return None |
| 1605 | if result.get("pruned"): |
| 1606 | bound = f"older than {days}d" + (f" or beyond {cap} rows" if cap else "") |
| 1607 | log(f"Analytics: pruned {result['removed']} page_view(s) {bound} " |
| 1608 | f"({result['rows_before']} -> {result['rows_after']})") |
| 1609 | return result |
| 1610 | |
| 1611 | |
| 1612 | ATTENTION_SOURCES_COLLECTION = "attention_sources" |
| 1613 | ATTENTION_COUNTS_COLLECTION = "attention_counts" |
| 1614 | ATTENTION_MARKER_NAME = ".attention_last_run" |
| 1615 | ATTENTION_INTERVAL_ENV = "DBBASIC_ATTENTION_INTERVAL_SECONDS" |
| 1616 | # Five minutes. A count that is five minutes stale is fine -- nobody |
| 1617 | # confirms a receipt in the seconds after it lands -- and the page says |
| 1618 | # how old the number is rather than implying it is live. |
| 1619 | _DEFAULT_ATTENTION_INTERVAL = 300 |
| 1620 | ATTENTION_ACTOR = "daemon:attention" |
| 1621 | ATTENTION_METHOD = "COUNT" |
| 1622 | _ATTENTION_DETAIL_LIMIT = 200 |
| 1623 | _ATTENTION_ERROR_LIMIT = 400 |
| 1624 | _TRUE_TEXT = ("true", "1", "yes", "on") |
| 1625 | |
| 1626 | |
| 1627 | def _attention_text(value) -> str: |
| 1628 | return str(value if value is not None else "").strip() |
| 1629 | |
| 1630 | |
| 1631 | def _attention_count_value(answer) -> int: |
| 1632 | """The integer a provider promised, or a refusal to guess. |
| 1633 | |
| 1634 | Raises ValueError rather than coercing, because every coercion here is |
| 1635 | a way for a broken provider to look like an empty queue. A provider |
| 1636 | that answers with a string, a float, a bool, or nothing at all has a |
| 1637 | bug, and the only useful thing to do with it is say so. |
| 1638 | """ |
| 1639 | if not isinstance(answer, dict): |
| 1640 | raise ValueError(f"provider returned {type(answer).__name__}, expected an object") |
| 1641 | if "count" not in answer: |
| 1642 | raise ValueError("provider returned no 'count'") |
| 1643 | count = answer["count"] |
| 1644 | if isinstance(count, bool) or not isinstance(count, int): |
| 1645 | raise ValueError(f"provider returned a {type(count).__name__} count, expected an integer") |
| 1646 | if count < 0: |
| 1647 | raise ValueError(f"provider returned a negative count: {count}") |
| 1648 | return count |
| 1649 | |
| 1650 | |
| 1651 | def process_attention(*, base_dir: Path | str = "data") -> dict | None: |
| 1652 | """Run every declared attention provider and write the counts down. |
| 1653 | |
| 1654 | Packages declare what "needs a human" means in their own domain |
| 1655 | (object_packages' `attention` section -> the attention_sources |
| 1656 | collection). This pass executes each declaration's object with method |
| 1657 | COUNT -- a distinct verb, so the same object may also expose GET or |
| 1658 | POST for a person -- and stores the answer in attention_counts, which |
| 1659 | is the one small table the home page and the app list read. |
| 1660 | |
| 1661 | **Why a rollup and not a live fold.** Because folding a dozen |
| 1662 | collections on every home render is how a home page becomes the |
| 1663 | slowest page, and worse than slow: this box went to 675MB resident and |
| 1664 | started swapping because something folded a big collection on a timer. |
| 1665 | A home screen that folds a dozen collections per render is that exact |
| 1666 | mistake with a nicer name -- and it makes the front page's cost grow |
| 1667 | with the number of installed packages, which is the one place cost |
| 1668 | must not grow. The pass is marker-gated (DBBASIC_ATTENTION_INTERVAL_ |
| 1669 | SECONDS, default 300) like process_compactions and |
| 1670 | process_materializations, so the page's cost is one small read no |
| 1671 | matter how many apps declare a queue, and the staleness is bounded and |
| 1672 | stated rather than pretended away. |
| 1673 | |
| 1674 | **A broken provider must look broken, never look calm.** A provider |
| 1675 | that raises, or answers with something that is not an integer, has its |
| 1676 | `error` written and its PREVIOUS count kept. Zeroing it would report |
| 1677 | an empty queue when what actually happened is that nobody looked, and |
| 1678 | a queue that looks handled is the single worst thing this table could |
| 1679 | say. `computed_at` is left at the last SUCCESSFUL run for the same |
| 1680 | reason: the age on the page should describe the number, not the |
| 1681 | attempt. |
| 1682 | |
| 1683 | Zero counts are written and kept. The absence of a row means nobody |
| 1684 | has ever run the provider, which is a different fact from "the queue |
| 1685 | is empty", and only the surfaces decide that zeros are not news. |
| 1686 | |
| 1687 | A source the operator muted is not executed at all, and its stored |
| 1688 | count is removed -- a muted queue should cost nothing and show |
| 1689 | nothing, and leaving the last number behind would show a stale one. |
| 1690 | |
| 1691 | Returns None when the collection is not installed (app-nav absent) or |
| 1692 | the interval has not elapsed; otherwise a summary of the pass. |
| 1693 | """ |
| 1694 | try: |
| 1695 | sources = object_records.read_collection_records( |
| 1696 | ATTENTION_SOURCES_COLLECTION, base_dir=base_dir) |
| 1697 | except (object_collections.CollectionNotFoundError, |
| 1698 | object_collections.InvalidCollectionNameError, OSError, ValueError): |
| 1699 | return None |
| 1700 | if not sources: |
| 1701 | return None |
| 1702 | |
| 1703 | marker_path = Path(base_dir) / ATTENTION_MARKER_NAME |
| 1704 | interval = _env_int(ATTENTION_INTERVAL_ENV, _DEFAULT_ATTENTION_INTERVAL) |
| 1705 | now = time.time() |
| 1706 | try: |
| 1707 | if now - marker_path.stat().st_mtime < interval: |
| 1708 | return None |
| 1709 | except OSError: |
| 1710 | pass # no marker yet -> run now |
| 1711 | marker_path.parent.mkdir(parents=True, exist_ok=True) |
| 1712 | marker_path.write_text(str(now)) |
| 1713 | |
| 1714 | try: |
| 1715 | from python_object_runtime import PythonObjectRuntime |
| 1716 | except ImportError: |
| 1717 | return None |
| 1718 | runtime = PythonObjectRuntime(base_dir=str(base_dir)) |
| 1719 | |
| 1720 | try: |
| 1721 | existing = {row["id"]: row for row in object_records.read_collection_records( |
| 1722 | ATTENTION_COUNTS_COLLECTION, base_dir=base_dir) if row.get("id")} |
| 1723 | except (object_collections.CollectionNotFoundError, |
| 1724 | object_collections.InvalidCollectionNameError, OSError, ValueError): |
| 1725 | # The counts collection is what this pass exists to write. Without |
| 1726 | # it there is nowhere to put an answer, so there is no point |
| 1727 | # running any provider. |
| 1728 | return None |
| 1729 | |
| 1730 | computed_at = datetime.now(timezone.utc).isoformat() |
| 1731 | counted = failed = muted = 0 |
| 1732 | |
| 1733 | for source in sources: |
| 1734 | source_id = _attention_text(source.get("id")) |
| 1735 | if not source_id: |
| 1736 | continue |
| 1737 | live = existing.get(source_id) |
| 1738 | |
| 1739 | if _attention_text(source.get("operator_muted")).lower() in _TRUE_TEXT: |
| 1740 | muted += 1 |
| 1741 | if live is not None: |
| 1742 | try: |
| 1743 | object_records.delete_collection_record( |
| 1744 | ATTENTION_COUNTS_COLLECTION, source_id, |
| 1745 | base_dir=base_dir, actor=ATTENTION_ACTOR) |
| 1746 | except Exception: # noqa: BLE001 -- a stuck row must not stop the pass |
| 1747 | pass |
| 1748 | continue |
| 1749 | |
| 1750 | row = { |
| 1751 | "source_id": source_id, |
| 1752 | "label": _attention_text(source.get("label")), |
| 1753 | "path": _attention_text(source.get("path")), |
| 1754 | "nav_id": _attention_text(source.get("nav_id")), |
| 1755 | "group": _attention_text(source.get("group")) or "Apps", |
| 1756 | "severity": _attention_text(source.get("severity")) or "normal", |
| 1757 | } |
| 1758 | |
| 1759 | object_id = _attention_text(source.get("object_id")) |
| 1760 | try: |
| 1761 | answer = _execute_target(runtime, object_id, ATTENTION_METHOD, {}) |
| 1762 | count = _attention_count_value(answer) |
| 1763 | except Exception as exc: # noqa: BLE001 -- one bad provider must not blind the rest |
| 1764 | failed += 1 |
| 1765 | log(f"Attention: {source_id} ({object_id}) failed: {exc}", "ERROR") |
| 1766 | row["error"] = f"{type(exc).__name__}: {exc}"[:_ATTENTION_ERROR_LIMIT] |
| 1767 | # The previous number survives, and so does its timestamp: an |
| 1768 | # unanswered question is not the answer zero. |
| 1769 | row["count"] = _attention_text((live or {}).get("count")) or "0" |
| 1770 | row["detail"] = _attention_text((live or {}).get("detail")) |
| 1771 | row["computed_at"] = _attention_text((live or {}).get("computed_at")) |
| 1772 | else: |
| 1773 | counted += 1 |
| 1774 | row["count"] = str(count) |
| 1775 | row["detail"] = _attention_text(answer.get("detail"))[:_ATTENTION_DETAIL_LIMIT] |
| 1776 | row["error"] = "" |
| 1777 | row["computed_at"] = computed_at |
| 1778 | |
| 1779 | try: |
| 1780 | if live is None: |
| 1781 | object_records.create_collection_record( |
| 1782 | ATTENTION_COUNTS_COLLECTION, {"id": source_id, **row}, |
| 1783 | base_dir=base_dir, actor=ATTENTION_ACTOR) |
| 1784 | elif any(_attention_text(live.get(field)) != row[field] for field in row): |
| 1785 | object_records.update_collection_record( |
| 1786 | ATTENTION_COUNTS_COLLECTION, source_id, row, |
| 1787 | base_dir=base_dir, actor=ATTENTION_ACTOR, preserve_read_only=True) |
| 1788 | except Exception as exc: # noqa: BLE001 |
| 1789 | log(f"Attention: {source_id} count could not be stored: {exc}", "ERROR") |
| 1790 | |
| 1791 | if counted or failed: |
| 1792 | log(f"Attention: counted {counted} queue(s)" |
| 1793 | + (f", {failed} provider(s) failed" if failed else "") |
| 1794 | + (f", {muted} muted" if muted else "")) |
| 1795 | return {"sources": len(sources), "counted": counted, "failed": failed, "muted": muted} |
| 1796 | |
| 1797 | |
| 1798 | def _connector_package_roots() -> list[str]: |
| 1799 | """Package source roots the reconcile pass loads connectors from, private |
| 1800 | overlay first -- mirrors object_server's resolution so a private connector |
| 1801 | (e.g. Mailcow, in packages-private/) is found on the deployment where it is |
| 1802 | installed, and a private declaration shadows an open one for a collection.""" |
| 1803 | roots: list[str] = [] |
| 1804 | packages = os.environ.get("DBBASIC_PACKAGES_DIR", object_packages.PACKAGES_DIR) |
| 1805 | # Private overlay defaults to a sibling of the active packages dir (same |
| 1806 | # rule as object_server._private_packages_dir), so overriding the packages |
| 1807 | # dir isolates the overlay too. |
| 1808 | private = os.environ.get("DBBASIC_PRIVATE_PACKAGES_DIR") or str(Path(packages).parent / "packages-private") |
| 1809 | if private and Path(private).is_dir(): |
| 1810 | roots.append(private) |
| 1811 | roots.append(packages) |
| 1812 | return roots |
| 1813 | |
| 1814 | |
| 1815 | def _connector_declarations() -> list[dict]: |
| 1816 | """All connector declarations across roots, deduped by collection with the |
| 1817 | private overlay winning (first root wins).""" |
| 1818 | decls: list[dict] = [] |
| 1819 | seen: set[str] = set() |
| 1820 | for root in _connector_package_roots(): |
| 1821 | for decl in object_packages.iter_connectors(root=root): |
| 1822 | if decl["collection"] in seen: |
| 1823 | continue |
| 1824 | seen.add(decl["collection"]) |
| 1825 | decls.append(decl) |
| 1826 | return decls |
| 1827 | |
| 1828 | |
| 1829 | def process_connectors(*, base_dir: Path | str = "data") -> dict | None: |
| 1830 | """03 external connectors: reconcile each declared collection against its |
| 1831 | outside system. |
| 1832 | |
| 1833 | For every collection a package declares a connector for, selects the rows |
| 1834 | whose desired state the external world doesn't match yet (pending / |
| 1835 | pending_delete, backoff due), calls the connector's `reconcile(record)`, and |
| 1836 | writes the outcome back via object_connectors.plan_sync -- synced/deleted on |
| 1837 | success, retry-with-backoff on a transient error, dead on a permanent one or |
| 1838 | exhausted attempts. The driver owns the lifecycle; the connector only |
| 1839 | reports ok/error/permanent. Per-row try/except isolates one bad row, and |
| 1840 | per-collection isolation keeps one bad connector from stopping the rest. |
| 1841 | |
| 1842 | Returns None when disabled or nothing declares a connector; else a summary. |
| 1843 | """ |
| 1844 | if not object_connectors.connectors_pass_enabled(base_dir=base_dir): |
| 1845 | return None |
| 1846 | decls = _connector_declarations() |
| 1847 | if not decls: |
| 1848 | return None |
| 1849 | |
| 1850 | config = object_connectors.connector_config_from_env() |
| 1851 | now = datetime.now(timezone.utc) |
| 1852 | reconciled = synced = dead = skipped = 0 |
| 1853 | |
| 1854 | for decl in decls: |
| 1855 | try: |
| 1856 | records = object_records.read_collection_records(decl["collection"], base_dir=base_dir) |
| 1857 | except (object_collections.CollectionNotFoundError, |
| 1858 | object_collections.InvalidCollectionNameError, OSError, ValueError): |
| 1859 | continue # collection not installed here -> nothing to reconcile |
| 1860 | due = [r for r in records if object_connectors.is_due(r, now)] |
| 1861 | if not due: |
| 1862 | continue |
| 1863 | due.sort(key=lambda r: (str(r.get("sync_next_at") or ""), str(r.get("created_at") or ""))) |
| 1864 | try: |
| 1865 | reconcile = object_connectors.load_connector(decl["module"], decl["entry"]) |
| 1866 | except object_connectors.ConnectorLoadError as exc: |
| 1867 | log(f"Connector: {decl['package_id']}/{decl['collection']} not loaded: {exc}", "ERROR") |
| 1868 | continue |
| 1869 | |
| 1870 | for record in due[:config.batch_size]: |
| 1871 | try: |
| 1872 | outcome = reconcile(record, base_dir=base_dir) |
| 1873 | if not isinstance(outcome, dict): |
| 1874 | outcome = {"ok": False, "error": f"connector returned {type(outcome).__name__}, expected dict"} |
| 1875 | except Exception as exc: # noqa: BLE001 -- transient by default; isolate one bad row |
| 1876 | outcome = {"ok": False, "error": str(exc)} |
| 1877 | if outcome.get("skip"): |
| 1878 | # The connector declined this tick (e.g. unconfigured): leave the |
| 1879 | # row exactly as-is -- not an attempt, not a failure, so it waits |
| 1880 | # inspectably instead of backing off toward `dead`. Mirrors the |
| 1881 | # outbox's "unconfigured = queue only" posture. |
| 1882 | skipped += 1 |
| 1883 | continue |
| 1884 | update = object_connectors.plan_sync(record, outcome, config, now=now) |
| 1885 | try: |
| 1886 | object_records.update_collection_record( |
| 1887 | decl["collection"], record["id"], update, |
| 1888 | base_dir=base_dir, actor=object_connectors.DEFAULT_ACTOR, |
| 1889 | preserve_read_only=True, |
| 1890 | ) |
| 1891 | except Exception as exc: # noqa: BLE001 |
| 1892 | log(f"Connector: {decl['collection']} {record.get('id')} status write failed: {exc}", "ERROR") |
| 1893 | continue |
| 1894 | reconciled += 1 |
| 1895 | status = update.get(object_connectors.SYNC_STATUS_FIELD) |
| 1896 | if status in (object_connectors.STATUS_SYNCED, object_connectors.STATUS_DELETED): |
| 1897 | synced += 1 |
| 1898 | elif status == object_connectors.STATUS_DEAD: |
| 1899 | dead += 1 |
| 1900 | |
| 1901 | if reconciled: |
| 1902 | log(f"Connectors: reconciled {reconciled} ({synced} synced, {dead} dead)") |
| 1903 | return {"reconciled": reconciled, "synced": synced, "dead": dead, "skipped": skipped} |
| 1904 | |
| 1905 | |
| 1906 | # --- Main --- |
| 1907 | |
| 1908 | def shutdown(signum, frame): |
| 1909 | global _running |
| 1910 | log("Shutting down...") |
| 1911 | _running = False |
| 1912 | |
| 1913 | |
| 1914 | def main(): |
| 1915 | import argparse |
| 1916 | |
| 1917 | parser = argparse.ArgumentParser(description="Object Primitive Daemon") |
| 1918 | parser.add_argument("--interval", type=float, default=1.0, |
| 1919 | help="Poll interval in seconds (default: 1.0)") |
| 1920 | args = parser.parse_args() |
| 1921 | |
| 1922 | # The primitive-runtime passes (scheduler/queue/events trigger objects) |
| 1923 | # need the optional dbbasic_object_core runtime. The storage passes |
| 1924 | # (compaction, auto-transitions, cleanups) are stdlib and must run |
| 1925 | # regardless -- a deployment without the runtime still gets them. |
| 1926 | try: |
| 1927 | from dbbasic_object_core.runtime.object_runtime import ObjectRuntime |
| 1928 | except ImportError: |
| 1929 | ObjectRuntime = None |
| 1930 | # Native fallback: the repo's own runtime serves the scheduler/queue/ |
| 1931 | # events passes (load_object + state_manager + execute_object are the |
| 1932 | # whole surface they use). The legacy dbbasic_object_core runtime keeps |
| 1933 | # precedence where it is installed; deployments without it -- the |
| 1934 | # droplet -- get working scheduled tasks instead of a silent disable |
| 1935 | # (found 2026-07-24: system_invoice_aging/fin_recurring had NO working |
| 1936 | # schedule path in production because this import was missing). |
| 1937 | if ObjectRuntime is None: |
| 1938 | try: |
| 1939 | from python_object_runtime import PythonObjectRuntime as _NativeRuntime |
| 1940 | except ImportError: |
| 1941 | _NativeRuntime = None |
| 1942 | else: |
| 1943 | _NativeRuntime = None |
| 1944 | |
| 1945 | # Same data-dir resolution as the server: env first, ./data fallback. |
| 1946 | base_dir = os.environ.get("DBBASIC_DATA_DIR", "data") |
| 1947 | |
| 1948 | signal.signal(signal.SIGTERM, shutdown) |
| 1949 | signal.signal(signal.SIGINT, shutdown) |
| 1950 | |
| 1951 | print("=" * 60) |
| 1952 | print("Object Primitive Daemon") |
| 1953 | print("=" * 60) |
| 1954 | print(f"Poll interval: {args.interval}s") |
| 1955 | print(f"Data dir: {base_dir}") |
| 1956 | print(f"Object roots: {', '.join(str(root) for root in _object_roots())}") |
| 1957 | if ObjectRuntime is None and _NativeRuntime is None: |
| 1958 | print("Object runtime: NOT installed (scheduler/queue/events passes disabled)") |
| 1959 | else: |
| 1960 | _runtime_label = "dbbasic_object_core" if ObjectRuntime is not None else "native (python_object_runtime)" |
| 1961 | print(f"Object runtime: {_runtime_label}") |
| 1962 | print(f"Scheduler: {'enabled' if _find_trigger_file('scheduler') else 'no scheduler object'}") |
| 1963 | print(f"Queue: {'enabled' if _find_trigger_file('queue') else 'no queue object'}") |
| 1964 | print(f"Events: {'enabled' if _find_trigger_file('events') else 'no events object'}") |
| 1965 | print(f"Croniter: {'available' if croniter else 'NOT installed (cron tasks disabled)'}") |
| 1966 | print(f"Rate limit cleanup: {'enabled' if (Path(base_dir) / 'ratelimit').exists() else 'no ratelimit dir yet'}") |
| 1967 | print(f"Compaction interval: {_env_int(COMPACTION_INTERVAL_SECONDS_ENV, _DEFAULT_COMPACTION_INTERVAL_SECONDS)}s") |
| 1968 | _startup_rules = auto_transition_rules_from_env() |
| 1969 | print( |
| 1970 | f"Auto-transition rules: {len(_startup_rules)} configured " |
| 1971 | f"(interval {_env_int(AUTO_TRANSITION_INTERVAL_SECONDS_ENV, _DEFAULT_AUTO_TRANSITION_INTERVAL_SECONDS)}s)" |
| 1972 | ) |
| 1973 | print(f"Rollups: {'enabled' if object_rollups.rollup_pass_enabled(base_dir=base_dir) else 'disabled (rollup_enabled flag off)'}") |
| 1974 | print( |
| 1975 | f"Materialize: {'enabled' if object_materialize.materialize_pass_enabled(base_dir=base_dir) else 'disabled (materialize_enabled flag off)'} " |
| 1976 | f"(interval {_env_int(MATERIALIZE_INTERVAL_SECONDS_ENV, _DEFAULT_MATERIALIZE_INTERVAL_SECONDS)}s)" |
| 1977 | ) |
| 1978 | print(f"Notify: {'enabled' if object_notify.notify_pass_enabled(base_dir=base_dir) else 'disabled (notify_enabled flag off)'} (every poll)") |
| 1979 | print(f"Change dispatch: {'enabled' if object_change_dispatch.change_dispatch_enabled(base_dir=base_dir) else 'disabled (DBBASIC_ENABLE_CHANGE_DISPATCH unset)'}") |
| 1980 | _clp = object_record_changes.retention_policy() |
| 1981 | if _clp["days"] or _clp["max_entries"]: |
| 1982 | _bounds = " and ".join( |
| 1983 | part for part in ( |
| 1984 | f"{_clp['days']}d" if _clp["days"] else "", |
| 1985 | f"{_clp['max_entries']} entries" if _clp["max_entries"] else "", |
| 1986 | ) if part) |
| 1987 | _fate = "archived" if _clp["archive"] else "DELETED" |
| 1988 | _keep = ("all archives kept" if not _clp["keep_archives"] |
| 1989 | else f"newest {_clp['keep_archives']} archives kept") |
| 1990 | print(f"Change logs: keep {_bounds}; older entries {_fate} ({_keep})") |
| 1991 | else: |
| 1992 | print("Change logs: unbounded (set DBBASIC_CHANGE_LOG_RETENTION_DAYS " |
| 1993 | "or DBBASIC_CHANGE_LOG_MAX_ENTRIES)") |
| 1994 | # Books readiness is advisory, never a gate -- the composers are |
| 1995 | # deliberately soft dependencies. But a composer that skips because its |
| 1996 | # accounts are unmapped otherwise says so only in a return value nobody |
| 1997 | # reads, which is harmless on a demo box and a data-integrity hole the |
| 1998 | # day real books arrive. Say it out loud at boot instead. |
| 1999 | try: |
| 2000 | _books = object_books_status.books_status(base_dir=base_dir) |
| 2001 | if _books["ready"]: |
| 2002 | print("Books: ready (every installed composer can post)") |
| 2003 | else: |
| 2004 | print(f"Books: NOT READY -- {_books['summary']}") |
| 2005 | for _problem in _books.get("problems", [])[:6]: |
| 2006 | print(f" - {_problem.get('id')}: {_problem.get('impact')}") |
| 2007 | except Exception as _exc: # never let a status check stop the daemon |
| 2008 | print(f"Books: status unavailable ({_exc})") |
| 2009 | _smtp_config = object_email.smtp_config_from_env() |
| 2010 | if not object_email.email_pass_enabled(base_dir=base_dir): |
| 2011 | _email_state = "disabled (email_enabled flag off)" |
| 2012 | elif not _smtp_config.configured: |
| 2013 | _email_state = f"queuing only (DBBASIC_SMTP_MODE={_smtp_config.mode!r}, not configured)" |
| 2014 | else: |
| 2015 | _email_state = f"enabled (mode={_smtp_config.mode})" |
| 2016 | print(f"Email: {_email_state}") |
| 2017 | _connector_decls = _connector_declarations() |
| 2018 | if not object_connectors.connectors_pass_enabled(base_dir=base_dir): |
| 2019 | _connectors_state = "disabled (connectors_enabled flag off)" |
| 2020 | elif not _connector_decls: |
| 2021 | _connectors_state = "no connectors declared" |
| 2022 | else: |
| 2023 | _connectors_state = f"{len(_connector_decls)} connector(s): " + ", ".join( |
| 2024 | f"{d['package_id']}/{d['collection']}" for d in _connector_decls) |
| 2025 | print(f"Connectors: {_connectors_state}") |
| 2026 | try: |
| 2027 | _attention_sources = object_records.read_collection_records( |
| 2028 | ATTENTION_SOURCES_COLLECTION, base_dir=base_dir) |
| 2029 | except Exception: |
| 2030 | _attention_sources = None |
| 2031 | if _attention_sources is None: |
| 2032 | _attention_state = "no attention_sources collection (app-nav not installed)" |
| 2033 | elif not _attention_sources: |
| 2034 | _attention_state = "no queues declared" |
| 2035 | else: |
| 2036 | _attention_state = ( |
| 2037 | f"{len(_attention_sources)} queue(s) " |
| 2038 | f"(interval {_env_int(ATTENTION_INTERVAL_ENV, _DEFAULT_ATTENTION_INTERVAL)}s)") |
| 2039 | print(f"Attention: {_attention_state}") |
| 2040 | if object_analytics.analytics_enabled(): |
| 2041 | print(f"Analytics: capture ON (retention {object_analytics.retention_days()}d)") |
| 2042 | else: |
| 2043 | print("Analytics: capture off (DBBASIC_ANALYTICS unset)") |
| 2044 | # Stated in words, every boot, because the cookie is the only thing |
| 2045 | # this server stores on somebody else's device and the setting that |
| 2046 | # turns it on is one word in an env file. An operator should not have |
| 2047 | # to go and read a document to find out what they took on. |
| 2048 | print(f"Analytics: {object_analytics.visitor_cookie_posture()}") |
| 2049 | print() |
| 2050 | print("Press Ctrl+C to stop") |
| 2051 | print("=" * 60) |
| 2052 | print() |
| 2053 | |
| 2054 | if ObjectRuntime is not None: |
| 2055 | runtime = ObjectRuntime(base_dir=str(base_dir)) |
| 2056 | elif _NativeRuntime is not None: |
| 2057 | runtime = _NativeRuntime() |
| 2058 | else: |
| 2059 | runtime = None |
| 2060 | last_event_cleanup = 0.0 |
| 2061 | |
| 2062 | while _running: |
| 2063 | if runtime is not None: |
| 2064 | try: |
| 2065 | process_scheduler(runtime) |
| 2066 | except Exception as e: |
| 2067 | log(f"Scheduler error: {e}", 'ERROR') |
| 2068 | |
| 2069 | try: |
| 2070 | process_queue(runtime) |
| 2071 | except Exception as e: |
| 2072 | log(f"Queue error: {e}", 'ERROR') |
| 2073 | |
| 2074 | try: |
| 2075 | process_events(runtime) |
| 2076 | except Exception as e: |
| 2077 | log(f"Events error: {e}", 'ERROR') |
| 2078 | |
| 2079 | try: |
| 2080 | cleanup_ratelimit(base_dir=base_dir) |
| 2081 | except Exception as e: |
| 2082 | log(f"Cleanup error: {e}", 'ERROR') |
| 2083 | |
| 2084 | try: |
| 2085 | process_compactions(base_dir=base_dir) |
| 2086 | except Exception as e: |
| 2087 | log(f"Compaction error: {e}", 'ERROR') |
| 2088 | |
| 2089 | try: |
| 2090 | process_stale_transitions(base_dir=base_dir) |
| 2091 | except Exception as e: |
| 2092 | log(f"Auto-transition error: {e}", 'ERROR') |
| 2093 | |
| 2094 | try: |
| 2095 | process_rollups(base_dir=base_dir) |
| 2096 | except Exception as e: |
| 2097 | log(f"Rollup error: {e}", 'ERROR') |
| 2098 | |
| 2099 | try: |
| 2100 | process_materializations(base_dir=base_dir) |
| 2101 | except Exception as e: |
| 2102 | log(f"Materialize error: {e}", 'ERROR') |
| 2103 | |
| 2104 | try: |
| 2105 | process_notifications(base_dir=base_dir) |
| 2106 | except Exception as e: |
| 2107 | log(f"Notify error: {e}", 'ERROR') |
| 2108 | |
| 2109 | try: |
| 2110 | object_change_dispatch.dispatch_pending(base_dir=base_dir) |
| 2111 | except Exception as e: |
| 2112 | log(f"Change dispatch error: {e}", 'ERROR') |
| 2113 | |
| 2114 | try: |
| 2115 | process_email_outbox(base_dir=base_dir) |
| 2116 | except Exception as e: |
| 2117 | log(f"Email error: {e}", 'ERROR') |
| 2118 | |
| 2119 | try: |
| 2120 | process_connectors(base_dir=base_dir) |
| 2121 | except Exception as e: |
| 2122 | log(f"Connectors error: {e}", 'ERROR') |
| 2123 | |
| 2124 | try: |
| 2125 | process_analytics_retention(base_dir=base_dir) |
| 2126 | except Exception as e: |
| 2127 | log(f"Analytics retention error: {e}", 'ERROR') |
| 2128 | |
| 2129 | try: |
| 2130 | process_change_log_retention(base_dir=base_dir) |
| 2131 | except Exception as e: |
| 2132 | log(f"Change log retention error: {e}", 'ERROR') |
| 2133 | |
| 2134 | try: |
| 2135 | process_restore_point_retention(base_dir=base_dir) |
| 2136 | except Exception as e: |
| 2137 | log(f"Restore point retention error: {e}", 'ERROR') |
| 2138 | |
| 2139 | try: |
| 2140 | process_attention(base_dir=base_dir) |
| 2141 | except Exception as e: |
| 2142 | log(f"Attention error: {e}", 'ERROR') |
| 2143 | |
| 2144 | now = time.time() |
| 2145 | if now - last_event_cleanup >= EVENT_CLEANUP_INTERVAL_SECONDS: |
| 2146 | last_event_cleanup = now |
| 2147 | try: |
| 2148 | cleanup_events(base_dir=base_dir) |
| 2149 | except Exception as e: |
| 2150 | log(f"Event cleanup error: {e}", 'ERROR') |
| 2151 | |
| 2152 | # Sleep in small increments for responsive shutdown |
| 2153 | deadline = now + args.interval |
| 2154 | while _running and time.time() < deadline: |
| 2155 | time.sleep(0.1) |
| 2156 | |
| 2157 | log("Daemon stopped.") |
| 2158 | |
| 2159 | |
| 2160 | if __name__ == "__main__": |
| 2161 | main() |