Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_daemon.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_daemon.py · 89.5 KB · 2161 lines
1#!/usr/bin/env python3
2"""
3Object Primitive Daemon — Scheduler, Queue, Events
4
5Background worker that polls trigger objects and executes target objects.
6
7Runs alongside the HTTP server, sharing file-based state (TSV).
8Does not require HTTP or auth — executes objects directly via ObjectRuntime.
9
10Usage:
11 python object_daemon.py
12 python object_daemon.py --interval 5 # poll every 5 seconds
13"""
14from __future__ import annotations
15
16import json
17import os
18import signal
19import time
20import urllib.error
21import urllib.request
22from datetime import datetime, timedelta, timezone
23from pathlib import Path
24from typing import TYPE_CHECKING, Any
25
26if TYPE_CHECKING:
27 from dbbasic_object_core.runtime.object_runtime import ObjectRuntime
28
29try:
30 from croniter import croniter
31except ImportError:
32 croniter = None
33
34import object_analytics
35import object_backup_index
36import object_books_status
37import object_change_dispatch
38import object_collections
39import object_connectors
40import object_email
41import object_events
42import object_identity
43import object_ids
44import object_materialize
45import object_notify
46import object_packages
47import object_record_changes
48import object_records
49import object_rollups
50from object_execution import ObjectExecutionFailure, ObjectExecutionRequest, execute_object
51from object_namespace import find_trigger_file, get_object_roots, resolve_object_id
52
53# Daemon state
54_running = True
55EVENT_KEEP_COUNT_ENV = "DBBASIC_EVENT_KEEP_COUNT"
56EVENT_KEEP_SECONDS_ENV = "DBBASIC_EVENT_KEEP_SECONDS"
57EVENT_CLEANUP_INTERVAL_SECONDS = 60.0
58
59# Compaction pass (docs/append-only-storage-design.md Compaction: "run on
60# a schedule or when the superseded-row ratio passes a threshold -- never
61# inline in a request"). See process_compactions.
62COMPACTION_INTERVAL_SECONDS_ENV = "DBBASIC_COMPACTION_INTERVAL_SECONDS"
63COMPACTION_BLOAT_RATIO_ENV = "DBBASIC_COMPACTION_BLOAT_RATIO"
64_DEFAULT_COMPACTION_INTERVAL_SECONDS = 3600
65_DEFAULT_COMPACTION_BLOAT_RATIO = 1.0
66COMPACTION_MARKER_NAME = ".compaction_last_run"
67
68# Stale-state auto-transition pass. The predecessor system's "48-hour
69# auto-approve" job (waiting_on_client -> approved on tasks) was written
70# but never scheduled anywhere, silently stranding 200+ records in a
71# waiting state forever. To make that class of mistake impossible here,
72# this pass ships ON by default -- see _DEFAULT_AUTO_TRANSITION_RULES --
73# and requires no setup beyond running the daemon. See
74# process_stale_transitions.
75AUTO_TRANSITION_RULES_ENV = "DBBASIC_AUTO_TRANSITION_RULES"
76AUTO_TRANSITION_INTERVAL_SECONDS_ENV = "DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS"
77_DEFAULT_AUTO_TRANSITION_INTERVAL_SECONDS = 3600
78AUTO_TRANSITION_MARKER_NAME = ".auto_transition_last_run"
79AUTO_TRANSITION_ACTOR = "daemon:auto-transition"
80_DEFAULT_AUTO_TRANSITION_RULES = json.dumps([
81 {"collection": "tasks", "field": "status", "from": "waiting_on_client",
82 "to": "approved", "after_hours": 48},
83])
84
85# Collections named in a rule but not found on disk (the owning package
86# isn't installed) are logged once per process, not once per poll --
87# see _apply_auto_transition_rule.
88_WARNED_UNKNOWN_AUTO_TRANSITION_COLLECTIONS: set[str] = set()
89
90# Materialize pass (plan/vocabulary/61-materialize-spec.md, object_materialize.py).
91# Unlike process_rollups (whose due-gate lives entirely on each
92# rollup_definitions row -- see process_rollups' own docstring),
93# materialize's Events section explicitly asks for process_stale_
94# transitions' marker-file-gated shape instead: "riding the exact
95# marker-file-gated, try/except-per-item pattern process_stale_
96# transitions already establishes." See process_materializations.
97MATERIALIZE_INTERVAL_SECONDS_ENV = "DBBASIC_MATERIALIZE_INTERVAL_SECONDS"
98_DEFAULT_MATERIALIZE_INTERVAL_SECONDS = 3600
99MATERIALIZE_MARKER_NAME = ".materialize_last_run"
100# 12 notify: the cursor file holds the ISO timestamp of the last record-change
101# entry process_notifications has already turned into notifications. Unlike the
102# other passes' interval markers, this is a POSITION (content), not a clock
103# (mtime) -- the pass runs every poll for near-instant delivery and advances
104# the cursor past what it processed. First run stamps it at "now" so a fresh
105# install never backfills notifications for the entire change history.
106NOTIFY_CURSOR_NAME = ".notify_cursor"
107
108# Definitions referencing a source/output/child collection that doesn't
109# exist yet (the owning package isn't installed) are logged once per
110# process, not once per poll -- mirrors _WARNED_UNKNOWN_AUTO_TRANSITION_
111# COLLECTIONS exactly (61's Degradation section names this precedent).
112_WARNED_UNKNOWN_MATERIALIZE_COLLECTIONS: set[str] = set()
113
114
115def log(msg, level='INFO'):
116 ts = time.strftime('%Y-%m-%d %H:%M:%S')
117 print(f"[{ts}] [{level}] {msg}")
118
119
120def _object_roots() -> list[Path]:
121 """Return object source roots in lookup order."""
122 return get_object_roots()
123
124
125def _find_trigger_file(trigger_name: str) -> Path | None:
126 """Find a trigger object in the configured object roots."""
127 return find_trigger_file(trigger_name)
128
129
130# --- Scheduler ---
131
132def process_scheduler(runtime: ObjectRuntime):
133 """Check scheduler for due tasks and execute them."""
134 scheduler_file = _find_trigger_file('scheduler')
135 if scheduler_file is None:
136 return
137
138 obj = runtime.load_object(scheduler_file) # ID = 'scheduler' (filename stem)
139 obj.state_manager.reload() # Re-read from disk (server may have written new tasks)
140 state = obj.state_manager.get_all()
141 now = int(time.time())
142
143 for key, value in list(state.items()):
144 if not key.startswith('task_'):
145 continue
146
147 try:
148 task = json.loads(value)
149 except (json.JSONDecodeError, TypeError):
150 continue
151
152 if task.get('status') != 'active':
153 continue
154
155 # Calculate next_run if not set
156 next_run = task.get('next_run')
157 if next_run is None:
158 next_run = _calculate_next_run(task)
159 if next_run is None:
160 continue
161 task['next_run'] = next_run
162 obj.state_manager.set(key, json.dumps(task))
163
164 if next_run > now:
165 continue
166
167 # Task is due — execute target
168 target_id = task.get('object_id')
169 method = task.get('method', 'POST')
170 payload = task.get('payload', {})
171
172 if not target_id:
173 continue
174
175 log(f"Scheduler: executing {target_id}.{method} (task {task['id']})")
176
177 run_started = time.time()
178 run_result = None
179 run_error: Exception | None = None
180 try:
181 run_result = _execute_target(runtime, target_id, method, payload)
182 log(f"Scheduler: {target_id}.{method} completed")
183 except Exception as e:
184 run_error = e
185 log(f"Scheduler: {target_id}.{method} failed: {e}", 'ERROR')
186 _record_scheduler_run(
187 task, target_id, method,
188 started=run_started,
189 duration_ms=int((time.time() - run_started) * 1000),
190 result=run_result,
191 error=run_error,
192 )
193
194 # Update task
195 task['last_run'] = now
196 task['run_count'] = task.get('run_count', 0) + 1
197
198 if task.get('type') == 'onetime':
199 task['status'] = 'completed'
200 task['next_run'] = None
201 else:
202 task['next_run'] = _calculate_next_run(task, after=now)
203
204 obj.state_manager.set(key, json.dumps(task))
205
206
207def _record_scheduler_run(task, target_id, method, *, started, duration_ms,
208 result, error):
209 """Append one scheduler_runs row -- run history an operator can query.
210
211 A run's outcome must be a RECORD, not a stdout line only ssh can reach
212 (the observability hole that let q9's auto-approve and this server's own
213 scheduler passes sit dead unnoticed). Best-effort: skips silently when
214 the scheduler_runs collection is not installed (system-dashboard ships
215 it) and never breaks the pass.
216 """
217 try:
218 base_dir = os.environ.get("DBBASIC_DATA_DIR", "data")
219 row = {
220 "id": object_ids.new_uuid4(),
221 "task_id": str(task.get("id") or ""),
222 "object_id": str(target_id),
223 "method": str(method),
224 "ok": "false" if error is not None else "true",
225 "started_at": datetime.fromtimestamp(started, tz=timezone.utc).isoformat(),
226 "duration_ms": str(int(duration_ms)),
227 "result": "" if error is not None or result is None else json.dumps(result)[:2000],
228 "error": str(error)[:2000] if error is not None else "",
229 "error_type": type(error).__name__ if error is not None else "",
230 }
231 object_records.create_collection_record(
232 "scheduler_runs", row, base_dir=base_dir, actor="daemon:scheduler"
233 )
234 except Exception:
235 pass # observability must never break the scheduling pass
236
237
238def _calculate_next_run(task, after=None):
239 """Calculate next run time for a task."""
240 schedule = task.get('schedule', '')
241 task_type = task.get('type', '')
242
243 if after is None:
244 after = int(time.time())
245
246 if task_type == 'cron' and croniter:
247 try:
248 cron = croniter(schedule, after)
249 return int(cron.get_next(float))
250 except (ValueError, KeyError):
251 return None
252
253 elif task_type == 'onetime':
254 try:
255 dt = datetime.fromisoformat(schedule.replace('Z', '+00:00'))
256 return int(dt.timestamp())
257 except (ValueError, TypeError):
258 return None
259
260 return None
261
262
263# --- Queue ---
264
265def process_queue(runtime: ObjectRuntime, max_messages=10):
266 """Dequeue messages and execute target objects."""
267 queue_file = _find_trigger_file('queue')
268 if queue_file is None:
269 return
270
271 obj = runtime.load_object(queue_file) # ID = 'queue' (filename stem)
272 obj.state_manager.reload() # Re-read from disk
273 state = obj.state_manager.get_all()
274 now = int(time.time())
275
276 # Find pending, visible messages
277 messages = []
278 for key, value in state.items():
279 if not key.startswith('msg_'):
280 continue
281
282 try:
283 msg = json.loads(value)
284 except (json.JSONDecodeError, TypeError):
285 continue
286
287 if msg.get('status') != 'pending':
288 continue
289
290 # Check expiration. `or`-style defaulting rather than .get(key,
291 # default): object_daemon_control.enqueue_message always WRITES
292 # expires_at, using None to mean "never expires", so the key exists
293 # and .get's default never applied -- every message enqueued through
294 # the official API raised TypeError here and took the whole pass
295 # down with it. The queue was fully built and unusable for exactly
296 # that reason.
297 expires_at = msg.get('expires_at')
298 if expires_at is not None and expires_at < now:
299 msg['status'] = 'expired'
300 obj.state_manager.set(key, json.dumps(msg))
301 continue
302
303 # Check visibility (same defaulting hazard: visible_after is always
304 # written, and a null there means "runnable now").
305 visible_after = msg.get('visible_after')
306 if visible_after is not None and visible_after > now:
307 continue
308
309 messages.append((key, msg))
310
311 # Sort by priority (highest first), then timestamp (oldest first)
312 messages.sort(key=lambda m: (-m[1].get('priority_level', 2), m[1].get('created_at', 0)))
313
314 # Process up to max_messages
315 for key, msg in messages[:max_messages]:
316 body = msg.get('message', {})
317 if not isinstance(body, dict):
318 continue
319
320 target_id = body.get('object_id')
321 if not target_id:
322 continue
323
324 method = body.get('method', 'POST')
325 payload = body.get('payload', {})
326
327 log(f"Queue: executing {target_id}.{method} (msg {msg['id']}, queue {msg.get('queue_name')})")
328
329 # Mark as processing
330 msg['status'] = 'processing'
331 msg['dequeued_at'] = now
332 obj.state_manager.set(key, json.dumps(msg))
333
334 try:
335 _execute_target(runtime, target_id, method, payload)
336 msg['status'] = 'completed'
337 msg['completed_at'] = int(time.time())
338 log(f"Queue: {target_id}.{method} completed")
339 except Exception as e:
340 log(f"Queue: {target_id}.{method} failed: {e}", 'ERROR')
341 msg['attempts'] = msg.get('attempts', 0) + 1
342 if msg['attempts'] >= msg.get('max_attempts', 3):
343 msg['status'] = 'failed'
344 msg['failed_at'] = int(time.time())
345 else:
346 msg['status'] = 'pending'
347 msg['visible_after'] = int(time.time()) + (2 ** msg['attempts'])
348
349 obj.state_manager.set(key, json.dumps(msg))
350
351
352# --- Events ---
353
354def process_events(runtime: ObjectRuntime):
355 """Deliver events to subscribers via callback URLs."""
356 events_file = _find_trigger_file('events')
357 if events_file is None:
358 return
359
360 obj = runtime.load_object(events_file) # ID = 'events' (filename stem)
361 obj.state_manager.reload() # Re-read from disk
362 state = obj.state_manager.get_all()
363
364 # Collect subscriptions and events
365 subscriptions = {}
366 events = {}
367
368 for key, value in state.items():
369 try:
370 data = json.loads(value)
371 except (json.JSONDecodeError, TypeError):
372 continue
373
374 if key.startswith('sub_'):
375 callback = data.get('callback_url')
376 if callback:
377 subscriptions[key] = data
378 elif key.startswith('event_'):
379 events[key] = data
380
381 if not subscriptions or not events:
382 return
383
384 # Sort events by timestamp
385 sorted_events = sorted(events.values(), key=lambda e: e.get('timestamp', 0))
386
387 for sub_key, sub in subscriptions.items():
388 event_type = sub.get('event_type')
389 last_event_id = sub.get('last_event_id')
390 callback_url = sub.get('callback_url')
391
392 # Find events for this subscription
393 matching = [e for e in sorted_events if e.get('event_type') == event_type]
394
395 # Skip events already delivered
396 if last_event_id:
397 found = False
398 pending = []
399 for e in matching:
400 if found:
401 pending.append(e)
402 elif e.get('id') == last_event_id:
403 found = True
404 if not found:
405 # last_event_id not found, deliver all
406 pending = matching
407 else:
408 pending = matching
409
410 if not pending:
411 continue
412
413 # Deliver events
414 for event in pending:
415 status_code = None
416 try:
417 req = urllib.request.Request(
418 callback_url,
419 data=json.dumps(event).encode('utf-8'),
420 headers={'Content-Type': 'application/json'},
421 method='POST',
422 )
423 with urllib.request.urlopen(req, timeout=5) as resp:
424 if hasattr(resp, 'getcode'):
425 status_code = resp.getcode()
426 else:
427 status_code = getattr(resp, 'status', None)
428 log(f"Events: delivered {event['event_type']} ({event['id']}) to {sub['id']}")
429 sub = object_events.record_subscription_delivery(
430 sub,
431 event,
432 success=True,
433 status_code=status_code,
434 )
435 obj.state_manager.set(sub_key, json.dumps(sub))
436 except (urllib.error.URLError, OSError) as e:
437 status_code = getattr(e, 'code', None)
438 log(f"Events: failed to deliver to {callback_url}: {e}", 'WARN')
439 sub = object_events.record_subscription_delivery(
440 sub,
441 event,
442 success=False,
443 status_code=status_code,
444 error=str(e),
445 )
446 obj.state_manager.set(sub_key, json.dumps(sub))
447 break
448
449
450# --- Rate Limit Cleanup ---
451
452def cleanup_ratelimit(max_age=120, *, base_dir: Path | str = "data"):
453 """Delete rate limit files older than max_age seconds."""
454 ratelimit_dir = Path(base_dir) / 'ratelimit'
455 if not ratelimit_dir.exists():
456 return
457
458 now = time.time()
459 cutoff = now - max_age
460 cleaned = 0
461
462 for f in ratelimit_dir.iterdir():
463 if not f.is_file() or not f.name.endswith('.txt'):
464 continue
465 try:
466 # Check if file has any recent timestamps
467 has_recent = False
468 for line in f.read_text().strip().split('\n'):
469 if line:
470 ts = float(line)
471 if ts > cutoff:
472 has_recent = True
473 break
474 if not has_recent:
475 f.unlink()
476 cleaned += 1
477 except (ValueError, IOError, OSError):
478 # Corrupt or unreadable — safe to remove
479 try:
480 f.unlink()
481 cleaned += 1
482 except OSError:
483 pass
484
485 if cleaned:
486 log(f"Cleanup: removed {cleaned} expired rate limit file(s)")
487
488
489def cleanup_events(
490 *,
491 base_dir: Path | str = "data",
492 keep_count: int | None = None,
493 keep_seconds: int | None = None,
494):
495 """Prune old event queue rows while preserving subscriptions."""
496 if keep_count is None:
497 keep_count = _env_int(EVENT_KEEP_COUNT_ENV, object_events.DEFAULT_EVENT_KEEP_COUNT)
498 if keep_seconds is None:
499 keep_seconds = _env_int(EVENT_KEEP_SECONDS_ENV, object_events.DEFAULT_EVENT_KEEP_SECONDS)
500
501 result = object_events.prune_events(
502 base_dir=base_dir,
503 keep_count=keep_count,
504 keep_seconds=keep_seconds,
505 )
506 if result["deleted"]:
507 log(f"Cleanup: removed {result['deleted']} expired event row(s)")
508 return result
509
510
511def _env_int(name: str, default: int) -> int:
512 value = os.environ.get(name, "").strip()
513 if not value:
514 return default
515 try:
516 return int(value)
517 except ValueError:
518 return default
519
520
521def _env_float(name: str, default: float) -> float:
522 value = os.environ.get(name, "").strip()
523 if not value:
524 return default
525 try:
526 return float(value)
527 except ValueError:
528 return default
529
530
531# --- Compaction ---
532
533def process_compactions(*, base_dir: Path | str = "data") -> dict | None:
534 """Compact over-threshold append-mode collections on a timer.
535
536 Runs at most once per DBBASIC_COMPACTION_INTERVAL_SECONDS (default
537 3600 -- an hour), gated by a marker FILE (`.compaction_last_run` under
538 `base_dir`) rather than an in-process variable like the event-cleanup
539 pass above uses: a file marker survives a daemon restart, and if more
540 than one daemon process ever points at the same data dir, whichever
541 one's write lands last simply wins that interval's marker -- there is
542 no harm in an occasional double-run of a compaction that is itself
543 idempotent-per-collection (compact_collection on an already-compacted
544 file is a correctly-reported no-op).
545
546 Every collection whose schema currently declares "storage": "append"
547 (object_records.list_append_collection_stats) is compacted when BOTH:
548 - its physical row count is at or above DBBASIC_APPEND_COMPACT_
549 MIN_ROWS (object_records.append_compact_min_rows -- the same
550 floor object_records._maybe_flag_auto_compact uses for its own
551 inline auto-compact trigger on ordinary writes), AND
552 - its bloat_ratio is at or above DBBASIC_COMPACTION_BLOAT_RATIO
553 (default 1.0 -- dead rows at least matching live rows).
554 A stats entry this call can't get real numbers for (list_append_
555 collection_stats defaults to allow_fold=True, so this is not expected
556 in practice, but a None/estimated physical_rows or bloat_ratio is
557 skipped rather than guessed at) is left alone; it will be reconsidered
558 on the next interval.
559
560 Each collection's compaction is wrapped in its own try/except: one
561 collection failing (a lock contention, a mid-flight schema change, a
562 permissions error) is logged and skipped, never allowed to stop the
563 rest of the pass or propagate out of this function -- the daemon is
564 an optional process (nothing else may depend on it running), and a
565 single bad collection must not take an entire poll interval's worth
566 of compaction down with it. The daemon's own main loop wraps this
567 call in a try/except too, belt and suspenders.
568
569 Returns None when the interval hasn't elapsed yet (no work attempted
570 this call). Otherwise returns {"checked": <int>, "compacted": [{
571 "collection", "rows_before", "rows_after", "bytes_before",
572 "bytes_after"}, ...]} -- mainly for tests and manual/CLI invocation;
573 the daemon's own loop only logs.
574 """
575 interval = _env_int(COMPACTION_INTERVAL_SECONDS_ENV, _DEFAULT_COMPACTION_INTERVAL_SECONDS)
576 marker_path = Path(base_dir) / COMPACTION_MARKER_NAME
577 now = time.time()
578 try:
579 last_run = marker_path.stat().st_mtime
580 except OSError:
581 last_run = 0.0
582
583 if now - last_run < interval:
584 return None
585
586 marker_path.parent.mkdir(parents=True, exist_ok=True)
587 marker_path.write_text(str(now))
588
589 bloat_threshold = _env_float(COMPACTION_BLOAT_RATIO_ENV, _DEFAULT_COMPACTION_BLOAT_RATIO)
590 min_rows = object_records.append_compact_min_rows()
591
592 try:
593 stats = object_records.list_append_collection_stats(base_dir=base_dir)
594 except Exception as e:
595 log(f"Compaction: could not list append collections: {e}", "ERROR")
596 return {"checked": 0, "compacted": []}
597
598 compacted = []
599 for entry in stats:
600 collection = entry.get("collection")
601 try:
602 physical_rows = entry.get("physical_rows")
603 bloat_ratio = entry.get("bloat_ratio")
604 if physical_rows is None or bloat_ratio is None:
605 continue
606 if physical_rows < min_rows or bloat_ratio < bloat_threshold:
607 continue
608
609 summary = object_records.compact_collection(collection, base_dir=base_dir)
610 log(
611 f"Compaction: {collection} rows {summary['rows_before']}->"
612 f"{summary['rows_after']}, bytes {summary['bytes_before']}->"
613 f"{summary['bytes_after']}"
614 )
615 compacted.append({"collection": collection, **summary})
616 except Exception as e:
617 log(f"Compaction: {collection} failed: {e}", "ERROR")
618 continue
619
620 return {"checked": len(stats), "compacted": compacted}
621
622
623# --- Stale-state auto-transition ---
624
625def auto_transition_rules_from_env() -> list[dict]:
626 """Parse DBBASIC_AUTO_TRANSITION_RULES into a list of clean rule dicts.
627
628 Unset -> _DEFAULT_AUTO_TRANSITION_RULES (tasks' waiting_on_client ->
629 approved after 48h). Set to "" or "[]" -> [] (the pass runs every
630 interval, finds nothing configured, and logs a zero-work summary
631 line rather than silently vanishing -- see process_stale_transitions).
632 A malformed entry (missing string field, unparseable after_hours) is
633 dropped with a warning rather than aborting the whole list; a
634 malformed *list* (bad JSON, not a JSON array) disables every rule and
635 is logged loudly, since silently falling back to the default here
636 would hide a real configuration mistake.
637 """
638 raw = os.environ.get(AUTO_TRANSITION_RULES_ENV)
639 if raw is None:
640 raw = _DEFAULT_AUTO_TRANSITION_RULES
641 text = raw.strip()
642 if not text:
643 return []
644
645 try:
646 parsed = json.loads(text)
647 except json.JSONDecodeError as e:
648 log(f"Auto-transition: {AUTO_TRANSITION_RULES_ENV} is not valid JSON, disabling: {e}", "ERROR")
649 return []
650 if not isinstance(parsed, list):
651 log(f"Auto-transition: {AUTO_TRANSITION_RULES_ENV} must be a JSON list, disabling", "ERROR")
652 return []
653
654 rules = []
655 for entry in parsed:
656 if not isinstance(entry, dict):
657 log(f"Auto-transition: skipping non-object rule: {entry!r}", "WARN")
658 continue
659 collection = entry.get("collection")
660 field = entry.get("field")
661 from_value = entry.get("from")
662 to_value = entry.get("to")
663 if not all(isinstance(v, str) and v for v in (collection, field, from_value, to_value)):
664 log(f"Auto-transition: skipping malformed rule (missing collection/field/from/to): {entry!r}", "WARN")
665 continue
666 try:
667 after_hours = float(entry.get("after_hours"))
668 except (TypeError, ValueError):
669 log(f"Auto-transition: skipping rule with invalid after_hours: {entry!r}", "WARN")
670 continue
671 rules.append({
672 "collection": collection,
673 "field": field,
674 "from": from_value,
675 "to": to_value,
676 "after_hours": after_hours,
677 })
678 return rules
679
680
681def _parse_iso_timestamp(value: Any) -> datetime | None:
682 if not isinstance(value, str) or not value:
683 return None
684 try:
685 parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
686 except ValueError:
687 return None
688 if parsed.tzinfo is None:
689 parsed = parsed.replace(tzinfo=timezone.utc)
690 return parsed
691
692
693def _latest_field_entry_timestamps(
694 collection: str, field: str, from_value: str, *, base_dir: Path | str
695) -> dict[str, str]:
696 """Best-effort record id -> timestamp of the most recent change that
697 set `field` to `from_value`, built from ONE sequential read of the
698 collection's record-change log (object_record_changes.CHANGES_FILE).
699
700 This is the most accurate signal available for "how long has this
701 record been sitting in this state": it answers exactly the question
702 the FSM cares about, unlike a record's own `created_at` (when the row
703 was first made, not when it entered THIS state) or a generic
704 `updated_at` (bumped by any field edit, not just this one). It only
705 stays cheap because it is read once per rule per pass -- not once per
706 candidate record -- and the log is append-ordered, so a later entry
707 for the same id simply overwrites an earlier one as this walks the
708 file forward, leaving the most recent one behind.
709
710 Returns {} when the log doesn't exist yet or can't be read; callers
711 fall back to the record's own fields in that case (see
712 _record_age_seconds).
713 """
714 try:
715 path = object_record_changes.record_changes_file(collection, base_dir=base_dir)
716 except object_collections.InvalidCollectionNameError:
717 return {}
718 if not path.exists():
719 return {}
720
721 try:
722 text = path.read_text(encoding="utf-8")
723 except OSError:
724 return {}
725
726 timestamps: dict[str, str] = {}
727 for line in text.splitlines():
728 if not line.strip():
729 continue
730 try:
731 entry = json.loads(line)
732 except json.JSONDecodeError:
733 continue
734 if not isinstance(entry, dict):
735 continue
736 after = entry.get("after")
737 if not isinstance(after, dict) or after.get(field) != from_value:
738 continue
739 record_id = entry.get("record_id")
740 timestamp = entry.get("timestamp")
741 if isinstance(record_id, str) and isinstance(timestamp, str):
742 timestamps[record_id] = timestamp
743 return timestamps
744
745
746def _record_age_seconds(
747 record: dict[str, str],
748 record_id: str,
749 change_timestamps: dict[str, str],
750 *,
751 now: datetime,
752) -> float | None:
753 """Return how long `record` has held its current value, in seconds, or
754 None when no usable timestamp exists at all (caller skips the record
755 rather than guessing).
756
757 Preference order, documented honestly: the record-change log's entry
758 for the change that set this record's field to its current value
759 (most accurate -- see _latest_field_entry_timestamps), then an
760 explicit `updated_at` field on the record itself (cheap, no schema in
761 this codebase declares one yet, but a future one might), then
762 `created_at` (tasks.json's only timestamp field today -- a real but
763 honestly-worse proxy, since it marks row creation, not state entry).
764 """
765 for source in (
766 change_timestamps.get(record_id),
767 record.get("updated_at"),
768 record.get("created_at"),
769 ):
770 parsed = _parse_iso_timestamp(source)
771 if parsed is not None:
772 return (now - parsed).total_seconds()
773 return None
774
775
776def _apply_auto_transition_rule(rule: dict, *, base_dir: Path | str) -> int:
777 """Apply one rule and return the count of records actually moved.
778
779 Every failure mode here is a skip, never an abort: an unknown
780 collection (the owning package isn't installed) is logged once and
781 skipped; a record with no usable timestamp is skipped; a record whose
782 live value has already moved on since it was listed is skipped; and a
783 write rejected by the schema (e.g. a transitions map that no longer
784 allows this move) is logged and skipped, matching process_compactions'
785 per-item isolation posture -- one bad record must never stop the rest
786 of the rule, and one bad rule must never stop the rest of the pass.
787 """
788 collection = rule["collection"]
789 field = rule["field"]
790 from_value = rule["from"]
791 to_value = rule["to"]
792 after_hours = rule["after_hours"]
793
794 try:
795 records = object_records.read_collection_records(collection, base_dir=base_dir)
796 except (object_collections.CollectionNotFoundError, object_collections.InvalidCollectionNameError):
797 if collection not in _WARNED_UNKNOWN_AUTO_TRANSITION_COLLECTIONS:
798 log(
799 f"Auto-transition: collection '{collection}' not found (package not "
800 f"installed?), skipping rule -- will keep checking, logged once",
801 "WARN",
802 )
803 _WARNED_UNKNOWN_AUTO_TRANSITION_COLLECTIONS.add(collection)
804 return 0
805
806 _WARNED_UNKNOWN_AUTO_TRANSITION_COLLECTIONS.discard(collection)
807
808 candidates = [r for r in records if r.get(field) == from_value]
809 if not candidates:
810 return 0
811
812 change_timestamps = _latest_field_entry_timestamps(collection, field, from_value, base_dir=base_dir)
813 cutoff_seconds = after_hours * 3600.0
814 now = datetime.now(timezone.utc)
815
816 moved = 0
817 for record in candidates:
818 record_id = record.get("id")
819 if not record_id:
820 continue
821
822 age_seconds = _record_age_seconds(record, record_id, change_timestamps, now=now)
823 if age_seconds is None or age_seconds < cutoff_seconds:
824 continue
825
826 try:
827 current = object_records.get_collection_record(collection, record_id, base_dir=base_dir)
828 except object_records.RecordNotFoundError:
829 continue # deleted since the listing above
830 if current.get(field) != from_value:
831 continue # moved on already -- someone/something got there first
832
833 try:
834 object_records.update_collection_record(
835 collection, record_id, {field: to_value},
836 base_dir=base_dir, actor=AUTO_TRANSITION_ACTOR,
837 )
838 moved += 1
839 except Exception as e:
840 log(
841 f"Auto-transition: {collection}/{record_id} {field} "
842 f"'{from_value}'->'{to_value}' failed: {e}",
843 "ERROR",
844 )
845 continue
846
847 return moved
848
849
850def process_stale_transitions(*, base_dir: Path | str = "data") -> dict | None:
851 """Auto-transition records that have sat in one FSM state too long.
852
853 Runs at most once per DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS
854 (default 3600) gated by a marker FILE (`.auto_transition_last_run`
855 under `base_dir`), the same pattern process_compactions above uses
856 and for the same reason: a file marker survives a daemon restart, so
857 the 48-hour clock this pass exists to enforce is never quietly reset
858 by a deploy.
859
860 Rules come from DBBASIC_AUTO_TRANSITION_RULES (see
861 auto_transition_rules_from_env), a compact JSON list of
862 {"collection", "field", "from", "to", "after_hours"}. This pass ships
863 ON by default -- the empty-env case still resolves to a real rule
864 (tasks' waiting_on_client -> approved after 48h) -- because the
865 predecessor system's version of this job was written but never
866 scheduled, and 200+ records sat stranded as a result. Setting the env
867 to "" or "[]" is how an operator opts back out.
868
869 Every rule is applied independently and wrapped in its own try/except
870 (_apply_auto_transition_rule already isolates per-record failures;
871 this is belt and suspenders against a rule failing before it gets
872 that far, e.g. a read_collection_records call raising something other
873 than the two "not installed" exceptions it explicitly handles) --
874 one bad rule is logged and skipped, never allowed to stop the rest of
875 the pass.
876
877 Always logs exactly one summary line per run (even a no-op run, with
878 zero rules configured or zero records moved) so the pass's activity
879 -- or deliberate silence -- is visible in the daemon's own log rather
880 than something that has to be inferred. Returns None when the
881 interval hasn't elapsed yet (no work attempted this call). Otherwise
882 returns {"checked": <rule count>, "moved": <total records moved>,
883 "rules": [{"collection", "field", "from", "to", "moved"}, ...]} --
884 mainly for tests and manual/CLI invocation, matching
885 process_compactions' contract.
886 """
887 interval = _env_int(AUTO_TRANSITION_INTERVAL_SECONDS_ENV, _DEFAULT_AUTO_TRANSITION_INTERVAL_SECONDS)
888 marker_path = Path(base_dir) / AUTO_TRANSITION_MARKER_NAME
889 now = time.time()
890 try:
891 last_run = marker_path.stat().st_mtime
892 except OSError:
893 last_run = 0.0
894
895 if now - last_run < interval:
896 return None
897
898 marker_path.parent.mkdir(parents=True, exist_ok=True)
899 marker_path.write_text(str(now))
900
901 rules = auto_transition_rules_from_env()
902 rule_results = []
903 total_moved = 0
904 for rule in rules:
905 try:
906 moved = _apply_auto_transition_rule(rule, base_dir=base_dir)
907 except Exception as e:
908 log(
909 f"Auto-transition: rule {rule['collection']}.{rule['field']} "
910 f"'{rule['from']}'->'{rule['to']}' failed: {e}",
911 "ERROR",
912 )
913 moved = 0
914 total_moved += moved
915 rule_results.append({
916 "collection": rule["collection"],
917 "field": rule["field"],
918 "from": rule["from"],
919 "to": rule["to"],
920 "moved": moved,
921 })
922
923 log(f"Auto-transition: checked {len(rules)} rule(s), moved {total_moved} record(s)")
924 return {"checked": len(rules), "moved": total_moved, "rules": rule_results}
925
926
927# --- Rollups ---
928
929def process_rollups(*, base_dir: Path | str = "data") -> dict | None:
930 """Recompute every due, enabled rollup_definitions row.
931
932 Rides this daemon's existing poll loop as one more pass
933 (plan/vocabulary/14-rollup-spec.md's Dependencies: "No new daemon...
934 process_rollups, alongside process_compactions"), but its due-gate is
935 NOT the marker-file pattern process_compactions/process_stale_
936 transitions use above: 14 is explicit that a rollup_definitions row
937 already IS the natural place to hold that state (last_computed_at),
938 unlike compaction's collection-wide pass, which has no equivalent
939 per-collection record. So there's no marker file here and no fixed
940 poll interval either -- each ENABLED definition is checked against
941 its own last_computed_at/refresh_interval_seconds every call
942 (object_rollups.is_definition_due), which is cheap (a timestamp
943 comparison) for every definition that isn't due yet.
944
945 Returns None when the block-wide `rollup_enabled` flag is off (14's
946 Degradation: "the daemon's rollup pass is skipped entirely; existing
947 target collections keep serving their last-computed data") or when
948 the rollup_definitions collection doesn't exist yet (package not
949 installed). Otherwise returns {"checked": <definitions considered>,
950 "computed": [{"definition_id", "target_collection", "groups",
951 "suppressed", ...}, ...]} -- computed only lists definitions that
952 were actually due and enabled; a skipped/disabled/not-yet-due
953 definition doesn't appear.
954
955 Every definition's recompute is wrapped in its own try/except, same
956 isolation pattern as process_compactions: one definition with a bad
957 filter, a missing source collection, or a lock contention is logged
958 and skipped, every other definition's pass proceeds.
959 """
960 if not object_rollups.rollup_pass_enabled(base_dir=base_dir):
961 return None
962
963 try:
964 definitions = object_records.read_collection_records(
965 object_rollups.ROLLUP_DEFINITIONS_COLLECTION, base_dir=base_dir
966 )
967 except (object_collections.CollectionNotFoundError, object_collections.InvalidCollectionNameError):
968 return None
969
970 now = datetime.now(timezone.utc)
971 computed = []
972 for definition in definitions:
973 definition_id = definition.get("id") or "<unknown>"
974
975 if not object_rollups.is_definition_enabled(definition):
976 continue
977 if not object_rollups.is_definition_due(definition, now=now):
978 continue
979
980 try:
981 result = object_rollups.compute_rollup(definition, base_dir=base_dir)
982 except Exception as e:
983 log(f"Rollup: {definition_id} failed: {e}", "ERROR")
984 continue
985
986 try:
987 object_records.update_collection_record(
988 object_rollups.ROLLUP_DEFINITIONS_COLLECTION,
989 definition_id,
990 {"last_computed_at": result["computed_at"]},
991 base_dir=base_dir,
992 actor=object_rollups.ROLLUP_ACTOR,
993 preserve_read_only=True,
994 )
995 except Exception as e:
996 # The recompute itself already succeeded and is live in the
997 # target collection; failing to stamp last_computed_at only
998 # means this definition looks "due" again next call (a
999 # harmless, self-correcting re-run), never lost or corrupted
1000 # data -- logged, not fatal to the rest of the pass.
1001 log(f"Rollup: {definition_id} computed but failed to stamp last_computed_at: {e}", "ERROR")
1002
1003 log(
1004 f"Rollup: {definition_id} -> {result['target_collection']} "
1005 f"({result['groups']} group(s), {result['suppressed']} suppressed)"
1006 )
1007 computed.append(result)
1008
1009 return {"checked": len(definitions), "computed": computed}
1010
1011
1012# --- Materialize ---
1013
1014def process_materializations(*, base_dir: Path | str = "data") -> dict | None:
1015 """Generate every due, enabled, non-blocked scheduled/scheduled_fixed
1016 materialize_definitions row's output.
1017
1018 Rides this daemon's existing poll loop as one more pass
1019 (plan/vocabulary/61-materialize-spec.md's Dependencies: "No new
1020 daemon... process_materializations, alongside process_compactions").
1021 Unlike process_rollups (whose due-gate lives entirely on the
1022 definition row, no marker file needed), 61's Events section asks for
1023 process_stale_transitions' marker-file-gated shape instead -- gated by
1024 DBBASIC_MATERIALIZE_INTERVAL_SECONDS (default 3600), marker file
1025 `.materialize_last_run` under `base_dir`, surviving a daemon restart
1026 the same way every other marker-gated pass does.
1027
1028 Event-mode definitions (trigger.mode == "event") are never driven by
1029 this pass -- in v1 they run via materialize_run's manual path only
1030 (degrade-to-manual, per 61's Degradation section). The automatic
1031 on-create dispatch (materialize_seed's HANDLES) is deliberately NOT
1032 auto-wired: this pass does not rewrite materialize_seed's source to
1033 track the definition set. Rewriting an installed object at runtime --
1034 and doing it every poll -- is the wrong shape (expensive, surprising,
1035 self-modifying code); if auto-synced HANDLES is ever wanted it belongs
1036 in a deliberate scheduled job with its own tests, not the poll loop.
1037 materialize_seed ships inert (HANDLES == []); the event-dispatch logic
1038 it would call still exists and is tested, ready for that future
1039 mechanism or a manual wiring.
1040
1041 For each enabled, non-blocked, due scheduled/scheduled_fixed
1042 definition: object_materialize.generate_definition reads
1043 source_collection outside row-filters (daemon posture, same as
1044 compaction/rollup/auto-transition), computes the due (source row,
1045 period) set from first principles every call (never trusting a
1046 row's own next_run/last_run as the correctness mechanism -- see that
1047 module's docstring), and for each checks the deterministic header id
1048 in output_collection: missing -> generate, present -> skip, not an
1049 error.
1050
1051 Two-level isolation, per 61's Events section (drawn from process_
1052 stale_transitions' _apply_auto_transition_rule, not process_rollups'
1053 per-definition-only isolation): one bad DEFINITION (a malformed
1054 mapping, a missing source/output/child collection) is caught here and
1055 never stops any other definition; one bad SOURCE ROW within an
1056 otherwise-good definition is caught inside object_materialize.
1057 generate_config and surfaces as one entry in that definition's own
1058 "errors" list, never stopping any other row in the same definition.
1059
1060 A missing source/output/child collection is logged once per process
1061 (object_materialize.MissingCollectionError, warned-once via
1062 _WARNED_UNKNOWN_MATERIALIZE_COLLECTIONS) -- mirrors process_stale_
1063 transitions' _WARNED_UNKNOWN_AUTO_TRANSITION_COLLECTIONS exactly, to
1064 avoid re-logging every poll interval.
1065
1066 Always logs exactly one summary line per run, even a no-op run
1067 (`"Materialize: checked N definition(s), generated M record(s),
1068 skipped K already-generated"`) -- every sibling pass's "activity or
1069 deliberate silence is visible in the daemon log" convention.
1070
1071 Returns None when the block-wide `materialize_enabled` flag is off,
1072 the materialize_definitions collection doesn't exist yet (package not
1073 installed), or the interval hasn't elapsed. Otherwise returns
1074 {"checked": <definitions considered>, "generated": <total records>,
1075 "skipped_already_generated": <total>, "results": [...]}.
1076 """
1077 if not object_materialize.materialize_pass_enabled(base_dir=base_dir):
1078 return None
1079
1080 interval = _env_int(MATERIALIZE_INTERVAL_SECONDS_ENV, _DEFAULT_MATERIALIZE_INTERVAL_SECONDS)
1081 marker_path = Path(base_dir) / MATERIALIZE_MARKER_NAME
1082 now_ts = time.time()
1083 try:
1084 last_run = marker_path.stat().st_mtime
1085 except OSError:
1086 last_run = 0.0
1087
1088 if now_ts - last_run < interval:
1089 return None
1090
1091 try:
1092 definitions = object_records.read_collection_records(
1093 object_materialize.MATERIALIZE_DEFINITIONS_COLLECTION, base_dir=base_dir
1094 )
1095 except (object_collections.CollectionNotFoundError, object_collections.InvalidCollectionNameError):
1096 return None
1097
1098 marker_path.parent.mkdir(parents=True, exist_ok=True)
1099 marker_path.write_text(str(now_ts))
1100
1101 now = datetime.now(timezone.utc)
1102 total_generated = 0
1103 total_skipped = 0
1104 results = []
1105
1106 for definition in definitions:
1107 definition_id = definition.get("id") or "<unknown>"
1108
1109 if object_materialize.is_definition_blocked(definition):
1110 continue
1111 if not object_materialize.is_definition_enabled(definition):
1112 continue
1113 if not object_materialize.is_definition_due(definition, now=now):
1114 continue
1115
1116 try:
1117 config = object_materialize.parse_definition(definition, base_dir=base_dir)
1118 except object_materialize.MissingCollectionError as e:
1119 if definition_id not in _WARNED_UNKNOWN_MATERIALIZE_COLLECTIONS:
1120 log(
1121 f"Materialize: {definition_id} references missing {e.role} collection "
1122 f"'{e.collection}' (package not installed?), skipping -- will keep "
1123 "checking, logged once",
1124 "WARN",
1125 )
1126 _WARNED_UNKNOWN_MATERIALIZE_COLLECTIONS.add(definition_id)
1127 continue
1128 except Exception as e:
1129 log(f"Materialize: {definition_id} failed: {e}", "ERROR")
1130 continue
1131
1132 if config.trigger_mode == "event":
1133 continue # event-mode definitions are driven by dispatch/manual only
1134
1135 _WARNED_UNKNOWN_MATERIALIZE_COLLECTIONS.discard(definition_id)
1136
1137 try:
1138 result = object_materialize.generate_config(config, base_dir=base_dir, now=now)
1139 except Exception as e:
1140 log(f"Materialize: {definition_id} failed: {e}", "ERROR")
1141 continue
1142
1143 for error in result["errors"]:
1144 log(
1145 f"Materialize: {definition_id} source={error['source_id']} "
1146 f"period={error['period_start']} failed: {error['error']}",
1147 "ERROR",
1148 )
1149
1150 try:
1151 object_records.update_collection_record(
1152 object_materialize.MATERIALIZE_DEFINITIONS_COLLECTION,
1153 definition_id,
1154 {"last_run_at": _now_iso_utc()},
1155 base_dir=base_dir,
1156 actor=config.actor,
1157 preserve_read_only=True,
1158 )
1159 except Exception as e:
1160 # The generation itself already succeeded (whatever was due is
1161 # live in output_collection); failing to stamp last_run_at only
1162 # means this definition looks "due" again next call -- a
1163 # harmless, self-correcting re-run (the deterministic header id
1164 # is still the real gate), never lost or corrupted data.
1165 log(f"Materialize: {definition_id} ran but failed to stamp last_run_at: {e}", "ERROR")
1166
1167 total_generated += result["generated"]
1168 total_skipped += result["skipped_already_generated"]
1169 results.append(result)
1170
1171 log(
1172 f"Materialize: checked {len(definitions)} definition(s), generated "
1173 f"{total_generated} record(s), skipped {total_skipped} already-generated"
1174 )
1175 return {
1176 "checked": len(definitions),
1177 "generated": total_generated,
1178 "skipped_already_generated": total_skipped,
1179 "results": results,
1180 }
1181
1182
1183def _now_iso_utc() -> str:
1184 return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
1185
1186
1187# --- Object Execution ---
1188
1189def _execute_target(runtime: ObjectRuntime, object_id: str, method: str, payload: dict):
1190 """Load and execute a target object."""
1191 request = ObjectExecutionRequest(object_id=object_id, method=method, payload=payload)
1192 result = execute_object(runtime, request)
1193 if not result.ok:
1194 if result.error and result.error.type == "ObjectNotFoundError":
1195 raise FileNotFoundError(result.error.message)
1196 raise ObjectExecutionFailure(result)
1197 return result.result
1198
1199
1200def _find_object_file(object_id: str) -> Path | None:
1201 """Find the .py file for an object ID."""
1202 return resolve_object_id(object_id)
1203
1204
1205def process_notifications(*, base_dir: Path | str = "data") -> dict | None:
1206 """12 notify: turn new record-change events into notifications.
1207
1208 Polls the record-change log rather than riding synchronous HANDLES
1209 dispatch -- HANDLES is gated behind DBBASIC_ENABLE_EVENT_HANDLERS (off in
1210 prod) and we deliberately don't rewrite installed objects to track dynamic
1211 event sets (see object_notify's docstring). Reads each watched collection's
1212 change entries newer than the cursor, matches them against enabled
1213 notify_rules (object_notify), and appends a notifications row per resolved
1214 recipient. At-least-once: a crash between writing a notification and
1215 advancing the cursor re-notifies, the delivery bar 12/01 accept.
1216
1217 Returns None when notify is disabled or there are no rules; otherwise
1218 {"changes": <processed>, "notifications": <written>}.
1219 """
1220 if not object_notify.notify_pass_enabled(base_dir=base_dir):
1221 return None
1222 try:
1223 rules = object_records.read_collection_records(
1224 object_notify.NOTIFY_RULES_COLLECTION, base_dir=base_dir
1225 )
1226 except (object_collections.CollectionNotFoundError, object_collections.InvalidCollectionNameError):
1227 return None
1228 enabled = [r for r in rules if str(r.get("enabled", "true")).strip().lower() not in {"off", "false", "0", "no"}]
1229 if not enabled:
1230 return None
1231
1232 cursor_path = Path(base_dir) / NOTIFY_CURSOR_NAME
1233 now_iso = datetime.now(timezone.utc).isoformat()
1234 try:
1235 cursor = cursor_path.read_text().strip()
1236 except OSError:
1237 cursor = ""
1238 if not cursor:
1239 # First run: stamp at now, notify only on FUTURE changes (never
1240 # backfill the entire history).
1241 cursor_path.parent.mkdir(parents=True, exist_ok=True)
1242 cursor_path.write_text(now_iso)
1243 return None
1244
1245 try:
1246 known = {c.get("name") for c in object_collections.list_collections(base_dir=base_dir) if c.get("name")}
1247 except (OSError, ValueError):
1248 known = set()
1249 watched = object_notify.watched_collections(enabled, known)
1250
1251 # Gather every change newer than the cursor, across watched collections.
1252 fresh: list[dict] = []
1253 for collection in watched:
1254 try:
1255 payload = object_record_changes.list_record_changes(collection, base_dir=base_dir, limit=1000)
1256 except (object_collections.CollectionNotFoundError, object_collections.InvalidCollectionNameError, OSError, ValueError):
1257 continue
1258 for change in payload.get("changes") or []:
1259 ts = str(change.get("timestamp") or "")
1260 if ts > cursor:
1261 fresh.append(change)
1262 fresh.sort(key=lambda c: str(c.get("timestamp") or ""))
1263
1264 written = 0
1265 emailed = 0
1266 max_ts = cursor
1267 for change in fresh:
1268 # Per-change try/except: one malformed rule or change never stops the
1269 # rest, and the cursor still advances past it (never re-storming).
1270 try:
1271 for rule in enabled:
1272 for note in object_notify.notifications_for_change(rule, change, base_dir=base_dir):
1273 object_records.create_collection_record(
1274 object_notify.NOTIFICATIONS_COLLECTION, note,
1275 base_dir=base_dir, actor=object_notify.DEFAULT_ACTOR,
1276 )
1277 written += 1
1278 # The email channel rides the GENERIC 01 outbox (object_email),
1279 # never the private mail package -- notify depends only on the
1280 # open adapter; whatever SMTP relay is configured delivers it.
1281 for intent in object_notify.email_intents_for_change(rule, change, base_dir=base_dir):
1282 if _enqueue_notify_email(intent, base_dir=base_dir):
1283 emailed += 1
1284 except Exception as exc: # noqa: BLE001 -- isolate one bad change
1285 log(f"Notify: change {change.get('change_id')} failed: {exc}", "ERROR")
1286 ts = str(change.get("timestamp") or "")
1287 if ts > max_ts:
1288 max_ts = ts
1289
1290 if max_ts != cursor:
1291 cursor_path.write_text(max_ts)
1292 if fresh or written or emailed:
1293 log(f"Notify: processed {len(fresh)} change(s), wrote {written} notification(s), queued {emailed} email(s)")
1294 return {"changes": len(fresh), "notifications": written, "emails": emailed}
1295
1296
1297def _enqueue_notify_email(intent: dict, *, base_dir: Path | str) -> bool:
1298 """Map a notify email intent's user_id to an address and enqueue it into the
1299 generic 01 outbox. Returns False (skips) when the user has no address on
1300 file or the email_outbox isn't installed -- email is best-effort, never
1301 blocks the in_app write."""
1302 try:
1303 user = object_identity.get_user(intent["user_id"], base_dir=base_dir)
1304 except (object_identity.UserNotFoundError, OSError, ValueError):
1305 return False
1306 address = (user.get("email") or "").strip()
1307 if not address:
1308 return False
1309 try:
1310 object_email.enqueue(
1311 address, intent.get("subject") or "", intent.get("body") or "",
1312 source_object_id="notify",
1313 extra={"user_id": intent["user_id"], "target": intent.get("target", "")},
1314 base_dir=base_dir,
1315 )
1316 except (object_collections.CollectionNotFoundError,
1317 object_collections.InvalidCollectionNameError):
1318 return False # app-email not installed -> email delivery unavailable
1319 return True
1320
1321
1322# 01 email adapter: rolling per-minute send-rate window, persisted as JSON
1323# ({window_start, sent_count}) so it survives a daemon restart -- the same
1324# "cheap state outside the collection" shape process_compactions' marker uses.
1325EMAIL_RATE_MARKER_NAME = ".email_rate_window"
1326# Whether we've already logged "SMTP not configured" this process. The pass
1327# runs every tick (it's one cheap collection read) but must not repeat the
1328# line every poll -- matching process_scheduler/process_queue when their own
1329# trigger object is absent.
1330_EMAIL_UNCONFIGURED_WARNED = False
1331
1332
1333def _read_email_rate_window(base_dir: Path | str) -> dict | None:
1334 try:
1335 return json.loads((Path(base_dir) / EMAIL_RATE_MARKER_NAME).read_text())
1336 except (OSError, ValueError):
1337 return None
1338
1339
1340def _write_email_rate_window(base_dir: Path | str, window: dict) -> None:
1341 path = Path(base_dir) / EMAIL_RATE_MARKER_NAME
1342 path.parent.mkdir(parents=True, exist_ok=True)
1343 path.write_text(json.dumps(window))
1344
1345
1346def process_email_outbox(*, base_dir: Path | str = "data") -> dict | None:
1347 """01 email adapter: drain the email_outbox queue.
1348
1349 Every tick, delivers the `queued` rows that are due (`next_attempt_at`
1350 arrived) up to DBBASIC_SMTP_BATCH_SIZE, bounded by a rolling per-minute
1351 rate window. Each send's outcome is written back onto its row: `sent`, or
1352 `queued` again with a backed-off `next_attempt_at` and `last_error` (a
1353 transient failure), or `dead` (a permanent SMTP error, or retries
1354 exhausted). Per-message try/except -- one bad send never blocks the batch,
1355 matching process_compactions' per-item isolation.
1356
1357 Returns None when email is disabled (feature flag), unconfigured (no SMTP
1358 mode / host -- everything just keeps queuing, fully inspectable), or the
1359 email_outbox collection isn't installed. Otherwise a small summary dict.
1360 """
1361 global _EMAIL_UNCONFIGURED_WARNED
1362 if not object_email.email_pass_enabled(base_dir=base_dir):
1363 return None
1364
1365 config = object_email.smtp_config_from_env()
1366 if not config.configured:
1367 if not _EMAIL_UNCONFIGURED_WARNED:
1368 log("Email: SMTP not configured (DBBASIC_SMTP_MODE) -- outbound mail is queuing only")
1369 _EMAIL_UNCONFIGURED_WARNED = True
1370 return None
1371 _EMAIL_UNCONFIGURED_WARNED = False # configured again -> re-warn if it flips back
1372
1373 try:
1374 records = object_records.read_collection_records(
1375 object_email.OUTBOX_COLLECTION, base_dir=base_dir
1376 )
1377 except (object_collections.CollectionNotFoundError,
1378 object_collections.InvalidCollectionNameError, OSError, ValueError):
1379 return None
1380
1381 now = datetime.now(timezone.utc)
1382 due = [r for r in records if object_email.is_due(r, now)]
1383 if not due:
1384 return {"attempted": 0, "sent": 0, "dead": 0}
1385 # Oldest-scheduled first, so a backlog drains fairly rather than starving
1386 # the earliest-queued behind newer arrivals.
1387 due.sort(key=lambda r: (str(r.get("next_attempt_at") or ""), str(r.get("created_at") or "")))
1388
1389 window = object_email.rate_window_reset(_read_email_rate_window(base_dir), time.time())
1390 sender = object_email.sender_for(config)
1391 attempted = sent = dead = rate_limited = 0
1392
1393 for record in due[:config.batch_size]:
1394 if window["sent_count"] >= config.rate_limit:
1395 rate_limited += 1
1396 continue # left `queued`; a scheduling skip is not a delivery attempt
1397 try:
1398 update = object_email.attempt_delivery(record, config, sender=sender, now=now)
1399 except Exception as exc: # noqa: BLE001 -- isolate one bad message
1400 log(f"Email: message {record.get('id')} delivery raised: {exc}", "ERROR")
1401 continue
1402 window["sent_count"] += 1 # a network attempt was made (success or not)
1403 attempted += 1
1404 try:
1405 object_records.update_collection_record(
1406 object_email.OUTBOX_COLLECTION, record["id"], update,
1407 base_dir=base_dir, actor=object_email.DEFAULT_ACTOR,
1408 preserve_read_only=True,
1409 )
1410 except Exception as exc: # noqa: BLE001
1411 # The send already happened; failing to record the outcome only
1412 # risks an at-least-once re-send next tick -- the delivery bar 01
1413 # accepts, never lost data.
1414 log(f"Email: message {record.get('id')} sent but status write failed: {exc}", "ERROR")
1415 if update.get("status") == object_email.STATUS_SENT:
1416 sent += 1
1417 elif update.get("status") == object_email.STATUS_DEAD:
1418 dead += 1
1419
1420 _write_email_rate_window(base_dir, window)
1421 if attempted or rate_limited:
1422 extra = f", {rate_limited} rate-limited" if rate_limited else ""
1423 log(f"Email: attempted {attempted} ({sent} sent, {dead} dead){extra}")
1424 return {"attempted": attempted, "sent": sent, "dead": dead, "rate_limited": rate_limited}
1425
1426
1427CHANGE_LOG_RETENTION_MARKER_NAME = ".change_log_retention_last_run"
1428CHANGE_LOG_RETENTION_INTERVAL_ENV = "DBBASIC_CHANGE_LOG_RETENTION_INTERVAL_SECONDS"
1429_DEFAULT_CHANGE_LOG_RETENTION_INTERVAL = 21600 # 6h -- rewriting every log is not cheap
1430
1431
1432def process_change_log_retention(*, base_dir: Path | str = "data") -> dict | None:
1433 """Rotate, archive and (only if asked) remove record change logs.
1434
1435 Returns None when no policy is configured or the pass is not due.
1436 Off unless an operator sets a retention window or an entry cap: a
1437 change log IS the audit trail, and deciding how much of it to keep is
1438 not a decision a daemon makes on somebody's behalf.
1439
1440 When it IS configured, expired entries are ARCHIVED by default rather
1441 than deleted -- rotated into a timestamped gzip segment beside the log
1442 (object_record_changes.prune_all_record_changes). Somebody asking for
1443 a smaller hot file has not asked to lose the history, and the hot file
1444 is the only part that costs anything: it is what every feed render
1445 reads. On the box that motivated this, 1.1GB of logs became 22MB hot
1446 plus a 115MB archive with all 1.7 million entries still in it.
1447
1448 Runs on a 6h marker, the same shape as the analytics retention pass,
1449 because rewriting every collection's log is not something to do on a
1450 tick -- the first live sweep took three minutes.
1451 """
1452 policy = object_record_changes.retention_policy()
1453 if not policy["days"] and not policy["max_entries"]:
1454 return None
1455
1456 marker_path = Path(base_dir) / CHANGE_LOG_RETENTION_MARKER_NAME
1457 interval = _env_int(CHANGE_LOG_RETENTION_INTERVAL_ENV,
1458 _DEFAULT_CHANGE_LOG_RETENTION_INTERVAL)
1459 now = time.time()
1460 try:
1461 if now - marker_path.stat().st_mtime < interval:
1462 return None
1463 except OSError:
1464 pass # no marker yet -> run now
1465 marker_path.parent.mkdir(parents=True, exist_ok=True)
1466 marker_path.write_text(str(now))
1467
1468 try:
1469 result = object_record_changes.prune_all_record_changes(
1470 base_dir=base_dir,
1471 days=policy["days"],
1472 max_entries=policy["max_entries"],
1473 archive=policy["archive"],
1474 keep_archives=policy["keep_archives"],
1475 )
1476 except OSError:
1477 return None
1478 if result.get("removed"):
1479 verb = "archived" if policy["archive"] else "removed"
1480 log(f"Change logs: {verb} {result['removed']} entr(ies) across "
1481 f"{result['collections']} log(s)")
1482 return result
1483
1484
1485RESTORE_POINT_RETENTION_MARKER_NAME = ".restore_point_retention_last_run"
1486RESTORE_POINT_RETENTION_INTERVAL_ENV = "DBBASIC_RESTORE_POINT_RETENTION_INTERVAL_SECONDS"
1487RESTORE_POINT_KEEP_LAST_ENV = "DBBASIC_RESTORE_POINT_KEEP_LAST"
1488RESTORE_POINT_KEEP_DAYS_ENV = "DBBASIC_RESTORE_POINT_KEEP_DAYS"
1489RESTORE_POINT_MAX_BYTES_ENV = "DBBASIC_RESTORE_POINT_MAX_BYTES"
1490_DEFAULT_RESTORE_POINT_INTERVAL = 21600 # 6h, like the other sweeps
1491_DEFAULT_RESTORE_POINT_KEEP_LAST = 20
1492_DEFAULT_RESTORE_POINT_KEEP_DAYS = 7
1493
1494
1495def process_restore_point_retention(*, base_dir: Path | str = "data") -> dict | None:
1496 """Sweep AUTOMATIC restore points. Never touches a manual backup.
1497
1498 ON BY DEFAULT, which is the opposite of the change-log pass above, and
1499 the difference is who asked for the files. A change log IS the audit
1500 trail, so how much to keep is not a decision a daemon makes on
1501 somebody's behalf. Restore points are scaffolding the server writes for
1502 itself before every package install -- nobody asked for them, and the
1503 hundredth-oldest one from three weeks ago is worth nothing.
1504
1505 The box that motivated this had 157 of them at ~91MB, 14.3GB against
1506 4.0GB free, on a machine also serving production, with 93 written in
1507 the preceding two days. Left alone it fills the disk in about a day,
1508 and a full disk on that box takes production down with it. That is the
1509 fourth time this shape of bug has appeared here (page_views, restore
1510 points, change logs, restore points again).
1511
1512 Keeps the newest DBBASIC_RESTORE_POINT_KEEP_LAST (default 20) and
1513 anything younger than DBBASIC_RESTORE_POINT_KEEP_DAYS (default 7), so
1514 a burst of installs in one afternoon cannot evict the morning's safety
1515 net. Setting KEEP_LAST to 0 does NOT disable the sweep -- the floor is
1516 object_backup_index.MIN_KEEP -- because sweeping to the bone would
1517 remove the very safety net this feature exists to provide.
1518
1519 The BUDGET (DBBASIC_RESTORE_POINT_MAX_BYTES, default 2GB) is what
1520 actually bounds the disk, and it outranks the window. A first cut of
1521 this pass had only a count and an age window; a dry run on the real box
1522 freed 78MB of 14.3GB, because 93 archives were two days old and the
1523 window protected them all. Count and age cannot bound bytes.
1524 """
1525 marker_path = Path(base_dir) / RESTORE_POINT_RETENTION_MARKER_NAME
1526 interval = _env_int(RESTORE_POINT_RETENTION_INTERVAL_ENV,
1527 _DEFAULT_RESTORE_POINT_INTERVAL)
1528 now = time.time()
1529 try:
1530 if now - marker_path.stat().st_mtime < interval:
1531 return None
1532 except OSError:
1533 pass # no marker yet -> run now
1534 marker_path.parent.mkdir(parents=True, exist_ok=True)
1535 marker_path.write_text(str(now))
1536
1537 try:
1538 result = object_backup_index.prune_backups(
1539 data_dir=base_dir,
1540 keep_last=_env_int(RESTORE_POINT_KEEP_LAST_ENV,
1541 _DEFAULT_RESTORE_POINT_KEEP_LAST),
1542 keep_newer_than_days=_env_int(RESTORE_POINT_KEEP_DAYS_ENV,
1543 _DEFAULT_RESTORE_POINT_KEEP_DAYS),
1544 max_total_bytes=_env_int(
1545 RESTORE_POINT_MAX_BYTES_ENV,
1546 object_backup_index.DEFAULT_MAX_TOTAL_BYTES),
1547 )
1548 except OSError:
1549 return None
1550 if result.get("removed"):
1551 freed_mb = int(result["freed_bytes"]) // (1024 * 1024)
1552 log(f"Restore points: removed {result['removed']} archive(s), "
1553 f"freed {freed_mb}MB, kept {result['kept']} "
1554 f"(+{result['protected']} manual, never swept)")
1555 for problem in result.get("errors") or []:
1556 log(f"Restore points: could not remove {problem}", "ERROR")
1557 return result
1558
1559
1560ANALYTICS_RETENTION_MARKER_NAME = ".analytics_retention_last_run"
1561ANALYTICS_RETENTION_INTERVAL_ENV = "DBBASIC_ANALYTICS_RETENTION_INTERVAL_SECONDS"
1562_DEFAULT_ANALYTICS_RETENTION_INTERVAL = 21600 # 6h -- a rewrite of a big file isn't cheap
1563
1564
1565def process_analytics_retention(*, base_dir: Path | str = "data") -> dict | None:
1566 """Age page_views rows out past the retention window, bounding the file.
1567
1568 page_views is the platform's heaviest write path and a pure event log --
1569 id-fold compaction can't shrink it (nothing is superseded), so retention is
1570 a time-windowed rewrite (object_records.prune_collection_records). Gated by
1571 a marker file + interval (default 6h), the same shape process_compactions
1572 uses -- a rewrite of a massive file is not something to do every tick.
1573
1574 Returns None when disabled/not-due/not-installed; else the prune summary.
1575 """
1576 if not object_analytics.analytics_enabled():
1577 return None
1578
1579 marker_path = Path(base_dir) / ANALYTICS_RETENTION_MARKER_NAME
1580 interval = _env_int(ANALYTICS_RETENTION_INTERVAL_ENV, _DEFAULT_ANALYTICS_RETENTION_INTERVAL)
1581 now = time.time()
1582 try:
1583 if now - marker_path.stat().st_mtime < interval:
1584 return None
1585 except OSError:
1586 pass # no marker yet -> run now
1587 marker_path.parent.mkdir(parents=True, exist_ok=True)
1588 marker_path.write_text(str(now))
1589
1590 days = object_analytics.retention_days()
1591 cutoff = (datetime.now(timezone.utc) - timedelta(days=days)).isoformat().replace("+00:00", "Z")
1592 cap = object_analytics.max_rows()
1593 try:
1594 result = object_records.prune_collection_records(
1595 object_analytics.PAGE_VIEWS_COLLECTION,
1596 keep_newer_than=cutoff, timestamp_field="created_at",
1597 # Two bounds, because a retention window says how far back to
1598 # keep and nothing at all about how much can arrive inside it.
1599 keep_last=(cap or None),
1600 base_dir=base_dir,
1601 )
1602 except (object_collections.CollectionNotFoundError,
1603 object_collections.InvalidCollectionNameError, OSError, ValueError):
1604 return None
1605 if result.get("pruned"):
1606 bound = f"older than {days}d" + (f" or beyond {cap} rows" if cap else "")
1607 log(f"Analytics: pruned {result['removed']} page_view(s) {bound} "
1608 f"({result['rows_before']} -> {result['rows_after']})")
1609 return result
1610
1611
1612ATTENTION_SOURCES_COLLECTION = "attention_sources"
1613ATTENTION_COUNTS_COLLECTION = "attention_counts"
1614ATTENTION_MARKER_NAME = ".attention_last_run"
1615ATTENTION_INTERVAL_ENV = "DBBASIC_ATTENTION_INTERVAL_SECONDS"
1616# Five minutes. A count that is five minutes stale is fine -- nobody
1617# confirms a receipt in the seconds after it lands -- and the page says
1618# how old the number is rather than implying it is live.
1619_DEFAULT_ATTENTION_INTERVAL = 300
1620ATTENTION_ACTOR = "daemon:attention"
1621ATTENTION_METHOD = "COUNT"
1622_ATTENTION_DETAIL_LIMIT = 200
1623_ATTENTION_ERROR_LIMIT = 400
1624_TRUE_TEXT = ("true", "1", "yes", "on")
1625
1626
1627def _attention_text(value) -> str:
1628 return str(value if value is not None else "").strip()
1629
1630
1631def _attention_count_value(answer) -> int:
1632 """The integer a provider promised, or a refusal to guess.
1633
1634 Raises ValueError rather than coercing, because every coercion here is
1635 a way for a broken provider to look like an empty queue. A provider
1636 that answers with a string, a float, a bool, or nothing at all has a
1637 bug, and the only useful thing to do with it is say so.
1638 """
1639 if not isinstance(answer, dict):
1640 raise ValueError(f"provider returned {type(answer).__name__}, expected an object")
1641 if "count" not in answer:
1642 raise ValueError("provider returned no 'count'")
1643 count = answer["count"]
1644 if isinstance(count, bool) or not isinstance(count, int):
1645 raise ValueError(f"provider returned a {type(count).__name__} count, expected an integer")
1646 if count < 0:
1647 raise ValueError(f"provider returned a negative count: {count}")
1648 return count
1649
1650
1651def process_attention(*, base_dir: Path | str = "data") -> dict | None:
1652 """Run every declared attention provider and write the counts down.
1653
1654 Packages declare what "needs a human" means in their own domain
1655 (object_packages' `attention` section -> the attention_sources
1656 collection). This pass executes each declaration's object with method
1657 COUNT -- a distinct verb, so the same object may also expose GET or
1658 POST for a person -- and stores the answer in attention_counts, which
1659 is the one small table the home page and the app list read.
1660
1661 **Why a rollup and not a live fold.** Because folding a dozen
1662 collections on every home render is how a home page becomes the
1663 slowest page, and worse than slow: this box went to 675MB resident and
1664 started swapping because something folded a big collection on a timer.
1665 A home screen that folds a dozen collections per render is that exact
1666 mistake with a nicer name -- and it makes the front page's cost grow
1667 with the number of installed packages, which is the one place cost
1668 must not grow. The pass is marker-gated (DBBASIC_ATTENTION_INTERVAL_
1669 SECONDS, default 300) like process_compactions and
1670 process_materializations, so the page's cost is one small read no
1671 matter how many apps declare a queue, and the staleness is bounded and
1672 stated rather than pretended away.
1673
1674 **A broken provider must look broken, never look calm.** A provider
1675 that raises, or answers with something that is not an integer, has its
1676 `error` written and its PREVIOUS count kept. Zeroing it would report
1677 an empty queue when what actually happened is that nobody looked, and
1678 a queue that looks handled is the single worst thing this table could
1679 say. `computed_at` is left at the last SUCCESSFUL run for the same
1680 reason: the age on the page should describe the number, not the
1681 attempt.
1682
1683 Zero counts are written and kept. The absence of a row means nobody
1684 has ever run the provider, which is a different fact from "the queue
1685 is empty", and only the surfaces decide that zeros are not news.
1686
1687 A source the operator muted is not executed at all, and its stored
1688 count is removed -- a muted queue should cost nothing and show
1689 nothing, and leaving the last number behind would show a stale one.
1690
1691 Returns None when the collection is not installed (app-nav absent) or
1692 the interval has not elapsed; otherwise a summary of the pass.
1693 """
1694 try:
1695 sources = object_records.read_collection_records(
1696 ATTENTION_SOURCES_COLLECTION, base_dir=base_dir)
1697 except (object_collections.CollectionNotFoundError,
1698 object_collections.InvalidCollectionNameError, OSError, ValueError):
1699 return None
1700 if not sources:
1701 return None
1702
1703 marker_path = Path(base_dir) / ATTENTION_MARKER_NAME
1704 interval = _env_int(ATTENTION_INTERVAL_ENV, _DEFAULT_ATTENTION_INTERVAL)
1705 now = time.time()
1706 try:
1707 if now - marker_path.stat().st_mtime < interval:
1708 return None
1709 except OSError:
1710 pass # no marker yet -> run now
1711 marker_path.parent.mkdir(parents=True, exist_ok=True)
1712 marker_path.write_text(str(now))
1713
1714 try:
1715 from python_object_runtime import PythonObjectRuntime
1716 except ImportError:
1717 return None
1718 runtime = PythonObjectRuntime(base_dir=str(base_dir))
1719
1720 try:
1721 existing = {row["id"]: row for row in object_records.read_collection_records(
1722 ATTENTION_COUNTS_COLLECTION, base_dir=base_dir) if row.get("id")}
1723 except (object_collections.CollectionNotFoundError,
1724 object_collections.InvalidCollectionNameError, OSError, ValueError):
1725 # The counts collection is what this pass exists to write. Without
1726 # it there is nowhere to put an answer, so there is no point
1727 # running any provider.
1728 return None
1729
1730 computed_at = datetime.now(timezone.utc).isoformat()
1731 counted = failed = muted = 0
1732
1733 for source in sources:
1734 source_id = _attention_text(source.get("id"))
1735 if not source_id:
1736 continue
1737 live = existing.get(source_id)
1738
1739 if _attention_text(source.get("operator_muted")).lower() in _TRUE_TEXT:
1740 muted += 1
1741 if live is not None:
1742 try:
1743 object_records.delete_collection_record(
1744 ATTENTION_COUNTS_COLLECTION, source_id,
1745 base_dir=base_dir, actor=ATTENTION_ACTOR)
1746 except Exception: # noqa: BLE001 -- a stuck row must not stop the pass
1747 pass
1748 continue
1749
1750 row = {
1751 "source_id": source_id,
1752 "label": _attention_text(source.get("label")),
1753 "path": _attention_text(source.get("path")),
1754 "nav_id": _attention_text(source.get("nav_id")),
1755 "group": _attention_text(source.get("group")) or "Apps",
1756 "severity": _attention_text(source.get("severity")) or "normal",
1757 }
1758
1759 object_id = _attention_text(source.get("object_id"))
1760 try:
1761 answer = _execute_target(runtime, object_id, ATTENTION_METHOD, {})
1762 count = _attention_count_value(answer)
1763 except Exception as exc: # noqa: BLE001 -- one bad provider must not blind the rest
1764 failed += 1
1765 log(f"Attention: {source_id} ({object_id}) failed: {exc}", "ERROR")
1766 row["error"] = f"{type(exc).__name__}: {exc}"[:_ATTENTION_ERROR_LIMIT]
1767 # The previous number survives, and so does its timestamp: an
1768 # unanswered question is not the answer zero.
1769 row["count"] = _attention_text((live or {}).get("count")) or "0"
1770 row["detail"] = _attention_text((live or {}).get("detail"))
1771 row["computed_at"] = _attention_text((live or {}).get("computed_at"))
1772 else:
1773 counted += 1
1774 row["count"] = str(count)
1775 row["detail"] = _attention_text(answer.get("detail"))[:_ATTENTION_DETAIL_LIMIT]
1776 row["error"] = ""
1777 row["computed_at"] = computed_at
1778
1779 try:
1780 if live is None:
1781 object_records.create_collection_record(
1782 ATTENTION_COUNTS_COLLECTION, {"id": source_id, **row},
1783 base_dir=base_dir, actor=ATTENTION_ACTOR)
1784 elif any(_attention_text(live.get(field)) != row[field] for field in row):
1785 object_records.update_collection_record(
1786 ATTENTION_COUNTS_COLLECTION, source_id, row,
1787 base_dir=base_dir, actor=ATTENTION_ACTOR, preserve_read_only=True)
1788 except Exception as exc: # noqa: BLE001
1789 log(f"Attention: {source_id} count could not be stored: {exc}", "ERROR")
1790
1791 if counted or failed:
1792 log(f"Attention: counted {counted} queue(s)"
1793 + (f", {failed} provider(s) failed" if failed else "")
1794 + (f", {muted} muted" if muted else ""))
1795 return {"sources": len(sources), "counted": counted, "failed": failed, "muted": muted}
1796
1797
1798def _connector_package_roots() -> list[str]:
1799 """Package source roots the reconcile pass loads connectors from, private
1800 overlay first -- mirrors object_server's resolution so a private connector
1801 (e.g. Mailcow, in packages-private/) is found on the deployment where it is
1802 installed, and a private declaration shadows an open one for a collection."""
1803 roots: list[str] = []
1804 packages = os.environ.get("DBBASIC_PACKAGES_DIR", object_packages.PACKAGES_DIR)
1805 # Private overlay defaults to a sibling of the active packages dir (same
1806 # rule as object_server._private_packages_dir), so overriding the packages
1807 # dir isolates the overlay too.
1808 private = os.environ.get("DBBASIC_PRIVATE_PACKAGES_DIR") or str(Path(packages).parent / "packages-private")
1809 if private and Path(private).is_dir():
1810 roots.append(private)
1811 roots.append(packages)
1812 return roots
1813
1814
1815def _connector_declarations() -> list[dict]:
1816 """All connector declarations across roots, deduped by collection with the
1817 private overlay winning (first root wins)."""
1818 decls: list[dict] = []
1819 seen: set[str] = set()
1820 for root in _connector_package_roots():
1821 for decl in object_packages.iter_connectors(root=root):
1822 if decl["collection"] in seen:
1823 continue
1824 seen.add(decl["collection"])
1825 decls.append(decl)
1826 return decls
1827
1828
1829def process_connectors(*, base_dir: Path | str = "data") -> dict | None:
1830 """03 external connectors: reconcile each declared collection against its
1831 outside system.
1832
1833 For every collection a package declares a connector for, selects the rows
1834 whose desired state the external world doesn't match yet (pending /
1835 pending_delete, backoff due), calls the connector's `reconcile(record)`, and
1836 writes the outcome back via object_connectors.plan_sync -- synced/deleted on
1837 success, retry-with-backoff on a transient error, dead on a permanent one or
1838 exhausted attempts. The driver owns the lifecycle; the connector only
1839 reports ok/error/permanent. Per-row try/except isolates one bad row, and
1840 per-collection isolation keeps one bad connector from stopping the rest.
1841
1842 Returns None when disabled or nothing declares a connector; else a summary.
1843 """
1844 if not object_connectors.connectors_pass_enabled(base_dir=base_dir):
1845 return None
1846 decls = _connector_declarations()
1847 if not decls:
1848 return None
1849
1850 config = object_connectors.connector_config_from_env()
1851 now = datetime.now(timezone.utc)
1852 reconciled = synced = dead = skipped = 0
1853
1854 for decl in decls:
1855 try:
1856 records = object_records.read_collection_records(decl["collection"], base_dir=base_dir)
1857 except (object_collections.CollectionNotFoundError,
1858 object_collections.InvalidCollectionNameError, OSError, ValueError):
1859 continue # collection not installed here -> nothing to reconcile
1860 due = [r for r in records if object_connectors.is_due(r, now)]
1861 if not due:
1862 continue
1863 due.sort(key=lambda r: (str(r.get("sync_next_at") or ""), str(r.get("created_at") or "")))
1864 try:
1865 reconcile = object_connectors.load_connector(decl["module"], decl["entry"])
1866 except object_connectors.ConnectorLoadError as exc:
1867 log(f"Connector: {decl['package_id']}/{decl['collection']} not loaded: {exc}", "ERROR")
1868 continue
1869
1870 for record in due[:config.batch_size]:
1871 try:
1872 outcome = reconcile(record, base_dir=base_dir)
1873 if not isinstance(outcome, dict):
1874 outcome = {"ok": False, "error": f"connector returned {type(outcome).__name__}, expected dict"}
1875 except Exception as exc: # noqa: BLE001 -- transient by default; isolate one bad row
1876 outcome = {"ok": False, "error": str(exc)}
1877 if outcome.get("skip"):
1878 # The connector declined this tick (e.g. unconfigured): leave the
1879 # row exactly as-is -- not an attempt, not a failure, so it waits
1880 # inspectably instead of backing off toward `dead`. Mirrors the
1881 # outbox's "unconfigured = queue only" posture.
1882 skipped += 1
1883 continue
1884 update = object_connectors.plan_sync(record, outcome, config, now=now)
1885 try:
1886 object_records.update_collection_record(
1887 decl["collection"], record["id"], update,
1888 base_dir=base_dir, actor=object_connectors.DEFAULT_ACTOR,
1889 preserve_read_only=True,
1890 )
1891 except Exception as exc: # noqa: BLE001
1892 log(f"Connector: {decl['collection']} {record.get('id')} status write failed: {exc}", "ERROR")
1893 continue
1894 reconciled += 1
1895 status = update.get(object_connectors.SYNC_STATUS_FIELD)
1896 if status in (object_connectors.STATUS_SYNCED, object_connectors.STATUS_DELETED):
1897 synced += 1
1898 elif status == object_connectors.STATUS_DEAD:
1899 dead += 1
1900
1901 if reconciled:
1902 log(f"Connectors: reconciled {reconciled} ({synced} synced, {dead} dead)")
1903 return {"reconciled": reconciled, "synced": synced, "dead": dead, "skipped": skipped}
1904
1905
1906# --- Main ---
1907
1908def shutdown(signum, frame):
1909 global _running
1910 log("Shutting down...")
1911 _running = False
1912
1913
1914def main():
1915 import argparse
1916
1917 parser = argparse.ArgumentParser(description="Object Primitive Daemon")
1918 parser.add_argument("--interval", type=float, default=1.0,
1919 help="Poll interval in seconds (default: 1.0)")
1920 args = parser.parse_args()
1921
1922 # The primitive-runtime passes (scheduler/queue/events trigger objects)
1923 # need the optional dbbasic_object_core runtime. The storage passes
1924 # (compaction, auto-transitions, cleanups) are stdlib and must run
1925 # regardless -- a deployment without the runtime still gets them.
1926 try:
1927 from dbbasic_object_core.runtime.object_runtime import ObjectRuntime
1928 except ImportError:
1929 ObjectRuntime = None
1930 # Native fallback: the repo's own runtime serves the scheduler/queue/
1931 # events passes (load_object + state_manager + execute_object are the
1932 # whole surface they use). The legacy dbbasic_object_core runtime keeps
1933 # precedence where it is installed; deployments without it -- the
1934 # droplet -- get working scheduled tasks instead of a silent disable
1935 # (found 2026-07-24: system_invoice_aging/fin_recurring had NO working
1936 # schedule path in production because this import was missing).
1937 if ObjectRuntime is None:
1938 try:
1939 from python_object_runtime import PythonObjectRuntime as _NativeRuntime
1940 except ImportError:
1941 _NativeRuntime = None
1942 else:
1943 _NativeRuntime = None
1944
1945 # Same data-dir resolution as the server: env first, ./data fallback.
1946 base_dir = os.environ.get("DBBASIC_DATA_DIR", "data")
1947
1948 signal.signal(signal.SIGTERM, shutdown)
1949 signal.signal(signal.SIGINT, shutdown)
1950
1951 print("=" * 60)
1952 print("Object Primitive Daemon")
1953 print("=" * 60)
1954 print(f"Poll interval: {args.interval}s")
1955 print(f"Data dir: {base_dir}")
1956 print(f"Object roots: {', '.join(str(root) for root in _object_roots())}")
1957 if ObjectRuntime is None and _NativeRuntime is None:
1958 print("Object runtime: NOT installed (scheduler/queue/events passes disabled)")
1959 else:
1960 _runtime_label = "dbbasic_object_core" if ObjectRuntime is not None else "native (python_object_runtime)"
1961 print(f"Object runtime: {_runtime_label}")
1962 print(f"Scheduler: {'enabled' if _find_trigger_file('scheduler') else 'no scheduler object'}")
1963 print(f"Queue: {'enabled' if _find_trigger_file('queue') else 'no queue object'}")
1964 print(f"Events: {'enabled' if _find_trigger_file('events') else 'no events object'}")
1965 print(f"Croniter: {'available' if croniter else 'NOT installed (cron tasks disabled)'}")
1966 print(f"Rate limit cleanup: {'enabled' if (Path(base_dir) / 'ratelimit').exists() else 'no ratelimit dir yet'}")
1967 print(f"Compaction interval: {_env_int(COMPACTION_INTERVAL_SECONDS_ENV, _DEFAULT_COMPACTION_INTERVAL_SECONDS)}s")
1968 _startup_rules = auto_transition_rules_from_env()
1969 print(
1970 f"Auto-transition rules: {len(_startup_rules)} configured "
1971 f"(interval {_env_int(AUTO_TRANSITION_INTERVAL_SECONDS_ENV, _DEFAULT_AUTO_TRANSITION_INTERVAL_SECONDS)}s)"
1972 )
1973 print(f"Rollups: {'enabled' if object_rollups.rollup_pass_enabled(base_dir=base_dir) else 'disabled (rollup_enabled flag off)'}")
1974 print(
1975 f"Materialize: {'enabled' if object_materialize.materialize_pass_enabled(base_dir=base_dir) else 'disabled (materialize_enabled flag off)'} "
1976 f"(interval {_env_int(MATERIALIZE_INTERVAL_SECONDS_ENV, _DEFAULT_MATERIALIZE_INTERVAL_SECONDS)}s)"
1977 )
1978 print(f"Notify: {'enabled' if object_notify.notify_pass_enabled(base_dir=base_dir) else 'disabled (notify_enabled flag off)'} (every poll)")
1979 print(f"Change dispatch: {'enabled' if object_change_dispatch.change_dispatch_enabled(base_dir=base_dir) else 'disabled (DBBASIC_ENABLE_CHANGE_DISPATCH unset)'}")
1980 _clp = object_record_changes.retention_policy()
1981 if _clp["days"] or _clp["max_entries"]:
1982 _bounds = " and ".join(
1983 part for part in (
1984 f"{_clp['days']}d" if _clp["days"] else "",
1985 f"{_clp['max_entries']} entries" if _clp["max_entries"] else "",
1986 ) if part)
1987 _fate = "archived" if _clp["archive"] else "DELETED"
1988 _keep = ("all archives kept" if not _clp["keep_archives"]
1989 else f"newest {_clp['keep_archives']} archives kept")
1990 print(f"Change logs: keep {_bounds}; older entries {_fate} ({_keep})")
1991 else:
1992 print("Change logs: unbounded (set DBBASIC_CHANGE_LOG_RETENTION_DAYS "
1993 "or DBBASIC_CHANGE_LOG_MAX_ENTRIES)")
1994 # Books readiness is advisory, never a gate -- the composers are
1995 # deliberately soft dependencies. But a composer that skips because its
1996 # accounts are unmapped otherwise says so only in a return value nobody
1997 # reads, which is harmless on a demo box and a data-integrity hole the
1998 # day real books arrive. Say it out loud at boot instead.
1999 try:
2000 _books = object_books_status.books_status(base_dir=base_dir)
2001 if _books["ready"]:
2002 print("Books: ready (every installed composer can post)")
2003 else:
2004 print(f"Books: NOT READY -- {_books['summary']}")
2005 for _problem in _books.get("problems", [])[:6]:
2006 print(f" - {_problem.get('id')}: {_problem.get('impact')}")
2007 except Exception as _exc: # never let a status check stop the daemon
2008 print(f"Books: status unavailable ({_exc})")
2009 _smtp_config = object_email.smtp_config_from_env()
2010 if not object_email.email_pass_enabled(base_dir=base_dir):
2011 _email_state = "disabled (email_enabled flag off)"
2012 elif not _smtp_config.configured:
2013 _email_state = f"queuing only (DBBASIC_SMTP_MODE={_smtp_config.mode!r}, not configured)"
2014 else:
2015 _email_state = f"enabled (mode={_smtp_config.mode})"
2016 print(f"Email: {_email_state}")
2017 _connector_decls = _connector_declarations()
2018 if not object_connectors.connectors_pass_enabled(base_dir=base_dir):
2019 _connectors_state = "disabled (connectors_enabled flag off)"
2020 elif not _connector_decls:
2021 _connectors_state = "no connectors declared"
2022 else:
2023 _connectors_state = f"{len(_connector_decls)} connector(s): " + ", ".join(
2024 f"{d['package_id']}/{d['collection']}" for d in _connector_decls)
2025 print(f"Connectors: {_connectors_state}")
2026 try:
2027 _attention_sources = object_records.read_collection_records(
2028 ATTENTION_SOURCES_COLLECTION, base_dir=base_dir)
2029 except Exception:
2030 _attention_sources = None
2031 if _attention_sources is None:
2032 _attention_state = "no attention_sources collection (app-nav not installed)"
2033 elif not _attention_sources:
2034 _attention_state = "no queues declared"
2035 else:
2036 _attention_state = (
2037 f"{len(_attention_sources)} queue(s) "
2038 f"(interval {_env_int(ATTENTION_INTERVAL_ENV, _DEFAULT_ATTENTION_INTERVAL)}s)")
2039 print(f"Attention: {_attention_state}")
2040 if object_analytics.analytics_enabled():
2041 print(f"Analytics: capture ON (retention {object_analytics.retention_days()}d)")
2042 else:
2043 print("Analytics: capture off (DBBASIC_ANALYTICS unset)")
2044 # Stated in words, every boot, because the cookie is the only thing
2045 # this server stores on somebody else's device and the setting that
2046 # turns it on is one word in an env file. An operator should not have
2047 # to go and read a document to find out what they took on.
2048 print(f"Analytics: {object_analytics.visitor_cookie_posture()}")
2049 print()
2050 print("Press Ctrl+C to stop")
2051 print("=" * 60)
2052 print()
2053
2054 if ObjectRuntime is not None:
2055 runtime = ObjectRuntime(base_dir=str(base_dir))
2056 elif _NativeRuntime is not None:
2057 runtime = _NativeRuntime()
2058 else:
2059 runtime = None
2060 last_event_cleanup = 0.0
2061
2062 while _running:
2063 if runtime is not None:
2064 try:
2065 process_scheduler(runtime)
2066 except Exception as e:
2067 log(f"Scheduler error: {e}", 'ERROR')
2068
2069 try:
2070 process_queue(runtime)
2071 except Exception as e:
2072 log(f"Queue error: {e}", 'ERROR')
2073
2074 try:
2075 process_events(runtime)
2076 except Exception as e:
2077 log(f"Events error: {e}", 'ERROR')
2078
2079 try:
2080 cleanup_ratelimit(base_dir=base_dir)
2081 except Exception as e:
2082 log(f"Cleanup error: {e}", 'ERROR')
2083
2084 try:
2085 process_compactions(base_dir=base_dir)
2086 except Exception as e:
2087 log(f"Compaction error: {e}", 'ERROR')
2088
2089 try:
2090 process_stale_transitions(base_dir=base_dir)
2091 except Exception as e:
2092 log(f"Auto-transition error: {e}", 'ERROR')
2093
2094 try:
2095 process_rollups(base_dir=base_dir)
2096 except Exception as e:
2097 log(f"Rollup error: {e}", 'ERROR')
2098
2099 try:
2100 process_materializations(base_dir=base_dir)
2101 except Exception as e:
2102 log(f"Materialize error: {e}", 'ERROR')
2103
2104 try:
2105 process_notifications(base_dir=base_dir)
2106 except Exception as e:
2107 log(f"Notify error: {e}", 'ERROR')
2108
2109 try:
2110 object_change_dispatch.dispatch_pending(base_dir=base_dir)
2111 except Exception as e:
2112 log(f"Change dispatch error: {e}", 'ERROR')
2113
2114 try:
2115 process_email_outbox(base_dir=base_dir)
2116 except Exception as e:
2117 log(f"Email error: {e}", 'ERROR')
2118
2119 try:
2120 process_connectors(base_dir=base_dir)
2121 except Exception as e:
2122 log(f"Connectors error: {e}", 'ERROR')
2123
2124 try:
2125 process_analytics_retention(base_dir=base_dir)
2126 except Exception as e:
2127 log(f"Analytics retention error: {e}", 'ERROR')
2128
2129 try:
2130 process_change_log_retention(base_dir=base_dir)
2131 except Exception as e:
2132 log(f"Change log retention error: {e}", 'ERROR')
2133
2134 try:
2135 process_restore_point_retention(base_dir=base_dir)
2136 except Exception as e:
2137 log(f"Restore point retention error: {e}", 'ERROR')
2138
2139 try:
2140 process_attention(base_dir=base_dir)
2141 except Exception as e:
2142 log(f"Attention error: {e}", 'ERROR')
2143
2144 now = time.time()
2145 if now - last_event_cleanup >= EVENT_CLEANUP_INTERVAL_SECONDS:
2146 last_event_cleanup = now
2147 try:
2148 cleanup_events(base_dir=base_dir)
2149 except Exception as e:
2150 log(f"Event cleanup error: {e}", 'ERROR')
2151
2152 # Sleep in small increments for responsive shutdown
2153 deadline = now + args.interval
2154 while _running and time.time() < deadline:
2155 time.sleep(0.1)
2156
2157 log("Daemon stopped.")
2158
2159
2160if __name__ == "__main__":
2161 main()