Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_object_daemon.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_object_daemon.py · 44.0 KB · 1170 lines
1import importlib.util
2import json
3import sys
4import time
5from datetime import datetime, timedelta, timezone
6from pathlib import Path
7
8import object_record_changes
9import object_records
10
11ROOT = Path(__file__).resolve().parents[1]
12spec = importlib.util.spec_from_file_location("object_daemon", ROOT / "object_daemon.py")
13object_daemon = importlib.util.module_from_spec(spec)
14sys.modules["object_daemon"] = object_daemon
15assert spec.loader is not None
16spec.loader.exec_module(object_daemon)
17
18
19def write_append_schema(data_dir: Path, collection: str, fields: list[dict] | None = None) -> Path:
20 path = data_dir / "schemas" / f"{collection}.json"
21 path.parent.mkdir(parents=True, exist_ok=True)
22 path.write_text(json.dumps({"fields": fields or [{"name": "id"}], "storage": "append"}))
23 return path
24
25
26class FakeStateManager:
27 def __init__(self):
28 self.values = {}
29
30 def reload(self):
31 return None
32
33 def get_all(self):
34 return dict(self.values)
35
36 def get(self, key):
37 return self.values.get(key)
38
39 def set(self, key, value):
40 self.values[key] = value
41
42
43class FakeObject:
44 def __init__(self):
45 self.state_manager = FakeStateManager()
46 self.calls = []
47
48 def execute(self, method, payload):
49 self.calls.append((method, payload))
50 return {"status": "ok"}
51
52
53class FailingObject(FakeObject):
54 def execute(self, method, payload):
55 self.calls.append((method, payload))
56 raise RuntimeError("target failed")
57
58
59class FakeRuntime:
60 def __init__(self):
61 self.objects = {}
62 self.loaded = []
63
64 def add(self, object_id, obj):
65 self.objects[object_id] = obj
66 return obj
67
68 def load_object(self, path, object_id=None):
69 resolved_id = object_id or Path(path).stem
70 self.loaded.append((resolved_id, Path(path)))
71 return self.objects[resolved_id]
72
73
74def write_object(path):
75 path.parent.mkdir(parents=True, exist_ok=True)
76 path.write_text("def GET(request):\n return {'status': 'ok'}\n")
77 return path
78
79
80def test_daemon_imports_without_runtime_package():
81 assert callable(object_daemon.process_queue)
82 assert callable(object_daemon.process_events)
83 assert callable(object_daemon.cleanup_ratelimit)
84 assert callable(object_daemon.cleanup_events)
85
86
87def test_calculate_onetime_next_run():
88 task = {"type": "onetime", "schedule": "2026-01-02T03:04:05Z"}
89
90 next_run = object_daemon._calculate_next_run(task)
91
92 assert isinstance(next_run, int)
93 assert next_run > 0
94
95
96def test_cleanup_ratelimit_removes_expired_and_corrupt_files(tmp_path, monkeypatch):
97 monkeypatch.chdir(tmp_path)
98 ratelimit_dir = tmp_path / "data" / "ratelimit"
99 ratelimit_dir.mkdir(parents=True)
100
101 now = time.time()
102 expired = ratelimit_dir / "expired.txt"
103 recent = ratelimit_dir / "recent.txt"
104 corrupt = ratelimit_dir / "corrupt.txt"
105 ignored = ratelimit_dir / "ignored.log"
106
107 expired.write_text(f"{now - 500}\n")
108 recent.write_text(f"{now}\n")
109 corrupt.write_text("not-a-timestamp\n")
110 ignored.write_text(f"{now - 500}\n")
111
112 object_daemon.cleanup_ratelimit(max_age=120)
113
114 assert not expired.exists()
115 assert recent.exists()
116 assert not corrupt.exists()
117 assert ignored.exists()
118
119
120def test_cleanup_events_prunes_event_queue(tmp_path):
121 data_dir = tmp_path / "data"
122 object_daemon.object_events.publish_event(
123 "test.event",
124 payload={"id": "first"},
125 base_dir=data_dir,
126 )
127 object_daemon.object_events.publish_event(
128 "test.event",
129 payload={"id": "second"},
130 base_dir=data_dir,
131 )
132
133 result = object_daemon.cleanup_events(base_dir=data_dir, keep_count=1, keep_seconds=0)
134
135 assert result["deleted"] == 1
136 assert object_daemon.object_events.list_events(base_dir=data_dir)["total"] == 1
137
138
139def test_process_queue_completes_pending_message(tmp_path, monkeypatch):
140 monkeypatch.chdir(tmp_path)
141 write_object(tmp_path / "objects" / "triggers" / "queue.py")
142 write_object(tmp_path / "objects" / "basics" / "target.py")
143
144 runtime = FakeRuntime()
145 queue = runtime.add("queue", FakeObject())
146 target = runtime.add("basics_target", FakeObject())
147
148 now = int(time.time())
149 key = "msg_tasks_2_1_msg_001"
150 queue.state_manager.set(
151 key,
152 json.dumps(
153 {
154 "id": "msg_001",
155 "queue_name": "tasks",
156 "message": {
157 "object_id": "basics_target",
158 "method": "POST",
159 "payload": {"action": "test"},
160 },
161 "priority_level": 2,
162 "status": "pending",
163 "created_at": now - 10,
164 "visible_after": now - 10,
165 "expires_at": now + 3600,
166 "attempts": 0,
167 "max_attempts": 3,
168 }
169 ),
170 )
171
172 object_daemon.process_queue(runtime)
173
174 updated = json.loads(queue.state_manager.get(key))
175 assert updated["status"] == "completed"
176 assert target.calls == [("POST", {"action": "test"})]
177
178
179def test_process_queue_marks_expired_message(tmp_path, monkeypatch):
180 monkeypatch.chdir(tmp_path)
181 write_object(tmp_path / "objects" / "triggers" / "queue.py")
182
183 runtime = FakeRuntime()
184 queue = runtime.add("queue", FakeObject())
185
186 now = int(time.time())
187 key = "msg_tasks_2_1_expired"
188 queue.state_manager.set(
189 key,
190 json.dumps(
191 {
192 "id": "expired",
193 "queue_name": "tasks",
194 "message": {"object_id": "basics_target"},
195 "priority_level": 2,
196 "status": "pending",
197 "created_at": now - 100,
198 "visible_after": now - 100,
199 "expires_at": now - 1,
200 "attempts": 0,
201 "max_attempts": 3,
202 }
203 ),
204 )
205
206 object_daemon.process_queue(runtime)
207
208 updated = json.loads(queue.state_manager.get(key))
209 assert updated["status"] == "expired"
210
211
212def test_find_object_file_honors_configured_objects_dir(tmp_path, monkeypatch):
213 monkeypatch.chdir(tmp_path)
214 custom_root = tmp_path / "custom_objects"
215 custom_path = write_object(custom_root / "basics" / "counter.py")
216 write_object(tmp_path / "objects" / "basics" / "counter.py")
217 monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(custom_root))
218
219 found = object_daemon._find_object_file("basics_counter")
220
221 assert found == custom_path
222
223
224def test_find_object_file_resolves_user_object_from_objects_dir(tmp_path, monkeypatch):
225 monkeypatch.chdir(tmp_path)
226 user_path = write_object(tmp_path / "objects" / "users" / "42" / "deals.py")
227
228 found = object_daemon._find_object_file("u_42_deals")
229
230 assert found is not None
231 assert found.resolve() == user_path.resolve()
232
233
234def test_process_queue_requeues_failed_message(tmp_path, monkeypatch):
235 monkeypatch.chdir(tmp_path)
236 write_object(tmp_path / "objects" / "triggers" / "queue.py")
237 write_object(tmp_path / "objects" / "basics" / "target.py")
238
239 runtime = FakeRuntime()
240 queue = runtime.add("queue", FakeObject())
241 runtime.add("basics_target", FailingObject())
242
243 now = int(time.time())
244 key = "msg_tasks_2_1_retry"
245 queue.state_manager.set(
246 key,
247 json.dumps(
248 {
249 "id": "retry",
250 "queue_name": "tasks",
251 "message": {"object_id": "basics_target", "method": "POST", "payload": {}},
252 "priority_level": 2,
253 "status": "pending",
254 "created_at": now - 10,
255 "visible_after": now - 10,
256 "expires_at": now + 3600,
257 "attempts": 0,
258 "max_attempts": 3,
259 }
260 ),
261 )
262
263 object_daemon.process_queue(runtime)
264
265 updated = json.loads(queue.state_manager.get(key))
266 assert updated["status"] == "pending"
267 assert updated["attempts"] == 1
268 assert updated["visible_after"] > now
269
270
271def test_process_events_delivers_matching_event(tmp_path, monkeypatch):
272 monkeypatch.chdir(tmp_path)
273 write_object(tmp_path / "objects" / "triggers" / "events.py")
274
275 runtime = FakeRuntime()
276 events = runtime.add("events", FakeObject())
277
278 now = int(time.time())
279 events.state_manager.set(
280 "sub_test.event_sub_001",
281 json.dumps(
282 {
283 "id": "sub_001",
284 "event_type": "test.event",
285 "callback_url": "http://127.0.0.1:9999/hook",
286 "last_event_id": None,
287 }
288 ),
289 )
290 events.state_manager.set(
291 f"event_{now}_evt_001",
292 json.dumps(
293 {
294 "id": "evt_001",
295 "event_type": "test.event",
296 "payload": {"value": 1},
297 "timestamp": now,
298 }
299 ),
300 )
301
302 delivered = []
303
304 class FakeResponse:
305 def getcode(self):
306 return 204
307
308 def __enter__(self):
309 return self
310
311 def __exit__(self, exc_type, exc, tb):
312 return False
313
314 def fake_urlopen(request, timeout):
315 delivered.append((request, timeout))
316 return FakeResponse()
317
318 monkeypatch.setattr(object_daemon.urllib.request, "urlopen", fake_urlopen)
319
320 object_daemon.process_events(runtime)
321
322 assert len(delivered) == 1
323 request, timeout = delivered[0]
324 assert request.full_url == "http://127.0.0.1:9999/hook"
325 assert timeout == 5
326 updated = json.loads(events.state_manager.get("sub_test.event_sub_001"))
327 assert updated["last_event_id"] == "evt_001"
328 assert updated["delivery"]["status"] == "ok"
329 assert updated["delivery"]["attempts"] == 1
330 assert updated["delivery"]["successes"] == 1
331 assert updated["delivery"]["last_status_code"] == 204
332
333
334def test_process_events_records_failure_without_advancing_cursor(tmp_path, monkeypatch):
335 monkeypatch.chdir(tmp_path)
336 write_object(tmp_path / "objects" / "triggers" / "events.py")
337
338 runtime = FakeRuntime()
339 events = runtime.add("events", FakeObject())
340
341 now = int(time.time())
342 events.state_manager.set(
343 "sub_test.event_sub_001",
344 json.dumps(
345 {
346 "id": "sub_001",
347 "event_type": "test.event",
348 "callback_url": "http://127.0.0.1:9999/hook",
349 "last_event_id": None,
350 }
351 ),
352 )
353 for index in (1, 2):
354 events.state_manager.set(
355 f"event_{now + index}_evt_00{index}",
356 json.dumps(
357 {
358 "id": f"evt_00{index}",
359 "event_type": "test.event",
360 "payload": {"value": index},
361 "timestamp": now + index,
362 }
363 ),
364 )
365
366 attempts = []
367
368 def fake_urlopen(request, timeout):
369 attempts.append((request, timeout))
370 raise object_daemon.urllib.error.URLError("callback down")
371
372 monkeypatch.setattr(object_daemon.urllib.request, "urlopen", fake_urlopen)
373
374 object_daemon.process_events(runtime)
375
376 assert len(attempts) == 1
377 updated = json.loads(events.state_manager.get("sub_test.event_sub_001"))
378 assert updated["last_event_id"] is None
379 assert updated["delivery"]["status"] == "failed"
380 assert updated["delivery"]["attempts"] == 1
381 assert updated["delivery"]["failures"] == 1
382 assert updated["delivery"]["last_attempted_event_id"] == "evt_001"
383 assert "callback down" in updated["delivery"]["last_error"]
384
385
386# --- Compaction pass (docs/append-only-storage-design.md Compaction) --------
387
388
389def _seed_bloated_append_collection(data_dir: Path, collection: str, *, live: int = 1, churn: int = 5) -> None:
390 """Create `collection` in append storage, then churn one record
391 `churn` times so physical rows pile up well past `live`."""
392 write_append_schema(data_dir, collection, [{"name": "id"}, {"name": "value"}])
393 for i in range(live):
394 object_records.create_collection_record(
395 collection, {"id": f"r{i}", "value": "v0"}, base_dir=data_dir, roots=[]
396 )
397 for i in range(churn):
398 object_records.update_collection_record(
399 collection, "r0", {"value": f"v{i}"}, base_dir=data_dir, roots=[]
400 )
401
402
403def test_process_compactions_compacts_collections_over_threshold(tmp_path, monkeypatch):
404 monkeypatch.setenv("DBBASIC_COMPACTION_INTERVAL_SECONDS", "0")
405 monkeypatch.setenv("DBBASIC_APPEND_COMPACT_MIN_ROWS", "3")
406 monkeypatch.setenv("DBBASIC_COMPACTION_BLOAT_RATIO", "1.0")
407 data_dir = tmp_path / "data"
408
409 # "hot": 1 live + 5 updates = 6 physical rows, 1 live -> well over both
410 # the row floor and the bloat ratio.
411 _seed_bloated_append_collection(data_dir, "hot", live=1, churn=5)
412 # "cold": 2 live rows, no churn -> under DBBASIC_APPEND_COMPACT_MIN_ROWS
413 # even though (trivially) bloat_ratio is 0, so it must be left alone.
414 write_append_schema(data_dir, "cold", [{"name": "id"}, {"name": "value"}])
415 object_records.create_collection_record("cold", {"id": "c1", "value": "v"}, base_dir=data_dir, roots=[])
416 object_records.create_collection_record("cold", {"id": "c2", "value": "v"}, base_dir=data_dir, roots=[])
417
418 hot_path = object_records.collection_records_file("hot", base_dir=data_dir)
419 cold_path = object_records.collection_records_file("cold", base_dir=data_dir)
420 hot_rows_before = len(hot_path.read_text().splitlines()) - 1
421 cold_rows_before = len(cold_path.read_text().splitlines()) - 1
422 assert hot_rows_before == 6
423 assert cold_rows_before == 2
424
425 result = object_daemon.process_compactions(base_dir=data_dir)
426
427 assert result is not None
428 assert [entry["collection"] for entry in result["compacted"]] == ["hot"]
429
430 hot_rows_after = len(hot_path.read_text().splitlines()) - 1
431 cold_rows_after = len(cold_path.read_text().splitlines()) - 1
432 assert hot_rows_after == 1 # compacted down to its one live record
433 assert cold_rows_after == cold_rows_before # untouched: under the row floor
434
435
436def test_process_compactions_honors_interval_marker(tmp_path, monkeypatch):
437 monkeypatch.setenv("DBBASIC_APPEND_COMPACT_MIN_ROWS", "3")
438 monkeypatch.setenv("DBBASIC_COMPACTION_BLOAT_RATIO", "1.0")
439 monkeypatch.setenv("DBBASIC_COMPACTION_INTERVAL_SECONDS", "3600")
440 data_dir = tmp_path / "data"
441 _seed_bloated_append_collection(data_dir, "hot", live=1, churn=5)
442 path = object_records.collection_records_file("hot", base_dir=data_dir)
443
444 # First call: no marker yet -> runs and compacts.
445 first = object_daemon.process_compactions(base_dir=data_dir)
446 assert first is not None
447 assert [entry["collection"] for entry in first["compacted"]] == ["hot"]
448 rows_after_first = len(path.read_text().splitlines()) - 1
449 assert rows_after_first == 1
450
451 # Re-bloat the file directly (bypassing the pass) and call again
452 # immediately: the marker was just written, so the interval hasn't
453 # elapsed and this call must be a pure no-op -- no listing, no compaction.
454 for i in range(5):
455 object_records.update_collection_record(
456 "hot", "r0", {"value": f"w{i}"}, base_dir=data_dir, roots=[]
457 )
458 rows_before_second = len(path.read_text().splitlines()) - 1
459 assert rows_before_second > 1
460
461 second = object_daemon.process_compactions(base_dir=data_dir)
462 assert second is None
463 rows_after_second = len(path.read_text().splitlines()) - 1
464 assert rows_after_second == rows_before_second # untouched
465
466
467def test_process_compactions_one_collection_failing_does_not_stop_others(tmp_path, monkeypatch):
468 monkeypatch.setenv("DBBASIC_COMPACTION_INTERVAL_SECONDS", "0")
469 monkeypatch.setenv("DBBASIC_APPEND_COMPACT_MIN_ROWS", "3")
470 monkeypatch.setenv("DBBASIC_COMPACTION_BLOAT_RATIO", "1.0")
471 data_dir = tmp_path / "data"
472 _seed_bloated_append_collection(data_dir, "bad", live=1, churn=5)
473 _seed_bloated_append_collection(data_dir, "good", live=1, churn=5)
474
475 real_compact = object_records.compact_collection
476
477 def flaky_compact(collection, **kwargs):
478 if collection == "bad":
479 raise RuntimeError("simulated compaction failure")
480 return real_compact(collection, **kwargs)
481
482 monkeypatch.setattr(object_records, "compact_collection", flaky_compact)
483
484 result = object_daemon.process_compactions(base_dir=data_dir)
485
486 assert result is not None
487 assert result["checked"] == 2
488 assert [entry["collection"] for entry in result["compacted"]] == ["good"]
489
490 good_path = object_records.collection_records_file("good", base_dir=data_dir)
491 bad_path = object_records.collection_records_file("bad", base_dir=data_dir)
492 assert len(good_path.read_text().splitlines()) - 1 == 1 # compacted
493 assert len(bad_path.read_text().splitlines()) - 1 == 6 # left as-is after its failure
494
495
496# --- Stale-state auto-transition pass ----------------------------------
497
498
499def write_tasks_schema(data_dir: Path) -> Path:
500 """A trimmed stand-in for packages/app-tasks/schemas/tasks.json: same
501 status enum/transitions and the same lone `created_at` timestamp
502 field (no `updated_at`), classic (non-append) storage."""
503 path = data_dir / "schemas" / "tasks.json"
504 path.parent.mkdir(parents=True, exist_ok=True)
505 path.write_text(json.dumps({
506 "fields": [
507 {"name": "id"},
508 {"name": "title", "type": "text", "required": True},
509 {
510 "name": "status",
511 "type": "enum",
512 "default": "open",
513 "enum": ["draft", "open", "assigned", "waiting_on_client",
514 "approved", "disputed", "cancelled"],
515 "transitions": {
516 "draft": ["open", "assigned", "cancelled"],
517 "open": ["assigned", "cancelled"],
518 "assigned": ["waiting_on_client", "open", "cancelled"],
519 "waiting_on_client": ["approved", "disputed", "assigned"],
520 "disputed": ["assigned", "cancelled"],
521 },
522 },
523 {"name": "created_at", "type": "datetime", "read_only": True},
524 ],
525 }))
526 return path
527
528
529def _backdate_change_log(data_dir: Path, collection: str, record_id: str, *, hours_ago: float) -> None:
530 """Rewrite every record-change log entry for `record_id` to look
531 `hours_ago` old, so a record created "now" can still exercise the
532 stale-transition age check without sleeping in a test."""
533 path = object_record_changes.record_changes_file(collection, base_dir=data_dir)
534 past = (datetime.now(timezone.utc) - timedelta(hours=hours_ago)).isoformat()
535 lines = path.read_text(encoding="utf-8").splitlines()
536 rewritten = []
537 for line in lines:
538 entry = json.loads(line)
539 if entry.get("record_id") == record_id:
540 entry["timestamp"] = past
541 rewritten.append(json.dumps(entry))
542 path.write_text("\n".join(rewritten) + "\n", encoding="utf-8")
543
544
545def test_auto_transition_rules_from_env_defaults_to_tasks_rule(monkeypatch):
546 # Unscheduled by default was exactly the predecessor bug (200+ records
547 # stranded in waiting_on_client) -- the default here must be a real,
548 # working rule, not an empty list a caller has to opt into.
549 monkeypatch.delenv("DBBASIC_AUTO_TRANSITION_RULES", raising=False)
550 rules = object_daemon.auto_transition_rules_from_env()
551 assert rules == [{
552 "collection": "tasks",
553 "field": "status",
554 "from": "waiting_on_client",
555 "to": "approved",
556 "after_hours": 48.0,
557 }]
558
559
560def test_auto_transition_rules_from_env_empty_string_disables(monkeypatch):
561 monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_RULES", "")
562 assert object_daemon.auto_transition_rules_from_env() == []
563 monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_RULES", "[]")
564 assert object_daemon.auto_transition_rules_from_env() == []
565
566
567def test_process_stale_transitions_moves_stale_record_with_daemon_actor(tmp_path, monkeypatch):
568 monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS", "0")
569 data_dir = tmp_path / "data"
570 write_tasks_schema(data_dir)
571 object_records.create_collection_record(
572 "tasks",
573 {"id": "t1", "title": "Stale task", "status": "waiting_on_client"},
574 base_dir=data_dir,
575 roots=[],
576 actor="tester",
577 )
578 _backdate_change_log(data_dir, "tasks", "t1", hours_ago=50)
579
580 result = object_daemon.process_stale_transitions(base_dir=data_dir)
581
582 assert result is not None
583 assert result["checked"] == 1
584 assert result["moved"] == 1
585 assert result["rules"][0]["collection"] == "tasks"
586
587 current = object_records.get_collection_record("tasks", "t1", base_dir=data_dir)
588 assert current["status"] == "approved"
589
590 changes = object_record_changes.list_record_changes("tasks", record_id="t1", base_dir=data_dir)
591 latest = changes["changes"][0]
592 assert latest["action"] == "update"
593 assert latest["actor"] == "daemon:auto-transition"
594 assert latest["after"]["status"] == "approved"
595
596
597def test_process_stale_transitions_leaves_fresh_record_alone(tmp_path, monkeypatch):
598 monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS", "0")
599 data_dir = tmp_path / "data"
600 write_tasks_schema(data_dir)
601 object_records.create_collection_record(
602 "tasks",
603 {"id": "t1", "title": "Fresh task", "status": "waiting_on_client"},
604 base_dir=data_dir,
605 roots=[],
606 actor="tester",
607 )
608 # No backdating: the record just entered waiting_on_client.
609
610 result = object_daemon.process_stale_transitions(base_dir=data_dir)
611
612 assert result is not None
613 assert result["moved"] == 0
614 current = object_records.get_collection_record("tasks", "t1", base_dir=data_dir)
615 assert current["status"] == "waiting_on_client"
616
617
618def test_process_stale_transitions_disabled_by_empty_rules(tmp_path, monkeypatch):
619 monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS", "0")
620 monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_RULES", "[]")
621 data_dir = tmp_path / "data"
622 write_tasks_schema(data_dir)
623 object_records.create_collection_record(
624 "tasks",
625 {"id": "t1", "title": "Stale task", "status": "waiting_on_client"},
626 base_dir=data_dir,
627 roots=[],
628 actor="tester",
629 )
630 _backdate_change_log(data_dir, "tasks", "t1", hours_ago=200)
631
632 result = object_daemon.process_stale_transitions(base_dir=data_dir)
633
634 assert result == {"checked": 0, "moved": 0, "rules": []}
635 current = object_records.get_collection_record("tasks", "t1", base_dir=data_dir)
636 assert current["status"] == "waiting_on_client" # untouched
637
638
639def test_process_stale_transitions_skips_unknown_collection(tmp_path, monkeypatch):
640 monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS", "0")
641 monkeypatch.setenv(
642 "DBBASIC_AUTO_TRANSITION_RULES",
643 json.dumps([{
644 "collection": "not_installed",
645 "field": "status",
646 "from": "waiting_on_client",
647 "to": "approved",
648 "after_hours": 48,
649 }]),
650 )
651 data_dir = tmp_path / "data" # no schema, no records -- package "not installed"
652
653 result = object_daemon.process_stale_transitions(base_dir=data_dir)
654
655 assert result == {
656 "checked": 1,
657 "moved": 0,
658 "rules": [{
659 "collection": "not_installed",
660 "field": "status",
661 "from": "waiting_on_client",
662 "to": "approved",
663 "moved": 0,
664 }],
665 }
666
667
668def test_process_stale_transitions_skips_record_that_moved_on(tmp_path, monkeypatch):
669 """A record staged as a stale candidate by the pass's own listing, but
670 whose live value has already moved off `from` by the time the pass
671 gets to writing it (a race between the listing and the per-record
672 write), must be left alone -- not force-moved back to `to`."""
673 monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS", "0")
674 data_dir = tmp_path / "data"
675 write_tasks_schema(data_dir)
676 object_records.create_collection_record(
677 "tasks",
678 {"id": "t1", "title": "Racing task", "status": "waiting_on_client"},
679 base_dir=data_dir,
680 roots=[],
681 actor="tester",
682 )
683 _backdate_change_log(data_dir, "tasks", "t1", hours_ago=50)
684
685 stale_snapshot = object_records.read_collection_records("tasks", base_dir=data_dir)
686
687 # The record moves on for real before the pass's write reaches it.
688 object_records.update_collection_record(
689 "tasks", "t1", {"status": "assigned"}, base_dir=data_dir, roots=[], actor="tester",
690 )
691
692 monkeypatch.setattr(
693 object_records,
694 "read_collection_records",
695 lambda collection, **kwargs: stale_snapshot if collection == "tasks" else [],
696 )
697
698 result = object_daemon.process_stale_transitions(base_dir=data_dir)
699
700 assert result["moved"] == 0
701 current = object_records.get_collection_record("tasks", "t1", base_dir=data_dir)
702 assert current["status"] == "assigned" # not force-moved to "approved"
703
704
705def test_process_stale_transitions_honors_interval_marker(tmp_path, monkeypatch):
706 monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS", "3600")
707 data_dir = tmp_path / "data"
708 write_tasks_schema(data_dir)
709 object_records.create_collection_record(
710 "tasks",
711 {"id": "t1", "title": "Stale task", "status": "waiting_on_client"},
712 base_dir=data_dir,
713 roots=[],
714 actor="tester",
715 )
716 _backdate_change_log(data_dir, "tasks", "t1", hours_ago=50)
717
718 first = object_daemon.process_stale_transitions(base_dir=data_dir)
719 assert first is not None
720 assert first["moved"] == 1
721
722 # Put another stale record in place directly (bypassing the pass) and
723 # call again immediately: the marker was just written, so the interval
724 # hasn't elapsed and this call must be a pure no-op.
725 object_records.create_collection_record(
726 "tasks",
727 {"id": "t2", "title": "Also stale", "status": "waiting_on_client"},
728 base_dir=data_dir,
729 roots=[],
730 actor="tester",
731 )
732 _backdate_change_log(data_dir, "tasks", "t2", hours_ago=50)
733
734 second = object_daemon.process_stale_transitions(base_dir=data_dir)
735 assert second is None
736 current = object_records.get_collection_record("tasks", "t2", base_dir=data_dir)
737 assert current["status"] == "waiting_on_client" # untouched
738
739
740def write_orders_schema(data_dir: Path) -> Path:
741 return write_schema(data_dir, "orders", [
742 {"name": "id"},
743 {"name": "channel", "type": "text"},
744 {"name": "status", "type": "text"},
745 {"name": "total_cents", "type": "integer"},
746 {"name": "created_at", "type": "datetime"},
747 ])
748
749
750def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path:
751 path = data_dir / "schemas" / f"{collection}.json"
752 path.parent.mkdir(parents=True, exist_ok=True)
753 path.write_text(json.dumps({"name": collection, "fields": fields}))
754 return path
755
756
757def _rollup_definition_row(**overrides) -> dict:
758 row = {
759 "id": "rollup_orders_by_day",
760 "name": "Orders per day",
761 "source_collection": "orders",
762 "filter": json.dumps({"status": "paid"}),
763 "group_by": json.dumps(["channel"]),
764 "time_bucket": json.dumps({"field": "created_at", "granularity": "day"}),
765 "metrics": json.dumps([{"op": "count", "as": "order_count"}]),
766 "target_collection": "rollup_orders_by_day",
767 "min_group_size": "",
768 "refresh_mode": "scheduled",
769 "refresh_interval_seconds": "3600",
770 "last_computed_at": "",
771 "enabled": "true",
772 }
773 row.update(overrides)
774 return row
775
776
777def _install_rollup_definitions(data_dir: Path, rows: list[dict]) -> None:
778 write_schema(data_dir, "rollup_definitions", [
779 {"name": "id"},
780 {"name": "name", "type": "text"},
781 {"name": "source_collection", "type": "text"},
782 {"name": "filter", "type": "textarea"},
783 {"name": "group_by", "type": "textarea"},
784 {"name": "time_bucket", "type": "textarea"},
785 {"name": "metrics", "type": "textarea"},
786 {"name": "target_collection", "type": "text"},
787 {"name": "min_group_size", "type": "integer"},
788 {"name": "refresh_mode", "type": "text"},
789 {"name": "refresh_interval_seconds", "type": "integer"},
790 {"name": "last_computed_at", "type": "datetime", "read_only": True},
791 {"name": "enabled", "type": "boolean"},
792 ])
793 for row in rows:
794 # preserve_read_only: seeding a fixture row directly, same posture
795 # as object_import.py replaying another system's history -- a real
796 # admin-authored create would omit last_computed_at entirely (the
797 # generated form never renders a read_only field), never submit it.
798 object_records.create_collection_record(
799 "rollup_definitions", row, base_dir=data_dir, roots=[], actor="tester",
800 preserve_read_only=True,
801 )
802
803
804def test_process_rollups_computes_a_due_definition_and_stamps_last_computed_at(tmp_path):
805 data_dir = tmp_path / "data"
806 write_orders_schema(data_dir)
807 object_records.create_collection_record(
808 "orders", {"id": "o1", "channel": "web", "status": "paid", "total_cents": "1000",
809 "created_at": "2026-07-01T00:00:00Z"},
810 base_dir=data_dir, roots=[],
811 )
812 _install_rollup_definitions(data_dir, [_rollup_definition_row()])
813
814 result = object_daemon.process_rollups(base_dir=data_dir)
815
816 assert result is not None
817 assert result["checked"] == 1
818 assert [c["definition_id"] for c in result["computed"]] == ["rollup_orders_by_day"]
819
820 target_rows = object_records.read_collection_records("rollup_orders_by_day", base_dir=data_dir, roots=[])
821 assert len(target_rows) == 1
822 assert target_rows[0]["order_count"] == "1"
823
824 definition = object_records.get_collection_record("rollup_definitions", "rollup_orders_by_day", base_dir=data_dir)
825 assert definition["last_computed_at"] # stamped, non-blank
826
827
828def test_process_rollups_skips_a_definition_not_yet_due(tmp_path):
829 data_dir = tmp_path / "data"
830 write_orders_schema(data_dir)
831 now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
832 _install_rollup_definitions(data_dir, [
833 _rollup_definition_row(last_computed_at=now, refresh_interval_seconds="3600"),
834 ])
835
836 result = object_daemon.process_rollups(base_dir=data_dir)
837
838 assert result is not None
839 assert result["checked"] == 1
840 assert result["computed"] == []
841
842
843def test_process_rollups_skips_a_disabled_definition(tmp_path):
844 data_dir = tmp_path / "data"
845 write_orders_schema(data_dir)
846 _install_rollup_definitions(data_dir, [_rollup_definition_row(enabled="false")])
847
848 result = object_daemon.process_rollups(base_dir=data_dir)
849
850 assert result is not None
851 assert result["computed"] == []
852
853
854def test_process_rollups_one_bad_definition_does_not_stop_others(tmp_path):
855 data_dir = tmp_path / "data"
856 write_orders_schema(data_dir)
857 object_records.create_collection_record(
858 "orders", {"id": "o1", "channel": "web", "status": "paid", "total_cents": "1000",
859 "created_at": "2026-07-01T00:00:00Z"},
860 base_dir=data_dir, roots=[],
861 )
862 _install_rollup_definitions(data_dir, [
863 _rollup_definition_row(id="bad", target_collection="rollup_bad", source_collection="no_such_collection"),
864 _rollup_definition_row(id="good", target_collection="rollup_good"),
865 ])
866
867 result = object_daemon.process_rollups(base_dir=data_dir)
868
869 assert result is not None
870 assert result["checked"] == 2
871 assert [c["definition_id"] for c in result["computed"]] == ["good"]
872 good_rows = object_records.read_collection_records("rollup_good", base_dir=data_dir, roots=[])
873 assert len(good_rows) == 1
874 # The bad definition's own last_computed_at is never stamped -- it
875 # never successfully computed.
876 bad_definition = object_records.get_collection_record("rollup_definitions", "bad", base_dir=data_dir)
877 assert bad_definition["last_computed_at"] == ""
878
879
880def test_process_rollups_returns_none_when_flag_off(tmp_path):
881 data_dir = tmp_path / "data"
882 write_orders_schema(data_dir)
883 write_schema(data_dir, "feature_flags", [
884 {"name": "id"}, {"name": "flag", "type": "text"}, {"name": "value", "type": "text"},
885 ])
886 object_records.create_collection_record(
887 "feature_flags", {"id": "f1", "flag": "rollup_enabled", "value": "off"}, base_dir=data_dir, roots=[],
888 )
889 _install_rollup_definitions(data_dir, [_rollup_definition_row()])
890
891 result = object_daemon.process_rollups(base_dir=data_dir)
892
893 assert result is None
894 # Untouched: no target collection was ever created.
895 assert not (data_dir / "collections" / "rollup_orders_by_day").exists()
896
897
898def test_process_rollups_returns_none_without_rollup_definitions_collection(tmp_path):
899 data_dir = tmp_path / "data"
900 result = object_daemon.process_rollups(base_dir=data_dir)
901 assert result is None
902
903
904# --- Materialize (plan/vocabulary/61-materialize-spec.md) ------------------
905
906def _materialize_definition_row(**overrides) -> dict:
907 row = {
908 "id": "matgen_fin_recurring",
909 "name": "Recurring journal generation",
910 "source_collection": "fin_recurring",
911 "source_filter": json.dumps({"is_active": "true"}),
912 "trigger": json.dumps({
913 "mode": "scheduled", "interval_seconds": 3600,
914 "period_field": "frequency", "anchor_field": "next_run",
915 }),
916 "output_collection": "fin_journals",
917 "child_collection": "fin_journal_lines",
918 "child_source_field": "template_lines",
919 "child_link_field": "journal_id",
920 "idempotency_key": "matgen_{definition_id}_{source_id}_{period_start}",
921 "mapping": json.dumps({
922 "date": {"from_period": "period_start"},
923 "description": {"literal": "Recurring"},
924 "status": {"literal": "draft"},
925 "currency": {"literal": "USD"},
926 }),
927 "balance_check": json.dumps({"debit_field": "debit_cents", "credit_field": "credit_cents"}),
928 "last_run_at": "",
929 "actor": "daemon:materialize",
930 "enabled": "true",
931 "block": "false",
932 }
933 row.update(overrides)
934 return row
935
936
937def _install_materialize_fixtures(data_dir: Path, definitions: list[dict], recurring_rows: list[dict]) -> None:
938 write_schema(data_dir, "materialize_definitions", [
939 {"name": "id"},
940 {"name": "name", "type": "text"},
941 {"name": "source_collection", "type": "text"},
942 {"name": "source_filter", "type": "textarea"},
943 {"name": "trigger", "type": "textarea"},
944 {"name": "output_collection", "type": "text"},
945 {"name": "child_collection", "type": "text"},
946 {"name": "child_source_field", "type": "text"},
947 {"name": "child_link_field", "type": "text"},
948 {"name": "idempotency_key", "type": "text"},
949 {"name": "mapping", "type": "textarea"},
950 {"name": "balance_check", "type": "textarea"},
951 {"name": "debit_account_id", "type": "text"},
952 {"name": "credit_account_id", "type": "text"},
953 {"name": "actor", "type": "text"},
954 {"name": "last_run_at", "type": "datetime", "read_only": True},
955 {"name": "enabled", "type": "boolean"},
956 {"name": "block", "type": "boolean"},
957 ])
958 write_schema(data_dir, "fin_recurring", [
959 {"name": "id"},
960 {"name": "name", "type": "text"},
961 {"name": "template_lines", "type": "textarea"},
962 {"name": "frequency", "type": "text"},
963 {"name": "next_run", "type": "date"},
964 {"name": "auto_post", "type": "boolean"},
965 {"name": "is_active", "type": "boolean"},
966 ])
967 write_schema(data_dir, "fin_journals", [
968 {"name": "id"}, {"name": "date", "type": "date"}, {"name": "description", "type": "text"},
969 {"name": "status", "type": "text"}, {"name": "currency", "type": "text"},
970 ])
971 write_append_schema(data_dir, "fin_journal_lines", [
972 {"name": "id"}, {"name": "journal_id", "type": "text"}, {"name": "account_id", "type": "text"},
973 {"name": "debit_cents", "type": "integer"}, {"name": "credit_cents", "type": "integer"},
974 ])
975
976 for row in recurring_rows:
977 object_records.create_collection_record("fin_recurring", row, base_dir=data_dir, roots=[], actor="tester")
978 for row in definitions:
979 object_records.create_collection_record(
980 "materialize_definitions", row, base_dir=data_dir, roots=[], actor="tester",
981 preserve_read_only=True,
982 )
983
984
985def _balanced_recurring_row(**overrides) -> dict:
986 row = {
987 "id": "rec1", "name": "Rent", "frequency": "monthly", "next_run": "2020-01-01",
988 "auto_post": "false", "is_active": "true",
989 "template_lines": json.dumps([
990 {"account_id": "acct_cash", "debit_cents": 100, "credit_cents": 0},
991 {"account_id": "acct_rev", "debit_cents": 0, "credit_cents": 100},
992 ]),
993 }
994 row.update(overrides)
995 return row
996
997
998def test_process_materializations_generates_a_due_definition_and_stamps_last_run_at(tmp_path):
999 data_dir = tmp_path / "data"
1000 _install_materialize_fixtures(data_dir, [_materialize_definition_row()], [_balanced_recurring_row()])
1001
1002 result = object_daemon.process_materializations(base_dir=data_dir)
1003
1004 assert result is not None
1005 assert result["checked"] == 1
1006 assert result["generated"] >= 1
1007
1008 journals = object_records.read_collection_records("fin_journals", base_dir=data_dir, roots=[])
1009 assert len(journals) >= 1
1010
1011 definition = object_records.get_collection_record(
1012 "materialize_definitions", "matgen_fin_recurring", base_dir=data_dir,
1013 )
1014 assert definition["last_run_at"]
1015
1016
1017def test_process_materializations_skips_a_disabled_definition(tmp_path):
1018 data_dir = tmp_path / "data"
1019 _install_materialize_fixtures(
1020 data_dir, [_materialize_definition_row(enabled="false")], [_balanced_recurring_row()],
1021 )
1022
1023 result = object_daemon.process_materializations(base_dir=data_dir)
1024
1025 assert result["generated"] == 0
1026 assert object_records.read_collection_records("fin_journals", base_dir=data_dir, roots=[]) == []
1027
1028
1029def test_process_materializations_refuses_a_blocked_definition(tmp_path):
1030 data_dir = tmp_path / "data"
1031 _install_materialize_fixtures(
1032 data_dir, [_materialize_definition_row(block="true")], [_balanced_recurring_row()],
1033 )
1034
1035 result = object_daemon.process_materializations(base_dir=data_dir)
1036
1037 assert result["generated"] == 0
1038 assert object_records.read_collection_records("fin_journals", base_dir=data_dir, roots=[]) == []
1039
1040
1041def test_process_materializations_skips_event_mode_definitions(tmp_path):
1042 """The scheduled daemon pass never drives an event-mode definition --
1043 that's materialize_seed's dispatch / materialize_run's job."""
1044 data_dir = tmp_path / "data"
1045 event_definition = _materialize_definition_row(
1046 id="matgen_event", source_collection="fin_recurring", output_collection="fin_recurring",
1047 trigger=json.dumps({"mode": "event", "on": "record.created"}),
1048 child_collection="", child_source_field="", child_link_field="",
1049 idempotency_key="{definition_id}_{source_id}", mapping=json.dumps({}), balance_check="",
1050 )
1051 _install_materialize_fixtures(data_dir, [event_definition], [_balanced_recurring_row()])
1052
1053 result = object_daemon.process_materializations(base_dir=data_dir)
1054
1055 assert result["checked"] == 1
1056 assert result["generated"] == 0
1057
1058
1059def test_process_materializations_one_bad_definition_does_not_stop_others(tmp_path):
1060 data_dir = tmp_path / "data"
1061 bad = _materialize_definition_row(id="bad", source_collection="no_such_collection")
1062 good = _materialize_definition_row(id="good")
1063 _install_materialize_fixtures(data_dir, [bad, good], [_balanced_recurring_row()])
1064
1065 result = object_daemon.process_materializations(base_dir=data_dir)
1066
1067 assert result["checked"] == 2
1068 assert result["generated"] >= 1
1069 good_definition = object_records.get_collection_record("materialize_definitions", "good", base_dir=data_dir)
1070 assert good_definition["last_run_at"]
1071 bad_definition = object_records.get_collection_record("materialize_definitions", "bad", base_dir=data_dir)
1072 assert not bad_definition["last_run_at"] # never reached the stamp step
1073
1074
1075def test_process_materializations_returns_none_when_flag_off(tmp_path):
1076 data_dir = tmp_path / "data"
1077 write_schema(data_dir, "feature_flags", [
1078 {"name": "id"}, {"name": "flag", "type": "text"}, {"name": "value", "type": "text"},
1079 ])
1080 object_records.create_collection_record(
1081 "feature_flags", {"id": "f1", "flag": "materialize_enabled", "value": "off"}, base_dir=data_dir, roots=[],
1082 )
1083 _install_materialize_fixtures(data_dir, [_materialize_definition_row()], [_balanced_recurring_row()])
1084
1085 result = object_daemon.process_materializations(base_dir=data_dir)
1086
1087 assert result is None
1088 assert object_records.read_collection_records("fin_journals", base_dir=data_dir, roots=[]) == []
1089
1090
1091def test_process_materializations_returns_none_without_materialize_definitions_collection(tmp_path):
1092 data_dir = tmp_path / "data"
1093 result = object_daemon.process_materializations(base_dir=data_dir)
1094 assert result is None
1095
1096
1097def test_process_materializations_honors_interval_marker(tmp_path, monkeypatch):
1098 monkeypatch.setenv("DBBASIC_MATERIALIZE_INTERVAL_SECONDS", "3600")
1099 data_dir = tmp_path / "data"
1100 _install_materialize_fixtures(data_dir, [_materialize_definition_row()], [_balanced_recurring_row()])
1101
1102 first = object_daemon.process_materializations(base_dir=data_dir)
1103 assert first is not None
1104 assert first["generated"] >= 1
1105 journal_count = len(object_records.read_collection_records("fin_journals", base_dir=data_dir, roots=[]))
1106
1107 # Add another due row and call again immediately: the marker was just
1108 # written, so the interval hasn't elapsed and this call must be a
1109 # pure no-op (nothing new generated).
1110 object_records.create_collection_record(
1111 "fin_recurring", _balanced_recurring_row(id="rec2", name="Rent 2"), base_dir=data_dir, roots=[],
1112 )
1113 second = object_daemon.process_materializations(base_dir=data_dir)
1114 assert second is None
1115 assert len(object_records.read_collection_records("fin_journals", base_dir=data_dir, roots=[])) == journal_count
1116
1117
1118def test_process_materializations_no_op_run_still_returns_zeroed_summary(tmp_path):
1119 data_dir = tmp_path / "data"
1120 _install_materialize_fixtures(data_dir, [], [])
1121
1122 result = object_daemon.process_materializations(base_dir=data_dir)
1123
1124 assert result == {"checked": 0, "generated": 0, "skipped_already_generated": 0, "results": []}
1125
1126
1127def test_process_materializations_second_call_skips_already_generated(tmp_path, monkeypatch):
1128 # DBBASIC_MATERIALIZE_INTERVAL_SECONDS gates the PASS's own marker file;
1129 # is_definition_due (the per-definition gate) reads back last_run_at vs.
1130 # the definition's own trigger.interval_seconds (3600 in this fixture),
1131 # so after the first call stamps last_run_at to "now" the definition
1132 # would look "not due yet" on an immediate second call -- backdate it
1133 # to isolate what this test actually checks: a due-again definition
1134 # whose periods were already generated reports skips, not new rows.
1135 monkeypatch.setenv("DBBASIC_MATERIALIZE_INTERVAL_SECONDS", "0")
1136 data_dir = tmp_path / "data"
1137 _install_materialize_fixtures(data_dir, [_materialize_definition_row()], [_balanced_recurring_row()])
1138
1139 first = object_daemon.process_materializations(base_dir=data_dir)
1140 assert first["generated"] >= 1
1141
1142 object_records.update_collection_record(
1143 "materialize_definitions", "matgen_fin_recurring",
1144 {"last_run_at": "2000-01-01T00:00:00Z"}, base_dir=data_dir, actor="tester", preserve_read_only=True,
1145 )
1146
1147 second = object_daemon.process_materializations(base_dir=data_dir)
1148 assert second["generated"] == 0
1149 assert second["skipped_already_generated"] >= 1
1150
1151
1152def test_daemon_entrypoint_is_runnable_without_optional_runtime():
1153 """The daemon's main() must start on a bare install: no
1154 dbbasic_object_core, no croniter -- the storage passes (compaction,
1155 auto-transitions, cleanups) are stdlib and must not be held hostage by
1156 an optional runtime import. --help exercises the full import path and
1157 argparse wiring without entering the loop. (This exact failure shipped:
1158 main() hard-imported the optional runtime and had never been run.)"""
1159 import subprocess
1160 import sys
1161
1162 result = subprocess.run(
1163 [sys.executable, "object_daemon.py", "--help"],
1164 capture_output=True,
1165 text=True,
1166 timeout=30,
1167 cwd=Path(__file__).resolve().parent.parent,
1168 )
1169 assert result.returncode == 0, result.stderr
1170 assert "Object Primitive Daemon" in result.stdout