askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_object_daemon.py · 44.0 KB · 1170 lines
| 1 | import importlib.util |
| 2 | import json |
| 3 | import sys |
| 4 | import time |
| 5 | from datetime import datetime, timedelta, timezone |
| 6 | from pathlib import Path |
| 7 | |
| 8 | import object_record_changes |
| 9 | import object_records |
| 10 | |
| 11 | ROOT = Path(__file__).resolve().parents[1] |
| 12 | spec = importlib.util.spec_from_file_location("object_daemon", ROOT / "object_daemon.py") |
| 13 | object_daemon = importlib.util.module_from_spec(spec) |
| 14 | sys.modules["object_daemon"] = object_daemon |
| 15 | assert spec.loader is not None |
| 16 | spec.loader.exec_module(object_daemon) |
| 17 | |
| 18 | |
| 19 | def write_append_schema(data_dir: Path, collection: str, fields: list[dict] | None = None) -> Path: |
| 20 | path = data_dir / "schemas" / f"{collection}.json" |
| 21 | path.parent.mkdir(parents=True, exist_ok=True) |
| 22 | path.write_text(json.dumps({"fields": fields or [{"name": "id"}], "storage": "append"})) |
| 23 | return path |
| 24 | |
| 25 | |
| 26 | class FakeStateManager: |
| 27 | def __init__(self): |
| 28 | self.values = {} |
| 29 | |
| 30 | def reload(self): |
| 31 | return None |
| 32 | |
| 33 | def get_all(self): |
| 34 | return dict(self.values) |
| 35 | |
| 36 | def get(self, key): |
| 37 | return self.values.get(key) |
| 38 | |
| 39 | def set(self, key, value): |
| 40 | self.values[key] = value |
| 41 | |
| 42 | |
| 43 | class FakeObject: |
| 44 | def __init__(self): |
| 45 | self.state_manager = FakeStateManager() |
| 46 | self.calls = [] |
| 47 | |
| 48 | def execute(self, method, payload): |
| 49 | self.calls.append((method, payload)) |
| 50 | return {"status": "ok"} |
| 51 | |
| 52 | |
| 53 | class FailingObject(FakeObject): |
| 54 | def execute(self, method, payload): |
| 55 | self.calls.append((method, payload)) |
| 56 | raise RuntimeError("target failed") |
| 57 | |
| 58 | |
| 59 | class FakeRuntime: |
| 60 | def __init__(self): |
| 61 | self.objects = {} |
| 62 | self.loaded = [] |
| 63 | |
| 64 | def add(self, object_id, obj): |
| 65 | self.objects[object_id] = obj |
| 66 | return obj |
| 67 | |
| 68 | def load_object(self, path, object_id=None): |
| 69 | resolved_id = object_id or Path(path).stem |
| 70 | self.loaded.append((resolved_id, Path(path))) |
| 71 | return self.objects[resolved_id] |
| 72 | |
| 73 | |
| 74 | def write_object(path): |
| 75 | path.parent.mkdir(parents=True, exist_ok=True) |
| 76 | path.write_text("def GET(request):\n return {'status': 'ok'}\n") |
| 77 | return path |
| 78 | |
| 79 | |
| 80 | def test_daemon_imports_without_runtime_package(): |
| 81 | assert callable(object_daemon.process_queue) |
| 82 | assert callable(object_daemon.process_events) |
| 83 | assert callable(object_daemon.cleanup_ratelimit) |
| 84 | assert callable(object_daemon.cleanup_events) |
| 85 | |
| 86 | |
| 87 | def test_calculate_onetime_next_run(): |
| 88 | task = {"type": "onetime", "schedule": "2026-01-02T03:04:05Z"} |
| 89 | |
| 90 | next_run = object_daemon._calculate_next_run(task) |
| 91 | |
| 92 | assert isinstance(next_run, int) |
| 93 | assert next_run > 0 |
| 94 | |
| 95 | |
| 96 | def test_cleanup_ratelimit_removes_expired_and_corrupt_files(tmp_path, monkeypatch): |
| 97 | monkeypatch.chdir(tmp_path) |
| 98 | ratelimit_dir = tmp_path / "data" / "ratelimit" |
| 99 | ratelimit_dir.mkdir(parents=True) |
| 100 | |
| 101 | now = time.time() |
| 102 | expired = ratelimit_dir / "expired.txt" |
| 103 | recent = ratelimit_dir / "recent.txt" |
| 104 | corrupt = ratelimit_dir / "corrupt.txt" |
| 105 | ignored = ratelimit_dir / "ignored.log" |
| 106 | |
| 107 | expired.write_text(f"{now - 500}\n") |
| 108 | recent.write_text(f"{now}\n") |
| 109 | corrupt.write_text("not-a-timestamp\n") |
| 110 | ignored.write_text(f"{now - 500}\n") |
| 111 | |
| 112 | object_daemon.cleanup_ratelimit(max_age=120) |
| 113 | |
| 114 | assert not expired.exists() |
| 115 | assert recent.exists() |
| 116 | assert not corrupt.exists() |
| 117 | assert ignored.exists() |
| 118 | |
| 119 | |
| 120 | def test_cleanup_events_prunes_event_queue(tmp_path): |
| 121 | data_dir = tmp_path / "data" |
| 122 | object_daemon.object_events.publish_event( |
| 123 | "test.event", |
| 124 | payload={"id": "first"}, |
| 125 | base_dir=data_dir, |
| 126 | ) |
| 127 | object_daemon.object_events.publish_event( |
| 128 | "test.event", |
| 129 | payload={"id": "second"}, |
| 130 | base_dir=data_dir, |
| 131 | ) |
| 132 | |
| 133 | result = object_daemon.cleanup_events(base_dir=data_dir, keep_count=1, keep_seconds=0) |
| 134 | |
| 135 | assert result["deleted"] == 1 |
| 136 | assert object_daemon.object_events.list_events(base_dir=data_dir)["total"] == 1 |
| 137 | |
| 138 | |
| 139 | def test_process_queue_completes_pending_message(tmp_path, monkeypatch): |
| 140 | monkeypatch.chdir(tmp_path) |
| 141 | write_object(tmp_path / "objects" / "triggers" / "queue.py") |
| 142 | write_object(tmp_path / "objects" / "basics" / "target.py") |
| 143 | |
| 144 | runtime = FakeRuntime() |
| 145 | queue = runtime.add("queue", FakeObject()) |
| 146 | target = runtime.add("basics_target", FakeObject()) |
| 147 | |
| 148 | now = int(time.time()) |
| 149 | key = "msg_tasks_2_1_msg_001" |
| 150 | queue.state_manager.set( |
| 151 | key, |
| 152 | json.dumps( |
| 153 | { |
| 154 | "id": "msg_001", |
| 155 | "queue_name": "tasks", |
| 156 | "message": { |
| 157 | "object_id": "basics_target", |
| 158 | "method": "POST", |
| 159 | "payload": {"action": "test"}, |
| 160 | }, |
| 161 | "priority_level": 2, |
| 162 | "status": "pending", |
| 163 | "created_at": now - 10, |
| 164 | "visible_after": now - 10, |
| 165 | "expires_at": now + 3600, |
| 166 | "attempts": 0, |
| 167 | "max_attempts": 3, |
| 168 | } |
| 169 | ), |
| 170 | ) |
| 171 | |
| 172 | object_daemon.process_queue(runtime) |
| 173 | |
| 174 | updated = json.loads(queue.state_manager.get(key)) |
| 175 | assert updated["status"] == "completed" |
| 176 | assert target.calls == [("POST", {"action": "test"})] |
| 177 | |
| 178 | |
| 179 | def test_process_queue_marks_expired_message(tmp_path, monkeypatch): |
| 180 | monkeypatch.chdir(tmp_path) |
| 181 | write_object(tmp_path / "objects" / "triggers" / "queue.py") |
| 182 | |
| 183 | runtime = FakeRuntime() |
| 184 | queue = runtime.add("queue", FakeObject()) |
| 185 | |
| 186 | now = int(time.time()) |
| 187 | key = "msg_tasks_2_1_expired" |
| 188 | queue.state_manager.set( |
| 189 | key, |
| 190 | json.dumps( |
| 191 | { |
| 192 | "id": "expired", |
| 193 | "queue_name": "tasks", |
| 194 | "message": {"object_id": "basics_target"}, |
| 195 | "priority_level": 2, |
| 196 | "status": "pending", |
| 197 | "created_at": now - 100, |
| 198 | "visible_after": now - 100, |
| 199 | "expires_at": now - 1, |
| 200 | "attempts": 0, |
| 201 | "max_attempts": 3, |
| 202 | } |
| 203 | ), |
| 204 | ) |
| 205 | |
| 206 | object_daemon.process_queue(runtime) |
| 207 | |
| 208 | updated = json.loads(queue.state_manager.get(key)) |
| 209 | assert updated["status"] == "expired" |
| 210 | |
| 211 | |
| 212 | def test_find_object_file_honors_configured_objects_dir(tmp_path, monkeypatch): |
| 213 | monkeypatch.chdir(tmp_path) |
| 214 | custom_root = tmp_path / "custom_objects" |
| 215 | custom_path = write_object(custom_root / "basics" / "counter.py") |
| 216 | write_object(tmp_path / "objects" / "basics" / "counter.py") |
| 217 | monkeypatch.setenv("DBBASIC_OBJECTS_DIR", str(custom_root)) |
| 218 | |
| 219 | found = object_daemon._find_object_file("basics_counter") |
| 220 | |
| 221 | assert found == custom_path |
| 222 | |
| 223 | |
| 224 | def test_find_object_file_resolves_user_object_from_objects_dir(tmp_path, monkeypatch): |
| 225 | monkeypatch.chdir(tmp_path) |
| 226 | user_path = write_object(tmp_path / "objects" / "users" / "42" / "deals.py") |
| 227 | |
| 228 | found = object_daemon._find_object_file("u_42_deals") |
| 229 | |
| 230 | assert found is not None |
| 231 | assert found.resolve() == user_path.resolve() |
| 232 | |
| 233 | |
| 234 | def test_process_queue_requeues_failed_message(tmp_path, monkeypatch): |
| 235 | monkeypatch.chdir(tmp_path) |
| 236 | write_object(tmp_path / "objects" / "triggers" / "queue.py") |
| 237 | write_object(tmp_path / "objects" / "basics" / "target.py") |
| 238 | |
| 239 | runtime = FakeRuntime() |
| 240 | queue = runtime.add("queue", FakeObject()) |
| 241 | runtime.add("basics_target", FailingObject()) |
| 242 | |
| 243 | now = int(time.time()) |
| 244 | key = "msg_tasks_2_1_retry" |
| 245 | queue.state_manager.set( |
| 246 | key, |
| 247 | json.dumps( |
| 248 | { |
| 249 | "id": "retry", |
| 250 | "queue_name": "tasks", |
| 251 | "message": {"object_id": "basics_target", "method": "POST", "payload": {}}, |
| 252 | "priority_level": 2, |
| 253 | "status": "pending", |
| 254 | "created_at": now - 10, |
| 255 | "visible_after": now - 10, |
| 256 | "expires_at": now + 3600, |
| 257 | "attempts": 0, |
| 258 | "max_attempts": 3, |
| 259 | } |
| 260 | ), |
| 261 | ) |
| 262 | |
| 263 | object_daemon.process_queue(runtime) |
| 264 | |
| 265 | updated = json.loads(queue.state_manager.get(key)) |
| 266 | assert updated["status"] == "pending" |
| 267 | assert updated["attempts"] == 1 |
| 268 | assert updated["visible_after"] > now |
| 269 | |
| 270 | |
| 271 | def test_process_events_delivers_matching_event(tmp_path, monkeypatch): |
| 272 | monkeypatch.chdir(tmp_path) |
| 273 | write_object(tmp_path / "objects" / "triggers" / "events.py") |
| 274 | |
| 275 | runtime = FakeRuntime() |
| 276 | events = runtime.add("events", FakeObject()) |
| 277 | |
| 278 | now = int(time.time()) |
| 279 | events.state_manager.set( |
| 280 | "sub_test.event_sub_001", |
| 281 | json.dumps( |
| 282 | { |
| 283 | "id": "sub_001", |
| 284 | "event_type": "test.event", |
| 285 | "callback_url": "http://127.0.0.1:9999/hook", |
| 286 | "last_event_id": None, |
| 287 | } |
| 288 | ), |
| 289 | ) |
| 290 | events.state_manager.set( |
| 291 | f"event_{now}_evt_001", |
| 292 | json.dumps( |
| 293 | { |
| 294 | "id": "evt_001", |
| 295 | "event_type": "test.event", |
| 296 | "payload": {"value": 1}, |
| 297 | "timestamp": now, |
| 298 | } |
| 299 | ), |
| 300 | ) |
| 301 | |
| 302 | delivered = [] |
| 303 | |
| 304 | class FakeResponse: |
| 305 | def getcode(self): |
| 306 | return 204 |
| 307 | |
| 308 | def __enter__(self): |
| 309 | return self |
| 310 | |
| 311 | def __exit__(self, exc_type, exc, tb): |
| 312 | return False |
| 313 | |
| 314 | def fake_urlopen(request, timeout): |
| 315 | delivered.append((request, timeout)) |
| 316 | return FakeResponse() |
| 317 | |
| 318 | monkeypatch.setattr(object_daemon.urllib.request, "urlopen", fake_urlopen) |
| 319 | |
| 320 | object_daemon.process_events(runtime) |
| 321 | |
| 322 | assert len(delivered) == 1 |
| 323 | request, timeout = delivered[0] |
| 324 | assert request.full_url == "http://127.0.0.1:9999/hook" |
| 325 | assert timeout == 5 |
| 326 | updated = json.loads(events.state_manager.get("sub_test.event_sub_001")) |
| 327 | assert updated["last_event_id"] == "evt_001" |
| 328 | assert updated["delivery"]["status"] == "ok" |
| 329 | assert updated["delivery"]["attempts"] == 1 |
| 330 | assert updated["delivery"]["successes"] == 1 |
| 331 | assert updated["delivery"]["last_status_code"] == 204 |
| 332 | |
| 333 | |
| 334 | def test_process_events_records_failure_without_advancing_cursor(tmp_path, monkeypatch): |
| 335 | monkeypatch.chdir(tmp_path) |
| 336 | write_object(tmp_path / "objects" / "triggers" / "events.py") |
| 337 | |
| 338 | runtime = FakeRuntime() |
| 339 | events = runtime.add("events", FakeObject()) |
| 340 | |
| 341 | now = int(time.time()) |
| 342 | events.state_manager.set( |
| 343 | "sub_test.event_sub_001", |
| 344 | json.dumps( |
| 345 | { |
| 346 | "id": "sub_001", |
| 347 | "event_type": "test.event", |
| 348 | "callback_url": "http://127.0.0.1:9999/hook", |
| 349 | "last_event_id": None, |
| 350 | } |
| 351 | ), |
| 352 | ) |
| 353 | for index in (1, 2): |
| 354 | events.state_manager.set( |
| 355 | f"event_{now + index}_evt_00{index}", |
| 356 | json.dumps( |
| 357 | { |
| 358 | "id": f"evt_00{index}", |
| 359 | "event_type": "test.event", |
| 360 | "payload": {"value": index}, |
| 361 | "timestamp": now + index, |
| 362 | } |
| 363 | ), |
| 364 | ) |
| 365 | |
| 366 | attempts = [] |
| 367 | |
| 368 | def fake_urlopen(request, timeout): |
| 369 | attempts.append((request, timeout)) |
| 370 | raise object_daemon.urllib.error.URLError("callback down") |
| 371 | |
| 372 | monkeypatch.setattr(object_daemon.urllib.request, "urlopen", fake_urlopen) |
| 373 | |
| 374 | object_daemon.process_events(runtime) |
| 375 | |
| 376 | assert len(attempts) == 1 |
| 377 | updated = json.loads(events.state_manager.get("sub_test.event_sub_001")) |
| 378 | assert updated["last_event_id"] is None |
| 379 | assert updated["delivery"]["status"] == "failed" |
| 380 | assert updated["delivery"]["attempts"] == 1 |
| 381 | assert updated["delivery"]["failures"] == 1 |
| 382 | assert updated["delivery"]["last_attempted_event_id"] == "evt_001" |
| 383 | assert "callback down" in updated["delivery"]["last_error"] |
| 384 | |
| 385 | |
| 386 | # --- Compaction pass (docs/append-only-storage-design.md Compaction) -------- |
| 387 | |
| 388 | |
| 389 | def _seed_bloated_append_collection(data_dir: Path, collection: str, *, live: int = 1, churn: int = 5) -> None: |
| 390 | """Create `collection` in append storage, then churn one record |
| 391 | `churn` times so physical rows pile up well past `live`.""" |
| 392 | write_append_schema(data_dir, collection, [{"name": "id"}, {"name": "value"}]) |
| 393 | for i in range(live): |
| 394 | object_records.create_collection_record( |
| 395 | collection, {"id": f"r{i}", "value": "v0"}, base_dir=data_dir, roots=[] |
| 396 | ) |
| 397 | for i in range(churn): |
| 398 | object_records.update_collection_record( |
| 399 | collection, "r0", {"value": f"v{i}"}, base_dir=data_dir, roots=[] |
| 400 | ) |
| 401 | |
| 402 | |
| 403 | def test_process_compactions_compacts_collections_over_threshold(tmp_path, monkeypatch): |
| 404 | monkeypatch.setenv("DBBASIC_COMPACTION_INTERVAL_SECONDS", "0") |
| 405 | monkeypatch.setenv("DBBASIC_APPEND_COMPACT_MIN_ROWS", "3") |
| 406 | monkeypatch.setenv("DBBASIC_COMPACTION_BLOAT_RATIO", "1.0") |
| 407 | data_dir = tmp_path / "data" |
| 408 | |
| 409 | # "hot": 1 live + 5 updates = 6 physical rows, 1 live -> well over both |
| 410 | # the row floor and the bloat ratio. |
| 411 | _seed_bloated_append_collection(data_dir, "hot", live=1, churn=5) |
| 412 | # "cold": 2 live rows, no churn -> under DBBASIC_APPEND_COMPACT_MIN_ROWS |
| 413 | # even though (trivially) bloat_ratio is 0, so it must be left alone. |
| 414 | write_append_schema(data_dir, "cold", [{"name": "id"}, {"name": "value"}]) |
| 415 | object_records.create_collection_record("cold", {"id": "c1", "value": "v"}, base_dir=data_dir, roots=[]) |
| 416 | object_records.create_collection_record("cold", {"id": "c2", "value": "v"}, base_dir=data_dir, roots=[]) |
| 417 | |
| 418 | hot_path = object_records.collection_records_file("hot", base_dir=data_dir) |
| 419 | cold_path = object_records.collection_records_file("cold", base_dir=data_dir) |
| 420 | hot_rows_before = len(hot_path.read_text().splitlines()) - 1 |
| 421 | cold_rows_before = len(cold_path.read_text().splitlines()) - 1 |
| 422 | assert hot_rows_before == 6 |
| 423 | assert cold_rows_before == 2 |
| 424 | |
| 425 | result = object_daemon.process_compactions(base_dir=data_dir) |
| 426 | |
| 427 | assert result is not None |
| 428 | assert [entry["collection"] for entry in result["compacted"]] == ["hot"] |
| 429 | |
| 430 | hot_rows_after = len(hot_path.read_text().splitlines()) - 1 |
| 431 | cold_rows_after = len(cold_path.read_text().splitlines()) - 1 |
| 432 | assert hot_rows_after == 1 # compacted down to its one live record |
| 433 | assert cold_rows_after == cold_rows_before # untouched: under the row floor |
| 434 | |
| 435 | |
| 436 | def test_process_compactions_honors_interval_marker(tmp_path, monkeypatch): |
| 437 | monkeypatch.setenv("DBBASIC_APPEND_COMPACT_MIN_ROWS", "3") |
| 438 | monkeypatch.setenv("DBBASIC_COMPACTION_BLOAT_RATIO", "1.0") |
| 439 | monkeypatch.setenv("DBBASIC_COMPACTION_INTERVAL_SECONDS", "3600") |
| 440 | data_dir = tmp_path / "data" |
| 441 | _seed_bloated_append_collection(data_dir, "hot", live=1, churn=5) |
| 442 | path = object_records.collection_records_file("hot", base_dir=data_dir) |
| 443 | |
| 444 | # First call: no marker yet -> runs and compacts. |
| 445 | first = object_daemon.process_compactions(base_dir=data_dir) |
| 446 | assert first is not None |
| 447 | assert [entry["collection"] for entry in first["compacted"]] == ["hot"] |
| 448 | rows_after_first = len(path.read_text().splitlines()) - 1 |
| 449 | assert rows_after_first == 1 |
| 450 | |
| 451 | # Re-bloat the file directly (bypassing the pass) and call again |
| 452 | # immediately: the marker was just written, so the interval hasn't |
| 453 | # elapsed and this call must be a pure no-op -- no listing, no compaction. |
| 454 | for i in range(5): |
| 455 | object_records.update_collection_record( |
| 456 | "hot", "r0", {"value": f"w{i}"}, base_dir=data_dir, roots=[] |
| 457 | ) |
| 458 | rows_before_second = len(path.read_text().splitlines()) - 1 |
| 459 | assert rows_before_second > 1 |
| 460 | |
| 461 | second = object_daemon.process_compactions(base_dir=data_dir) |
| 462 | assert second is None |
| 463 | rows_after_second = len(path.read_text().splitlines()) - 1 |
| 464 | assert rows_after_second == rows_before_second # untouched |
| 465 | |
| 466 | |
| 467 | def test_process_compactions_one_collection_failing_does_not_stop_others(tmp_path, monkeypatch): |
| 468 | monkeypatch.setenv("DBBASIC_COMPACTION_INTERVAL_SECONDS", "0") |
| 469 | monkeypatch.setenv("DBBASIC_APPEND_COMPACT_MIN_ROWS", "3") |
| 470 | monkeypatch.setenv("DBBASIC_COMPACTION_BLOAT_RATIO", "1.0") |
| 471 | data_dir = tmp_path / "data" |
| 472 | _seed_bloated_append_collection(data_dir, "bad", live=1, churn=5) |
| 473 | _seed_bloated_append_collection(data_dir, "good", live=1, churn=5) |
| 474 | |
| 475 | real_compact = object_records.compact_collection |
| 476 | |
| 477 | def flaky_compact(collection, **kwargs): |
| 478 | if collection == "bad": |
| 479 | raise RuntimeError("simulated compaction failure") |
| 480 | return real_compact(collection, **kwargs) |
| 481 | |
| 482 | monkeypatch.setattr(object_records, "compact_collection", flaky_compact) |
| 483 | |
| 484 | result = object_daemon.process_compactions(base_dir=data_dir) |
| 485 | |
| 486 | assert result is not None |
| 487 | assert result["checked"] == 2 |
| 488 | assert [entry["collection"] for entry in result["compacted"]] == ["good"] |
| 489 | |
| 490 | good_path = object_records.collection_records_file("good", base_dir=data_dir) |
| 491 | bad_path = object_records.collection_records_file("bad", base_dir=data_dir) |
| 492 | assert len(good_path.read_text().splitlines()) - 1 == 1 # compacted |
| 493 | assert len(bad_path.read_text().splitlines()) - 1 == 6 # left as-is after its failure |
| 494 | |
| 495 | |
| 496 | # --- Stale-state auto-transition pass ---------------------------------- |
| 497 | |
| 498 | |
| 499 | def write_tasks_schema(data_dir: Path) -> Path: |
| 500 | """A trimmed stand-in for packages/app-tasks/schemas/tasks.json: same |
| 501 | status enum/transitions and the same lone `created_at` timestamp |
| 502 | field (no `updated_at`), classic (non-append) storage.""" |
| 503 | path = data_dir / "schemas" / "tasks.json" |
| 504 | path.parent.mkdir(parents=True, exist_ok=True) |
| 505 | path.write_text(json.dumps({ |
| 506 | "fields": [ |
| 507 | {"name": "id"}, |
| 508 | {"name": "title", "type": "text", "required": True}, |
| 509 | { |
| 510 | "name": "status", |
| 511 | "type": "enum", |
| 512 | "default": "open", |
| 513 | "enum": ["draft", "open", "assigned", "waiting_on_client", |
| 514 | "approved", "disputed", "cancelled"], |
| 515 | "transitions": { |
| 516 | "draft": ["open", "assigned", "cancelled"], |
| 517 | "open": ["assigned", "cancelled"], |
| 518 | "assigned": ["waiting_on_client", "open", "cancelled"], |
| 519 | "waiting_on_client": ["approved", "disputed", "assigned"], |
| 520 | "disputed": ["assigned", "cancelled"], |
| 521 | }, |
| 522 | }, |
| 523 | {"name": "created_at", "type": "datetime", "read_only": True}, |
| 524 | ], |
| 525 | })) |
| 526 | return path |
| 527 | |
| 528 | |
| 529 | def _backdate_change_log(data_dir: Path, collection: str, record_id: str, *, hours_ago: float) -> None: |
| 530 | """Rewrite every record-change log entry for `record_id` to look |
| 531 | `hours_ago` old, so a record created "now" can still exercise the |
| 532 | stale-transition age check without sleeping in a test.""" |
| 533 | path = object_record_changes.record_changes_file(collection, base_dir=data_dir) |
| 534 | past = (datetime.now(timezone.utc) - timedelta(hours=hours_ago)).isoformat() |
| 535 | lines = path.read_text(encoding="utf-8").splitlines() |
| 536 | rewritten = [] |
| 537 | for line in lines: |
| 538 | entry = json.loads(line) |
| 539 | if entry.get("record_id") == record_id: |
| 540 | entry["timestamp"] = past |
| 541 | rewritten.append(json.dumps(entry)) |
| 542 | path.write_text("\n".join(rewritten) + "\n", encoding="utf-8") |
| 543 | |
| 544 | |
| 545 | def test_auto_transition_rules_from_env_defaults_to_tasks_rule(monkeypatch): |
| 546 | # Unscheduled by default was exactly the predecessor bug (200+ records |
| 547 | # stranded in waiting_on_client) -- the default here must be a real, |
| 548 | # working rule, not an empty list a caller has to opt into. |
| 549 | monkeypatch.delenv("DBBASIC_AUTO_TRANSITION_RULES", raising=False) |
| 550 | rules = object_daemon.auto_transition_rules_from_env() |
| 551 | assert rules == [{ |
| 552 | "collection": "tasks", |
| 553 | "field": "status", |
| 554 | "from": "waiting_on_client", |
| 555 | "to": "approved", |
| 556 | "after_hours": 48.0, |
| 557 | }] |
| 558 | |
| 559 | |
| 560 | def test_auto_transition_rules_from_env_empty_string_disables(monkeypatch): |
| 561 | monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_RULES", "") |
| 562 | assert object_daemon.auto_transition_rules_from_env() == [] |
| 563 | monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_RULES", "[]") |
| 564 | assert object_daemon.auto_transition_rules_from_env() == [] |
| 565 | |
| 566 | |
| 567 | def test_process_stale_transitions_moves_stale_record_with_daemon_actor(tmp_path, monkeypatch): |
| 568 | monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS", "0") |
| 569 | data_dir = tmp_path / "data" |
| 570 | write_tasks_schema(data_dir) |
| 571 | object_records.create_collection_record( |
| 572 | "tasks", |
| 573 | {"id": "t1", "title": "Stale task", "status": "waiting_on_client"}, |
| 574 | base_dir=data_dir, |
| 575 | roots=[], |
| 576 | actor="tester", |
| 577 | ) |
| 578 | _backdate_change_log(data_dir, "tasks", "t1", hours_ago=50) |
| 579 | |
| 580 | result = object_daemon.process_stale_transitions(base_dir=data_dir) |
| 581 | |
| 582 | assert result is not None |
| 583 | assert result["checked"] == 1 |
| 584 | assert result["moved"] == 1 |
| 585 | assert result["rules"][0]["collection"] == "tasks" |
| 586 | |
| 587 | current = object_records.get_collection_record("tasks", "t1", base_dir=data_dir) |
| 588 | assert current["status"] == "approved" |
| 589 | |
| 590 | changes = object_record_changes.list_record_changes("tasks", record_id="t1", base_dir=data_dir) |
| 591 | latest = changes["changes"][0] |
| 592 | assert latest["action"] == "update" |
| 593 | assert latest["actor"] == "daemon:auto-transition" |
| 594 | assert latest["after"]["status"] == "approved" |
| 595 | |
| 596 | |
| 597 | def test_process_stale_transitions_leaves_fresh_record_alone(tmp_path, monkeypatch): |
| 598 | monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS", "0") |
| 599 | data_dir = tmp_path / "data" |
| 600 | write_tasks_schema(data_dir) |
| 601 | object_records.create_collection_record( |
| 602 | "tasks", |
| 603 | {"id": "t1", "title": "Fresh task", "status": "waiting_on_client"}, |
| 604 | base_dir=data_dir, |
| 605 | roots=[], |
| 606 | actor="tester", |
| 607 | ) |
| 608 | # No backdating: the record just entered waiting_on_client. |
| 609 | |
| 610 | result = object_daemon.process_stale_transitions(base_dir=data_dir) |
| 611 | |
| 612 | assert result is not None |
| 613 | assert result["moved"] == 0 |
| 614 | current = object_records.get_collection_record("tasks", "t1", base_dir=data_dir) |
| 615 | assert current["status"] == "waiting_on_client" |
| 616 | |
| 617 | |
| 618 | def test_process_stale_transitions_disabled_by_empty_rules(tmp_path, monkeypatch): |
| 619 | monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS", "0") |
| 620 | monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_RULES", "[]") |
| 621 | data_dir = tmp_path / "data" |
| 622 | write_tasks_schema(data_dir) |
| 623 | object_records.create_collection_record( |
| 624 | "tasks", |
| 625 | {"id": "t1", "title": "Stale task", "status": "waiting_on_client"}, |
| 626 | base_dir=data_dir, |
| 627 | roots=[], |
| 628 | actor="tester", |
| 629 | ) |
| 630 | _backdate_change_log(data_dir, "tasks", "t1", hours_ago=200) |
| 631 | |
| 632 | result = object_daemon.process_stale_transitions(base_dir=data_dir) |
| 633 | |
| 634 | assert result == {"checked": 0, "moved": 0, "rules": []} |
| 635 | current = object_records.get_collection_record("tasks", "t1", base_dir=data_dir) |
| 636 | assert current["status"] == "waiting_on_client" # untouched |
| 637 | |
| 638 | |
| 639 | def test_process_stale_transitions_skips_unknown_collection(tmp_path, monkeypatch): |
| 640 | monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS", "0") |
| 641 | monkeypatch.setenv( |
| 642 | "DBBASIC_AUTO_TRANSITION_RULES", |
| 643 | json.dumps([{ |
| 644 | "collection": "not_installed", |
| 645 | "field": "status", |
| 646 | "from": "waiting_on_client", |
| 647 | "to": "approved", |
| 648 | "after_hours": 48, |
| 649 | }]), |
| 650 | ) |
| 651 | data_dir = tmp_path / "data" # no schema, no records -- package "not installed" |
| 652 | |
| 653 | result = object_daemon.process_stale_transitions(base_dir=data_dir) |
| 654 | |
| 655 | assert result == { |
| 656 | "checked": 1, |
| 657 | "moved": 0, |
| 658 | "rules": [{ |
| 659 | "collection": "not_installed", |
| 660 | "field": "status", |
| 661 | "from": "waiting_on_client", |
| 662 | "to": "approved", |
| 663 | "moved": 0, |
| 664 | }], |
| 665 | } |
| 666 | |
| 667 | |
| 668 | def test_process_stale_transitions_skips_record_that_moved_on(tmp_path, monkeypatch): |
| 669 | """A record staged as a stale candidate by the pass's own listing, but |
| 670 | whose live value has already moved off `from` by the time the pass |
| 671 | gets to writing it (a race between the listing and the per-record |
| 672 | write), must be left alone -- not force-moved back to `to`.""" |
| 673 | monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS", "0") |
| 674 | data_dir = tmp_path / "data" |
| 675 | write_tasks_schema(data_dir) |
| 676 | object_records.create_collection_record( |
| 677 | "tasks", |
| 678 | {"id": "t1", "title": "Racing task", "status": "waiting_on_client"}, |
| 679 | base_dir=data_dir, |
| 680 | roots=[], |
| 681 | actor="tester", |
| 682 | ) |
| 683 | _backdate_change_log(data_dir, "tasks", "t1", hours_ago=50) |
| 684 | |
| 685 | stale_snapshot = object_records.read_collection_records("tasks", base_dir=data_dir) |
| 686 | |
| 687 | # The record moves on for real before the pass's write reaches it. |
| 688 | object_records.update_collection_record( |
| 689 | "tasks", "t1", {"status": "assigned"}, base_dir=data_dir, roots=[], actor="tester", |
| 690 | ) |
| 691 | |
| 692 | monkeypatch.setattr( |
| 693 | object_records, |
| 694 | "read_collection_records", |
| 695 | lambda collection, **kwargs: stale_snapshot if collection == "tasks" else [], |
| 696 | ) |
| 697 | |
| 698 | result = object_daemon.process_stale_transitions(base_dir=data_dir) |
| 699 | |
| 700 | assert result["moved"] == 0 |
| 701 | current = object_records.get_collection_record("tasks", "t1", base_dir=data_dir) |
| 702 | assert current["status"] == "assigned" # not force-moved to "approved" |
| 703 | |
| 704 | |
| 705 | def test_process_stale_transitions_honors_interval_marker(tmp_path, monkeypatch): |
| 706 | monkeypatch.setenv("DBBASIC_AUTO_TRANSITION_INTERVAL_SECONDS", "3600") |
| 707 | data_dir = tmp_path / "data" |
| 708 | write_tasks_schema(data_dir) |
| 709 | object_records.create_collection_record( |
| 710 | "tasks", |
| 711 | {"id": "t1", "title": "Stale task", "status": "waiting_on_client"}, |
| 712 | base_dir=data_dir, |
| 713 | roots=[], |
| 714 | actor="tester", |
| 715 | ) |
| 716 | _backdate_change_log(data_dir, "tasks", "t1", hours_ago=50) |
| 717 | |
| 718 | first = object_daemon.process_stale_transitions(base_dir=data_dir) |
| 719 | assert first is not None |
| 720 | assert first["moved"] == 1 |
| 721 | |
| 722 | # Put another stale record in place directly (bypassing the pass) and |
| 723 | # call again immediately: the marker was just written, so the interval |
| 724 | # hasn't elapsed and this call must be a pure no-op. |
| 725 | object_records.create_collection_record( |
| 726 | "tasks", |
| 727 | {"id": "t2", "title": "Also stale", "status": "waiting_on_client"}, |
| 728 | base_dir=data_dir, |
| 729 | roots=[], |
| 730 | actor="tester", |
| 731 | ) |
| 732 | _backdate_change_log(data_dir, "tasks", "t2", hours_ago=50) |
| 733 | |
| 734 | second = object_daemon.process_stale_transitions(base_dir=data_dir) |
| 735 | assert second is None |
| 736 | current = object_records.get_collection_record("tasks", "t2", base_dir=data_dir) |
| 737 | assert current["status"] == "waiting_on_client" # untouched |
| 738 | |
| 739 | |
| 740 | def write_orders_schema(data_dir: Path) -> Path: |
| 741 | return write_schema(data_dir, "orders", [ |
| 742 | {"name": "id"}, |
| 743 | {"name": "channel", "type": "text"}, |
| 744 | {"name": "status", "type": "text"}, |
| 745 | {"name": "total_cents", "type": "integer"}, |
| 746 | {"name": "created_at", "type": "datetime"}, |
| 747 | ]) |
| 748 | |
| 749 | |
| 750 | def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 751 | path = data_dir / "schemas" / f"{collection}.json" |
| 752 | path.parent.mkdir(parents=True, exist_ok=True) |
| 753 | path.write_text(json.dumps({"name": collection, "fields": fields})) |
| 754 | return path |
| 755 | |
| 756 | |
| 757 | def _rollup_definition_row(**overrides) -> dict: |
| 758 | row = { |
| 759 | "id": "rollup_orders_by_day", |
| 760 | "name": "Orders per day", |
| 761 | "source_collection": "orders", |
| 762 | "filter": json.dumps({"status": "paid"}), |
| 763 | "group_by": json.dumps(["channel"]), |
| 764 | "time_bucket": json.dumps({"field": "created_at", "granularity": "day"}), |
| 765 | "metrics": json.dumps([{"op": "count", "as": "order_count"}]), |
| 766 | "target_collection": "rollup_orders_by_day", |
| 767 | "min_group_size": "", |
| 768 | "refresh_mode": "scheduled", |
| 769 | "refresh_interval_seconds": "3600", |
| 770 | "last_computed_at": "", |
| 771 | "enabled": "true", |
| 772 | } |
| 773 | row.update(overrides) |
| 774 | return row |
| 775 | |
| 776 | |
| 777 | def _install_rollup_definitions(data_dir: Path, rows: list[dict]) -> None: |
| 778 | write_schema(data_dir, "rollup_definitions", [ |
| 779 | {"name": "id"}, |
| 780 | {"name": "name", "type": "text"}, |
| 781 | {"name": "source_collection", "type": "text"}, |
| 782 | {"name": "filter", "type": "textarea"}, |
| 783 | {"name": "group_by", "type": "textarea"}, |
| 784 | {"name": "time_bucket", "type": "textarea"}, |
| 785 | {"name": "metrics", "type": "textarea"}, |
| 786 | {"name": "target_collection", "type": "text"}, |
| 787 | {"name": "min_group_size", "type": "integer"}, |
| 788 | {"name": "refresh_mode", "type": "text"}, |
| 789 | {"name": "refresh_interval_seconds", "type": "integer"}, |
| 790 | {"name": "last_computed_at", "type": "datetime", "read_only": True}, |
| 791 | {"name": "enabled", "type": "boolean"}, |
| 792 | ]) |
| 793 | for row in rows: |
| 794 | # preserve_read_only: seeding a fixture row directly, same posture |
| 795 | # as object_import.py replaying another system's history -- a real |
| 796 | # admin-authored create would omit last_computed_at entirely (the |
| 797 | # generated form never renders a read_only field), never submit it. |
| 798 | object_records.create_collection_record( |
| 799 | "rollup_definitions", row, base_dir=data_dir, roots=[], actor="tester", |
| 800 | preserve_read_only=True, |
| 801 | ) |
| 802 | |
| 803 | |
| 804 | def test_process_rollups_computes_a_due_definition_and_stamps_last_computed_at(tmp_path): |
| 805 | data_dir = tmp_path / "data" |
| 806 | write_orders_schema(data_dir) |
| 807 | object_records.create_collection_record( |
| 808 | "orders", {"id": "o1", "channel": "web", "status": "paid", "total_cents": "1000", |
| 809 | "created_at": "2026-07-01T00:00:00Z"}, |
| 810 | base_dir=data_dir, roots=[], |
| 811 | ) |
| 812 | _install_rollup_definitions(data_dir, [_rollup_definition_row()]) |
| 813 | |
| 814 | result = object_daemon.process_rollups(base_dir=data_dir) |
| 815 | |
| 816 | assert result is not None |
| 817 | assert result["checked"] == 1 |
| 818 | assert [c["definition_id"] for c in result["computed"]] == ["rollup_orders_by_day"] |
| 819 | |
| 820 | target_rows = object_records.read_collection_records("rollup_orders_by_day", base_dir=data_dir, roots=[]) |
| 821 | assert len(target_rows) == 1 |
| 822 | assert target_rows[0]["order_count"] == "1" |
| 823 | |
| 824 | definition = object_records.get_collection_record("rollup_definitions", "rollup_orders_by_day", base_dir=data_dir) |
| 825 | assert definition["last_computed_at"] # stamped, non-blank |
| 826 | |
| 827 | |
| 828 | def test_process_rollups_skips_a_definition_not_yet_due(tmp_path): |
| 829 | data_dir = tmp_path / "data" |
| 830 | write_orders_schema(data_dir) |
| 831 | now = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") |
| 832 | _install_rollup_definitions(data_dir, [ |
| 833 | _rollup_definition_row(last_computed_at=now, refresh_interval_seconds="3600"), |
| 834 | ]) |
| 835 | |
| 836 | result = object_daemon.process_rollups(base_dir=data_dir) |
| 837 | |
| 838 | assert result is not None |
| 839 | assert result["checked"] == 1 |
| 840 | assert result["computed"] == [] |
| 841 | |
| 842 | |
| 843 | def test_process_rollups_skips_a_disabled_definition(tmp_path): |
| 844 | data_dir = tmp_path / "data" |
| 845 | write_orders_schema(data_dir) |
| 846 | _install_rollup_definitions(data_dir, [_rollup_definition_row(enabled="false")]) |
| 847 | |
| 848 | result = object_daemon.process_rollups(base_dir=data_dir) |
| 849 | |
| 850 | assert result is not None |
| 851 | assert result["computed"] == [] |
| 852 | |
| 853 | |
| 854 | def test_process_rollups_one_bad_definition_does_not_stop_others(tmp_path): |
| 855 | data_dir = tmp_path / "data" |
| 856 | write_orders_schema(data_dir) |
| 857 | object_records.create_collection_record( |
| 858 | "orders", {"id": "o1", "channel": "web", "status": "paid", "total_cents": "1000", |
| 859 | "created_at": "2026-07-01T00:00:00Z"}, |
| 860 | base_dir=data_dir, roots=[], |
| 861 | ) |
| 862 | _install_rollup_definitions(data_dir, [ |
| 863 | _rollup_definition_row(id="bad", target_collection="rollup_bad", source_collection="no_such_collection"), |
| 864 | _rollup_definition_row(id="good", target_collection="rollup_good"), |
| 865 | ]) |
| 866 | |
| 867 | result = object_daemon.process_rollups(base_dir=data_dir) |
| 868 | |
| 869 | assert result is not None |
| 870 | assert result["checked"] == 2 |
| 871 | assert [c["definition_id"] for c in result["computed"]] == ["good"] |
| 872 | good_rows = object_records.read_collection_records("rollup_good", base_dir=data_dir, roots=[]) |
| 873 | assert len(good_rows) == 1 |
| 874 | # The bad definition's own last_computed_at is never stamped -- it |
| 875 | # never successfully computed. |
| 876 | bad_definition = object_records.get_collection_record("rollup_definitions", "bad", base_dir=data_dir) |
| 877 | assert bad_definition["last_computed_at"] == "" |
| 878 | |
| 879 | |
| 880 | def test_process_rollups_returns_none_when_flag_off(tmp_path): |
| 881 | data_dir = tmp_path / "data" |
| 882 | write_orders_schema(data_dir) |
| 883 | write_schema(data_dir, "feature_flags", [ |
| 884 | {"name": "id"}, {"name": "flag", "type": "text"}, {"name": "value", "type": "text"}, |
| 885 | ]) |
| 886 | object_records.create_collection_record( |
| 887 | "feature_flags", {"id": "f1", "flag": "rollup_enabled", "value": "off"}, base_dir=data_dir, roots=[], |
| 888 | ) |
| 889 | _install_rollup_definitions(data_dir, [_rollup_definition_row()]) |
| 890 | |
| 891 | result = object_daemon.process_rollups(base_dir=data_dir) |
| 892 | |
| 893 | assert result is None |
| 894 | # Untouched: no target collection was ever created. |
| 895 | assert not (data_dir / "collections" / "rollup_orders_by_day").exists() |
| 896 | |
| 897 | |
| 898 | def test_process_rollups_returns_none_without_rollup_definitions_collection(tmp_path): |
| 899 | data_dir = tmp_path / "data" |
| 900 | result = object_daemon.process_rollups(base_dir=data_dir) |
| 901 | assert result is None |
| 902 | |
| 903 | |
| 904 | # --- Materialize (plan/vocabulary/61-materialize-spec.md) ------------------ |
| 905 | |
| 906 | def _materialize_definition_row(**overrides) -> dict: |
| 907 | row = { |
| 908 | "id": "matgen_fin_recurring", |
| 909 | "name": "Recurring journal generation", |
| 910 | "source_collection": "fin_recurring", |
| 911 | "source_filter": json.dumps({"is_active": "true"}), |
| 912 | "trigger": json.dumps({ |
| 913 | "mode": "scheduled", "interval_seconds": 3600, |
| 914 | "period_field": "frequency", "anchor_field": "next_run", |
| 915 | }), |
| 916 | "output_collection": "fin_journals", |
| 917 | "child_collection": "fin_journal_lines", |
| 918 | "child_source_field": "template_lines", |
| 919 | "child_link_field": "journal_id", |
| 920 | "idempotency_key": "matgen_{definition_id}_{source_id}_{period_start}", |
| 921 | "mapping": json.dumps({ |
| 922 | "date": {"from_period": "period_start"}, |
| 923 | "description": {"literal": "Recurring"}, |
| 924 | "status": {"literal": "draft"}, |
| 925 | "currency": {"literal": "USD"}, |
| 926 | }), |
| 927 | "balance_check": json.dumps({"debit_field": "debit_cents", "credit_field": "credit_cents"}), |
| 928 | "last_run_at": "", |
| 929 | "actor": "daemon:materialize", |
| 930 | "enabled": "true", |
| 931 | "block": "false", |
| 932 | } |
| 933 | row.update(overrides) |
| 934 | return row |
| 935 | |
| 936 | |
| 937 | def _install_materialize_fixtures(data_dir: Path, definitions: list[dict], recurring_rows: list[dict]) -> None: |
| 938 | write_schema(data_dir, "materialize_definitions", [ |
| 939 | {"name": "id"}, |
| 940 | {"name": "name", "type": "text"}, |
| 941 | {"name": "source_collection", "type": "text"}, |
| 942 | {"name": "source_filter", "type": "textarea"}, |
| 943 | {"name": "trigger", "type": "textarea"}, |
| 944 | {"name": "output_collection", "type": "text"}, |
| 945 | {"name": "child_collection", "type": "text"}, |
| 946 | {"name": "child_source_field", "type": "text"}, |
| 947 | {"name": "child_link_field", "type": "text"}, |
| 948 | {"name": "idempotency_key", "type": "text"}, |
| 949 | {"name": "mapping", "type": "textarea"}, |
| 950 | {"name": "balance_check", "type": "textarea"}, |
| 951 | {"name": "debit_account_id", "type": "text"}, |
| 952 | {"name": "credit_account_id", "type": "text"}, |
| 953 | {"name": "actor", "type": "text"}, |
| 954 | {"name": "last_run_at", "type": "datetime", "read_only": True}, |
| 955 | {"name": "enabled", "type": "boolean"}, |
| 956 | {"name": "block", "type": "boolean"}, |
| 957 | ]) |
| 958 | write_schema(data_dir, "fin_recurring", [ |
| 959 | {"name": "id"}, |
| 960 | {"name": "name", "type": "text"}, |
| 961 | {"name": "template_lines", "type": "textarea"}, |
| 962 | {"name": "frequency", "type": "text"}, |
| 963 | {"name": "next_run", "type": "date"}, |
| 964 | {"name": "auto_post", "type": "boolean"}, |
| 965 | {"name": "is_active", "type": "boolean"}, |
| 966 | ]) |
| 967 | write_schema(data_dir, "fin_journals", [ |
| 968 | {"name": "id"}, {"name": "date", "type": "date"}, {"name": "description", "type": "text"}, |
| 969 | {"name": "status", "type": "text"}, {"name": "currency", "type": "text"}, |
| 970 | ]) |
| 971 | write_append_schema(data_dir, "fin_journal_lines", [ |
| 972 | {"name": "id"}, {"name": "journal_id", "type": "text"}, {"name": "account_id", "type": "text"}, |
| 973 | {"name": "debit_cents", "type": "integer"}, {"name": "credit_cents", "type": "integer"}, |
| 974 | ]) |
| 975 | |
| 976 | for row in recurring_rows: |
| 977 | object_records.create_collection_record("fin_recurring", row, base_dir=data_dir, roots=[], actor="tester") |
| 978 | for row in definitions: |
| 979 | object_records.create_collection_record( |
| 980 | "materialize_definitions", row, base_dir=data_dir, roots=[], actor="tester", |
| 981 | preserve_read_only=True, |
| 982 | ) |
| 983 | |
| 984 | |
| 985 | def _balanced_recurring_row(**overrides) -> dict: |
| 986 | row = { |
| 987 | "id": "rec1", "name": "Rent", "frequency": "monthly", "next_run": "2020-01-01", |
| 988 | "auto_post": "false", "is_active": "true", |
| 989 | "template_lines": json.dumps([ |
| 990 | {"account_id": "acct_cash", "debit_cents": 100, "credit_cents": 0}, |
| 991 | {"account_id": "acct_rev", "debit_cents": 0, "credit_cents": 100}, |
| 992 | ]), |
| 993 | } |
| 994 | row.update(overrides) |
| 995 | return row |
| 996 | |
| 997 | |
| 998 | def test_process_materializations_generates_a_due_definition_and_stamps_last_run_at(tmp_path): |
| 999 | data_dir = tmp_path / "data" |
| 1000 | _install_materialize_fixtures(data_dir, [_materialize_definition_row()], [_balanced_recurring_row()]) |
| 1001 | |
| 1002 | result = object_daemon.process_materializations(base_dir=data_dir) |
| 1003 | |
| 1004 | assert result is not None |
| 1005 | assert result["checked"] == 1 |
| 1006 | assert result["generated"] >= 1 |
| 1007 | |
| 1008 | journals = object_records.read_collection_records("fin_journals", base_dir=data_dir, roots=[]) |
| 1009 | assert len(journals) >= 1 |
| 1010 | |
| 1011 | definition = object_records.get_collection_record( |
| 1012 | "materialize_definitions", "matgen_fin_recurring", base_dir=data_dir, |
| 1013 | ) |
| 1014 | assert definition["last_run_at"] |
| 1015 | |
| 1016 | |
| 1017 | def test_process_materializations_skips_a_disabled_definition(tmp_path): |
| 1018 | data_dir = tmp_path / "data" |
| 1019 | _install_materialize_fixtures( |
| 1020 | data_dir, [_materialize_definition_row(enabled="false")], [_balanced_recurring_row()], |
| 1021 | ) |
| 1022 | |
| 1023 | result = object_daemon.process_materializations(base_dir=data_dir) |
| 1024 | |
| 1025 | assert result["generated"] == 0 |
| 1026 | assert object_records.read_collection_records("fin_journals", base_dir=data_dir, roots=[]) == [] |
| 1027 | |
| 1028 | |
| 1029 | def test_process_materializations_refuses_a_blocked_definition(tmp_path): |
| 1030 | data_dir = tmp_path / "data" |
| 1031 | _install_materialize_fixtures( |
| 1032 | data_dir, [_materialize_definition_row(block="true")], [_balanced_recurring_row()], |
| 1033 | ) |
| 1034 | |
| 1035 | result = object_daemon.process_materializations(base_dir=data_dir) |
| 1036 | |
| 1037 | assert result["generated"] == 0 |
| 1038 | assert object_records.read_collection_records("fin_journals", base_dir=data_dir, roots=[]) == [] |
| 1039 | |
| 1040 | |
| 1041 | def test_process_materializations_skips_event_mode_definitions(tmp_path): |
| 1042 | """The scheduled daemon pass never drives an event-mode definition -- |
| 1043 | that's materialize_seed's dispatch / materialize_run's job.""" |
| 1044 | data_dir = tmp_path / "data" |
| 1045 | event_definition = _materialize_definition_row( |
| 1046 | id="matgen_event", source_collection="fin_recurring", output_collection="fin_recurring", |
| 1047 | trigger=json.dumps({"mode": "event", "on": "record.created"}), |
| 1048 | child_collection="", child_source_field="", child_link_field="", |
| 1049 | idempotency_key="{definition_id}_{source_id}", mapping=json.dumps({}), balance_check="", |
| 1050 | ) |
| 1051 | _install_materialize_fixtures(data_dir, [event_definition], [_balanced_recurring_row()]) |
| 1052 | |
| 1053 | result = object_daemon.process_materializations(base_dir=data_dir) |
| 1054 | |
| 1055 | assert result["checked"] == 1 |
| 1056 | assert result["generated"] == 0 |
| 1057 | |
| 1058 | |
| 1059 | def test_process_materializations_one_bad_definition_does_not_stop_others(tmp_path): |
| 1060 | data_dir = tmp_path / "data" |
| 1061 | bad = _materialize_definition_row(id="bad", source_collection="no_such_collection") |
| 1062 | good = _materialize_definition_row(id="good") |
| 1063 | _install_materialize_fixtures(data_dir, [bad, good], [_balanced_recurring_row()]) |
| 1064 | |
| 1065 | result = object_daemon.process_materializations(base_dir=data_dir) |
| 1066 | |
| 1067 | assert result["checked"] == 2 |
| 1068 | assert result["generated"] >= 1 |
| 1069 | good_definition = object_records.get_collection_record("materialize_definitions", "good", base_dir=data_dir) |
| 1070 | assert good_definition["last_run_at"] |
| 1071 | bad_definition = object_records.get_collection_record("materialize_definitions", "bad", base_dir=data_dir) |
| 1072 | assert not bad_definition["last_run_at"] # never reached the stamp step |
| 1073 | |
| 1074 | |
| 1075 | def test_process_materializations_returns_none_when_flag_off(tmp_path): |
| 1076 | data_dir = tmp_path / "data" |
| 1077 | write_schema(data_dir, "feature_flags", [ |
| 1078 | {"name": "id"}, {"name": "flag", "type": "text"}, {"name": "value", "type": "text"}, |
| 1079 | ]) |
| 1080 | object_records.create_collection_record( |
| 1081 | "feature_flags", {"id": "f1", "flag": "materialize_enabled", "value": "off"}, base_dir=data_dir, roots=[], |
| 1082 | ) |
| 1083 | _install_materialize_fixtures(data_dir, [_materialize_definition_row()], [_balanced_recurring_row()]) |
| 1084 | |
| 1085 | result = object_daemon.process_materializations(base_dir=data_dir) |
| 1086 | |
| 1087 | assert result is None |
| 1088 | assert object_records.read_collection_records("fin_journals", base_dir=data_dir, roots=[]) == [] |
| 1089 | |
| 1090 | |
| 1091 | def test_process_materializations_returns_none_without_materialize_definitions_collection(tmp_path): |
| 1092 | data_dir = tmp_path / "data" |
| 1093 | result = object_daemon.process_materializations(base_dir=data_dir) |
| 1094 | assert result is None |
| 1095 | |
| 1096 | |
| 1097 | def test_process_materializations_honors_interval_marker(tmp_path, monkeypatch): |
| 1098 | monkeypatch.setenv("DBBASIC_MATERIALIZE_INTERVAL_SECONDS", "3600") |
| 1099 | data_dir = tmp_path / "data" |
| 1100 | _install_materialize_fixtures(data_dir, [_materialize_definition_row()], [_balanced_recurring_row()]) |
| 1101 | |
| 1102 | first = object_daemon.process_materializations(base_dir=data_dir) |
| 1103 | assert first is not None |
| 1104 | assert first["generated"] >= 1 |
| 1105 | journal_count = len(object_records.read_collection_records("fin_journals", base_dir=data_dir, roots=[])) |
| 1106 | |
| 1107 | # Add another due row and call again immediately: the marker was just |
| 1108 | # written, so the interval hasn't elapsed and this call must be a |
| 1109 | # pure no-op (nothing new generated). |
| 1110 | object_records.create_collection_record( |
| 1111 | "fin_recurring", _balanced_recurring_row(id="rec2", name="Rent 2"), base_dir=data_dir, roots=[], |
| 1112 | ) |
| 1113 | second = object_daemon.process_materializations(base_dir=data_dir) |
| 1114 | assert second is None |
| 1115 | assert len(object_records.read_collection_records("fin_journals", base_dir=data_dir, roots=[])) == journal_count |
| 1116 | |
| 1117 | |
| 1118 | def test_process_materializations_no_op_run_still_returns_zeroed_summary(tmp_path): |
| 1119 | data_dir = tmp_path / "data" |
| 1120 | _install_materialize_fixtures(data_dir, [], []) |
| 1121 | |
| 1122 | result = object_daemon.process_materializations(base_dir=data_dir) |
| 1123 | |
| 1124 | assert result == {"checked": 0, "generated": 0, "skipped_already_generated": 0, "results": []} |
| 1125 | |
| 1126 | |
| 1127 | def test_process_materializations_second_call_skips_already_generated(tmp_path, monkeypatch): |
| 1128 | # DBBASIC_MATERIALIZE_INTERVAL_SECONDS gates the PASS's own marker file; |
| 1129 | # is_definition_due (the per-definition gate) reads back last_run_at vs. |
| 1130 | # the definition's own trigger.interval_seconds (3600 in this fixture), |
| 1131 | # so after the first call stamps last_run_at to "now" the definition |
| 1132 | # would look "not due yet" on an immediate second call -- backdate it |
| 1133 | # to isolate what this test actually checks: a due-again definition |
| 1134 | # whose periods were already generated reports skips, not new rows. |
| 1135 | monkeypatch.setenv("DBBASIC_MATERIALIZE_INTERVAL_SECONDS", "0") |
| 1136 | data_dir = tmp_path / "data" |
| 1137 | _install_materialize_fixtures(data_dir, [_materialize_definition_row()], [_balanced_recurring_row()]) |
| 1138 | |
| 1139 | first = object_daemon.process_materializations(base_dir=data_dir) |
| 1140 | assert first["generated"] >= 1 |
| 1141 | |
| 1142 | object_records.update_collection_record( |
| 1143 | "materialize_definitions", "matgen_fin_recurring", |
| 1144 | {"last_run_at": "2000-01-01T00:00:00Z"}, base_dir=data_dir, actor="tester", preserve_read_only=True, |
| 1145 | ) |
| 1146 | |
| 1147 | second = object_daemon.process_materializations(base_dir=data_dir) |
| 1148 | assert second["generated"] == 0 |
| 1149 | assert second["skipped_already_generated"] >= 1 |
| 1150 | |
| 1151 | |
| 1152 | def test_daemon_entrypoint_is_runnable_without_optional_runtime(): |
| 1153 | """The daemon's main() must start on a bare install: no |
| 1154 | dbbasic_object_core, no croniter -- the storage passes (compaction, |
| 1155 | auto-transitions, cleanups) are stdlib and must not be held hostage by |
| 1156 | an optional runtime import. --help exercises the full import path and |
| 1157 | argparse wiring without entering the loop. (This exact failure shipped: |
| 1158 | main() hard-imported the optional runtime and had never been run.)""" |
| 1159 | import subprocess |
| 1160 | import sys |
| 1161 | |
| 1162 | result = subprocess.run( |
| 1163 | [sys.executable, "object_daemon.py", "--help"], |
| 1164 | capture_output=True, |
| 1165 | text=True, |
| 1166 | timeout=30, |
| 1167 | cwd=Path(__file__).resolve().parent.parent, |
| 1168 | ) |
| 1169 | assert result.returncode == 0, result.stderr |
| 1170 | assert "Object Primitive Daemon" in result.stdout |