askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_object_connectors.py · 9.9 KB · 212 lines
| 1 | """03 external connectors: the generic reconcile engine (sync lifecycle, |
| 2 | retry/backoff planning, dynamic loading), manifest discovery, and the daemon |
| 3 | pass end to end against a real (fake) connector package. |
| 4 | """ |
| 5 | |
| 6 | import json |
| 7 | from datetime import datetime, timezone |
| 8 | from pathlib import Path |
| 9 | |
| 10 | import object_connectors |
| 11 | import object_daemon |
| 12 | import object_packages |
| 13 | import object_records |
| 14 | |
| 15 | NOW = datetime(2026, 7, 21, 12, 0, tzinfo=timezone.utc) |
| 16 | |
| 17 | |
| 18 | # ---- selection + planning (pure) ------------------------------------------ |
| 19 | |
| 20 | def test_is_due_selects_active_statuses_and_respects_backoff(): |
| 21 | assert object_connectors.is_due({"sync_status": "pending"}, NOW) |
| 22 | assert object_connectors.is_due({"sync_status": ""}, NOW) # unset == pending |
| 23 | assert object_connectors.is_due({"sync_status": "pending_delete"}, NOW) |
| 24 | assert not object_connectors.is_due({"sync_status": "synced"}, NOW) |
| 25 | assert not object_connectors.is_due({"sync_status": "deleted"}, NOW) |
| 26 | assert not object_connectors.is_due({"sync_status": "dead"}, NOW) |
| 27 | # backoff gate |
| 28 | assert not object_connectors.is_due({"sync_status": "pending", "sync_next_at": "2026-07-21T12:01:00Z"}, NOW) |
| 29 | assert object_connectors.is_due({"sync_status": "pending", "sync_next_at": "2026-07-21T11:59:00Z"}, NOW) |
| 30 | |
| 31 | |
| 32 | def test_backoff_exponential_capped(): |
| 33 | cfg = object_connectors.ConnectorConfig(retry_base=60, retry_max=3600) |
| 34 | assert object_connectors.backoff_seconds(1, cfg) == 60 |
| 35 | assert object_connectors.backoff_seconds(3, cfg) == 240 |
| 36 | assert object_connectors.backoff_seconds(99, cfg) == 3600 |
| 37 | |
| 38 | |
| 39 | def test_plan_sync_success_create_and_delete(): |
| 40 | cfg = object_connectors.ConnectorConfig() |
| 41 | synced = object_connectors.plan_sync({"sync_status": "pending"}, {"ok": True}, cfg, now=NOW) |
| 42 | assert synced["sync_status"] == "synced" and synced["sync_attempts"] == "0" and synced["sync_error"] == "" |
| 43 | # a successful reconcile of a delete tombstones |
| 44 | deleted = object_connectors.plan_sync({"sync_status": "pending_delete"}, {"ok": True}, cfg, now=NOW) |
| 45 | assert deleted["sync_status"] == "deleted" |
| 46 | |
| 47 | |
| 48 | def test_plan_sync_transient_keeps_direction_and_backs_off(): |
| 49 | cfg = object_connectors.ConnectorConfig(max_attempts=5) |
| 50 | out = object_connectors.plan_sync( |
| 51 | {"sync_status": "pending_delete", "sync_attempts": "0"}, |
| 52 | {"ok": False, "error": "network"}, cfg, now=NOW) |
| 53 | assert out["sync_status"] == "pending_delete" # still a delete target on retry |
| 54 | assert out["sync_attempts"] == "1" and out["sync_error"] == "network" |
| 55 | assert out["sync_next_at"] > "2026-07-21T12:00" |
| 56 | |
| 57 | |
| 58 | def test_plan_sync_permanent_and_exhausted_go_dead(): |
| 59 | cfg = object_connectors.ConnectorConfig(max_attempts=3) |
| 60 | perm = object_connectors.plan_sync({"sync_status": "pending"}, {"ok": False, "permanent": True, "error": "bad"}, cfg, now=NOW) |
| 61 | assert perm["sync_status"] == "dead" |
| 62 | exhausted = object_connectors.plan_sync({"sync_status": "pending", "sync_attempts": "2"}, {"ok": False, "error": "x"}, cfg, now=NOW) |
| 63 | assert exhausted["sync_status"] == "dead" and exhausted["sync_attempts"] == "3" |
| 64 | |
| 65 | |
| 66 | # ---- dynamic loader ------------------------------------------------------- |
| 67 | |
| 68 | def test_load_connector_loads_entry_and_rejects_bad(tmp_path): |
| 69 | mod = tmp_path / "c.py" |
| 70 | mod.write_text("def reconcile(record, *, base_dir):\n return {'ok': True, 'seen': record['id']}\n") |
| 71 | fn = object_connectors.load_connector(mod, "reconcile") |
| 72 | assert fn({"id": "x"}, base_dir="/nope") == {"ok": True, "seen": "x"} |
| 73 | |
| 74 | try: |
| 75 | object_connectors.load_connector(mod, "missing_entry") |
| 76 | assert False, "expected ConnectorLoadError" |
| 77 | except object_connectors.ConnectorLoadError: |
| 78 | pass |
| 79 | try: |
| 80 | object_connectors.load_connector(tmp_path / "nope.py") |
| 81 | assert False, "expected ConnectorLoadError" |
| 82 | except object_connectors.ConnectorLoadError: |
| 83 | pass |
| 84 | |
| 85 | |
| 86 | # ---- manifest discovery --------------------------------------------------- |
| 87 | |
| 88 | RECONCILE_SRC = """ |
| 89 | def reconcile(record, *, base_dir): |
| 90 | name = record.get("name") |
| 91 | if name == "boom": |
| 92 | return {"ok": False, "error": "temporary boom"} |
| 93 | if name == "fatal": |
| 94 | return {"ok": False, "error": "bad request", "permanent": True} |
| 95 | return {"ok": True} |
| 96 | """ |
| 97 | |
| 98 | |
| 99 | def _make_pkg(root, pkg_id, *, collection, reconcile_src=RECONCILE_SRC, module="connectors/fake.py"): |
| 100 | d = root / pkg_id |
| 101 | (d / "connectors").mkdir(parents=True) |
| 102 | (d / "schemas").mkdir(parents=True) |
| 103 | (d / module).write_text(reconcile_src) |
| 104 | (d / "schemas" / f"{collection}.json").write_text(json.dumps({ |
| 105 | "name": collection, "title": collection, "version": 1, |
| 106 | "fields": [ |
| 107 | {"name": "id"}, {"name": "owner_id", "type": "text"}, {"name": "name", "type": "text"}, |
| 108 | {"name": "sync_status", "type": "text"}, {"name": "sync_attempts", "type": "number"}, |
| 109 | {"name": "sync_error", "type": "text"}, {"name": "sync_next_at", "type": "datetime"}, |
| 110 | {"name": "created_at", "type": "datetime", "read_only": True}, |
| 111 | {"name": "updated_at", "type": "datetime", "read_only": True}, |
| 112 | ], |
| 113 | })) |
| 114 | (d / "dbbasic-package.json").write_text(json.dumps({ |
| 115 | "id": pkg_id, "name": pkg_id, "version": "0.1.0", |
| 116 | "compatibility": {"dbbasic_object_server": ">=0.1.0"}, |
| 117 | "objects": [], "schemas": [{"collection": collection, "path": f"schemas/{collection}.json"}], |
| 118 | "permissions": [], "seed": [], "migrations": [], |
| 119 | "connectors": [{"collection": collection, "module": module, "entry": "reconcile"}], |
| 120 | })) |
| 121 | return d |
| 122 | |
| 123 | |
| 124 | def test_manifest_normalizes_connectors(tmp_path): |
| 125 | root = tmp_path / "packages"; root.mkdir() |
| 126 | _make_pkg(root, "app-fake", collection="widgets") |
| 127 | package = object_packages.get_package("app-fake", root=root) |
| 128 | assert package["connectors"] == [{"collection": "widgets", "module": "connectors/fake.py", "entry": "reconcile"}] |
| 129 | assert object_packages._package_summary(package)["connector_count"] == 1 |
| 130 | |
| 131 | |
| 132 | def test_iter_connectors_resolves_module_and_skips_bad(tmp_path): |
| 133 | root = tmp_path / "packages"; root.mkdir() |
| 134 | _make_pkg(root, "app-fake", collection="widgets") |
| 135 | decls = object_packages.iter_connectors(root=root) |
| 136 | assert len(decls) == 1 |
| 137 | decl = decls[0] |
| 138 | assert decl["package_id"] == "app-fake" and decl["collection"] == "widgets" |
| 139 | assert Path(decl["module"]).is_file() and decl["module"].endswith("connectors/fake.py") |
| 140 | |
| 141 | # a declaration whose module file is missing is skipped, not raised |
| 142 | bad = _make_pkg(root, "app-broken", collection="gadgets") |
| 143 | (bad / "connectors" / "fake.py").unlink() |
| 144 | ids = {d["package_id"] for d in object_packages.iter_connectors(root=root)} |
| 145 | assert "app-broken" not in ids and "app-fake" in ids |
| 146 | |
| 147 | |
| 148 | # ---- daemon pass end to end ---------------------------------------------- |
| 149 | |
| 150 | def test_process_connectors_reconciles_each_outcome(tmp_path, monkeypatch): |
| 151 | pkg_root = tmp_path / "packages"; pkg_root.mkdir() |
| 152 | _make_pkg(pkg_root, "app-fake", collection="widgets") |
| 153 | data_dir = tmp_path / "data" |
| 154 | obj = tmp_path / "objects"; obj.mkdir() |
| 155 | object_packages.install_package("app-fake", root=pkg_root, base_dir=data_dir, |
| 156 | object_roots=[obj], allow_replace=True) |
| 157 | |
| 158 | for name in ("ok-one", "boom", "fatal"): |
| 159 | object_records.create_collection_record( |
| 160 | "widgets", {"owner_id": "1", "name": name, "sync_status": "pending"}, |
| 161 | base_dir=data_dir, actor="t") |
| 162 | object_records.create_collection_record( |
| 163 | "widgets", {"owner_id": "1", "name": "gone", "sync_status": "pending_delete"}, |
| 164 | base_dir=data_dir, actor="t") |
| 165 | |
| 166 | monkeypatch.setenv("DBBASIC_PACKAGES_DIR", str(pkg_root)) |
| 167 | monkeypatch.setenv("DBBASIC_PRIVATE_PACKAGES_DIR", str(tmp_path / "no-private")) |
| 168 | |
| 169 | result = object_daemon.process_connectors(base_dir=data_dir) |
| 170 | assert result == {"reconciled": 4, "synced": 2, "dead": 1, "skipped": 0} |
| 171 | |
| 172 | rows = {r["name"]: r for r in object_records.read_collection_records("widgets", base_dir=data_dir)} |
| 173 | assert rows["ok-one"]["sync_status"] == "synced" |
| 174 | assert rows["gone"]["sync_status"] == "deleted" |
| 175 | assert rows["fatal"]["sync_status"] == "dead" |
| 176 | assert rows["boom"]["sync_status"] == "pending" # transient -> stays a target |
| 177 | assert rows["boom"]["sync_attempts"] == "1" |
| 178 | assert "boom" in rows["boom"]["sync_error"] and rows["boom"]["sync_next_at"] |
| 179 | |
| 180 | # re-run: synced/deleted/dead are terminal; only boom is due (its backoff is |
| 181 | # in the future now), so nothing reconciles this tick |
| 182 | assert object_daemon.process_connectors(base_dir=data_dir) == {"reconciled": 0, "synced": 0, "dead": 0, "skipped": 0} |
| 183 | |
| 184 | |
| 185 | SKIP_SRC = """ |
| 186 | def reconcile(record, *, base_dir): |
| 187 | return {"skip": True} # e.g. connector unconfigured |
| 188 | """ |
| 189 | |
| 190 | |
| 191 | def test_process_connectors_skip_leaves_rows_untouched(tmp_path, monkeypatch): |
| 192 | pkg_root = tmp_path / "packages"; pkg_root.mkdir() |
| 193 | _make_pkg(pkg_root, "app-skip", collection="things", reconcile_src=SKIP_SRC) |
| 194 | data_dir = tmp_path / "data" |
| 195 | obj = tmp_path / "objects"; obj.mkdir() |
| 196 | object_packages.install_package("app-skip", root=pkg_root, base_dir=data_dir, |
| 197 | object_roots=[obj], allow_replace=True) |
| 198 | object_records.create_collection_record( |
| 199 | "things", {"owner_id": "1", "name": "x", "sync_status": "pending"}, base_dir=data_dir, actor="t") |
| 200 | monkeypatch.setenv("DBBASIC_PACKAGES_DIR", str(pkg_root)) |
| 201 | monkeypatch.setenv("DBBASIC_PRIVATE_PACKAGES_DIR", str(tmp_path / "no-private")) |
| 202 | |
| 203 | result = object_daemon.process_connectors(base_dir=data_dir) |
| 204 | assert result == {"reconciled": 0, "synced": 0, "dead": 0, "skipped": 1} |
| 205 | row = object_records.read_collection_records("things", base_dir=data_dir)[0] |
| 206 | assert row["sync_status"] == "pending" and (row.get("sync_attempts") or "0") == "0" # untouched |
| 207 | |
| 208 | |
| 209 | def test_process_connectors_none_when_nothing_declared(tmp_path, monkeypatch): |
| 210 | monkeypatch.setenv("DBBASIC_PACKAGES_DIR", str(tmp_path / "empty")) |
| 211 | monkeypatch.setenv("DBBASIC_PRIVATE_PACKAGES_DIR", str(tmp_path / "empty2")) |
| 212 | assert object_daemon.process_connectors(base_dir=tmp_path / "data") is None |