askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_connectors.py · 9.9 KB · 241 lines
| 1 | """External connectors -- reconcile a collection against an outside system. |
| 2 | |
| 3 | The generic half of plan/vocabulary/03-external-connectors-spec.md. A package |
| 4 | declares that one of its collections is kept in sync with an external system |
| 5 | (a mail server, a payment processor, ...) by pointing at a connector module: |
| 6 | |
| 7 | "connectors": [ |
| 8 | {"collection": "email_mailboxes", "module": "connectors/mailcow.py", |
| 9 | "entry": "reconcile"} |
| 10 | ] |
| 11 | |
| 12 | Records are DESIRED STATE; the daemon's process_connectors pass converges the |
| 13 | outside world to match them and writes the outcome back onto each row. This is |
| 14 | the 01 outbox drain pointed outward, and it is the same fold-over-a-collection |
| 15 | every other daemon pass is. Open core never statically imports connector code: |
| 16 | the module is loaded dynamically at runtime, only on a deployment where its |
| 17 | package is installed. |
| 18 | |
| 19 | This module owns the parts that are NOT connector-specific: the sync lifecycle |
| 20 | vocabulary, the pure retry/backoff planning (so every connector inherits the |
| 21 | same at-least-once, dead-lettering behavior), the dynamic loader, and the |
| 22 | feature flag. A connector itself is just a function: |
| 23 | |
| 24 | reconcile(record, *, base_dir) -> dict |
| 25 | # {"ok": True} external world now matches |
| 26 | # {"ok": False, "error": "..."} transient -> retry w/ backoff |
| 27 | # {"ok": False, "error": "...", |
| 28 | # "permanent": True} permanent -> dead, no retry |
| 29 | # {"skip": True} declined this tick (e.g. the |
| 30 | # connector is unconfigured) -> |
| 31 | # row untouched, NOT an attempt, |
| 32 | # so it waits inspectably instead |
| 33 | # of backing off toward dead |
| 34 | |
| 35 | The connector never touches the record's sync_* fields or performs its own |
| 36 | write -- the driver owns the lifecycle, so backoff/attempts/dead-lettering live |
| 37 | in ONE place, not re-implemented per connector. Connectors MUST be idempotent |
| 38 | (create-or-adopt; treat an already-gone object as a done delete): delivery is |
| 39 | at-least-once, so a crash between a successful external call and the status |
| 40 | write re-reconciles next tick. |
| 41 | """ |
| 42 | |
| 43 | from __future__ import annotations |
| 44 | |
| 45 | import importlib.util |
| 46 | from datetime import datetime, timedelta, timezone |
| 47 | from pathlib import Path |
| 48 | from typing import Any, Callable, Mapping |
| 49 | |
| 50 | import object_feature_flags |
| 51 | |
| 52 | # --- sync lifecycle fields + statuses -------------------------------------- |
| 53 | |
| 54 | SYNC_STATUS_FIELD = "sync_status" |
| 55 | SYNC_ATTEMPTS_FIELD = "sync_attempts" |
| 56 | SYNC_ERROR_FIELD = "sync_error" |
| 57 | SYNC_NEXT_AT_FIELD = "sync_next_at" |
| 58 | |
| 59 | # Desired = the external object should EXIST; not yet reconciled. A fresh row or |
| 60 | # a retrying one -- sync_attempts/sync_error tell which. "" is treated as this. |
| 61 | STATUS_PENDING = "pending" |
| 62 | # Desired = the external object should be GONE; not yet removed externally. |
| 63 | STATUS_PENDING_DELETE = "pending_delete" |
| 64 | # Terminal: external world matches (exists). |
| 65 | STATUS_SYNCED = "synced" |
| 66 | # Terminal: external object removed (tombstone -- we keep the row for audit, |
| 67 | # per the spec's lean; a hard purge is a separate, later choice). |
| 68 | STATUS_DELETED = "deleted" |
| 69 | # Terminal: gave up (permanent error, or attempts exhausted). Needs a human. |
| 70 | STATUS_DEAD = "dead" |
| 71 | |
| 72 | _ACTIVE_STATUSES = frozenset({"", STATUS_PENDING, STATUS_PENDING_DELETE}) |
| 73 | |
| 74 | FEATURE_FLAGS_COLLECTION = "feature_flags" |
| 75 | CONNECTORS_ENABLED_FLAG = "connectors_enabled" |
| 76 | DEFAULT_ACTOR = "daemon:connector" |
| 77 | |
| 78 | |
| 79 | # --- config ---------------------------------------------------------------- |
| 80 | |
| 81 | class ConnectorConfig: |
| 82 | """Generic, connector-agnostic tunables (retry/backoff/batch). A connector |
| 83 | reads its OWN credentials from its own env -- the driver knows nothing |
| 84 | system-specific.""" |
| 85 | |
| 86 | __slots__ = ("max_attempts", "retry_base", "retry_max", "batch_size") |
| 87 | |
| 88 | def __init__(self, *, max_attempts=8, retry_base=60, retry_max=3600, batch_size=20): |
| 89 | self.max_attempts = max_attempts |
| 90 | self.retry_base = retry_base |
| 91 | self.retry_max = retry_max |
| 92 | self.batch_size = batch_size |
| 93 | |
| 94 | |
| 95 | def _env_int(env: Mapping[str, str], key: str, default: int) -> int: |
| 96 | try: |
| 97 | return int(str(env.get(key, "")).strip()) |
| 98 | except (TypeError, ValueError): |
| 99 | return default |
| 100 | |
| 101 | |
| 102 | def connector_config_from_env(env: Mapping[str, str] | None = None) -> ConnectorConfig: |
| 103 | import os |
| 104 | env = os.environ if env is None else env |
| 105 | return ConnectorConfig( |
| 106 | max_attempts=_env_int(env, "DBBASIC_CONNECTOR_MAX_ATTEMPTS", 8), |
| 107 | retry_base=_env_int(env, "DBBASIC_CONNECTOR_RETRY_BASE_SECONDS", 60), |
| 108 | retry_max=_env_int(env, "DBBASIC_CONNECTOR_RETRY_MAX_SECONDS", 3600), |
| 109 | batch_size=_env_int(env, "DBBASIC_CONNECTOR_BATCH_SIZE", 20), |
| 110 | ) |
| 111 | |
| 112 | |
| 113 | def connectors_pass_enabled(*, base_dir: Any) -> bool: |
| 114 | """Brownout kill switch -- a feature_flags row `connectors_enabled`, default |
| 115 | ON. Mirrors object_email.email_pass_enabled / object_rollups exactly.""" |
| 116 | return object_feature_flags.kill_switch_on(CONNECTORS_ENABLED_FLAG, base_dir=base_dir) |
| 117 | |
| 118 | |
| 119 | # --- time helpers ---------------------------------------------------------- |
| 120 | |
| 121 | def _now_iso(now: datetime | None = None) -> str: |
| 122 | now = datetime.now(timezone.utc) if now is None else now |
| 123 | return now.isoformat().replace("+00:00", "Z") |
| 124 | |
| 125 | |
| 126 | def _parse_iso(value: str) -> datetime | None: |
| 127 | raw = (value or "").strip() |
| 128 | if not raw: |
| 129 | return None |
| 130 | try: |
| 131 | return datetime.fromisoformat(raw.replace("Z", "+00:00")) |
| 132 | except ValueError: |
| 133 | return None |
| 134 | |
| 135 | |
| 136 | # --- selection + planning (pure) ------------------------------------------- |
| 137 | |
| 138 | def is_due(record: Mapping[str, Any], now: datetime | None = None) -> bool: |
| 139 | """A row the pass should reconcile this tick: still in an active |
| 140 | (pending / pending_delete / unset) status, and its `sync_next_at` backoff |
| 141 | (blank = immediately) has arrived.""" |
| 142 | status = record.get(SYNC_STATUS_FIELD) or "" |
| 143 | if status not in _ACTIVE_STATUSES: |
| 144 | return False |
| 145 | when = _parse_iso(record.get(SYNC_NEXT_AT_FIELD) or "") |
| 146 | if when is None: |
| 147 | return True |
| 148 | now = datetime.now(timezone.utc) if now is None else now |
| 149 | return when <= now |
| 150 | |
| 151 | |
| 152 | def backoff_seconds(attempts: int, config: ConnectorConfig) -> int: |
| 153 | """`retry_base * 2 ** (attempts - 1)` capped at `retry_max` -- the same |
| 154 | exponential shape as the outbox drain. `attempts` is the count AFTER this |
| 155 | failed try (>= 1).""" |
| 156 | exp = max(0, attempts - 1) |
| 157 | try: |
| 158 | delay = config.retry_base * (2 ** exp) |
| 159 | except OverflowError: |
| 160 | delay = config.retry_max |
| 161 | return int(min(delay, config.retry_max)) |
| 162 | |
| 163 | |
| 164 | def plan_sync( |
| 165 | record: Mapping[str, Any], outcome: Mapping[str, Any], config: ConnectorConfig, |
| 166 | *, now: datetime | None = None, |
| 167 | ) -> dict[str, str]: |
| 168 | """The record update one reconcile outcome implies -- pure, no IO. The |
| 169 | driver owns the lifecycle vocabulary; the connector only says ok / error / |
| 170 | permanent. A successful delete tombstones (`deleted`); a successful |
| 171 | create/edit is `synced`; a transient failure keeps the desired status with |
| 172 | an advanced `sync_next_at`; a permanent one (or exhausted attempts) is |
| 173 | `dead`.""" |
| 174 | now = datetime.now(timezone.utc) if now is None else now |
| 175 | stamp = _now_iso(now) |
| 176 | deleting = (record.get(SYNC_STATUS_FIELD) or "") == STATUS_PENDING_DELETE |
| 177 | |
| 178 | if outcome.get("ok"): |
| 179 | return { |
| 180 | SYNC_STATUS_FIELD: STATUS_DELETED if deleting else STATUS_SYNCED, |
| 181 | SYNC_ERROR_FIELD: "", |
| 182 | SYNC_ATTEMPTS_FIELD: "0", |
| 183 | "updated_at": stamp, |
| 184 | } |
| 185 | |
| 186 | try: |
| 187 | attempts = int(str(record.get(SYNC_ATTEMPTS_FIELD) or "0")) + 1 |
| 188 | except (TypeError, ValueError): |
| 189 | attempts = 1 |
| 190 | update = { |
| 191 | SYNC_ATTEMPTS_FIELD: str(attempts), |
| 192 | SYNC_ERROR_FIELD: str(outcome.get("error") or "")[:500], |
| 193 | "updated_at": stamp, |
| 194 | } |
| 195 | if outcome.get("permanent") or attempts >= config.max_attempts: |
| 196 | update[SYNC_STATUS_FIELD] = STATUS_DEAD |
| 197 | else: |
| 198 | # keep the desired action (create/edit -> pending, delete -> |
| 199 | # pending_delete) so a retry still reconciles the right direction |
| 200 | update[SYNC_STATUS_FIELD] = STATUS_PENDING_DELETE if deleting else STATUS_PENDING |
| 201 | update[SYNC_NEXT_AT_FIELD] = _now_iso(now + timedelta(seconds=backoff_seconds(attempts, config))) |
| 202 | return update |
| 203 | |
| 204 | |
| 205 | # --- dynamic connector loading --------------------------------------------- |
| 206 | |
| 207 | class ConnectorLoadError(Exception): |
| 208 | """Raised when a connector module cannot be loaded or lacks its entry.""" |
| 209 | |
| 210 | |
| 211 | _MODULE_CACHE: dict[tuple[str, str], Callable[..., Any]] = {} |
| 212 | |
| 213 | |
| 214 | def load_connector(module_path: Path | str, entry: str = "reconcile") -> Callable[..., Any]: |
| 215 | """Load a connector's entry function from a file path, at runtime. Cached |
| 216 | for the process lifetime (a redeploy restarts the daemon, so staleness is a |
| 217 | non-issue). Open core calls this on a path a package declared; it never |
| 218 | imports connector code statically.""" |
| 219 | key = (str(Path(module_path).resolve()), entry) |
| 220 | cached = _MODULE_CACHE.get(key) |
| 221 | if cached is not None: |
| 222 | return cached |
| 223 | path = Path(module_path) |
| 224 | if not path.is_file(): |
| 225 | raise ConnectorLoadError(f"connector module not found: {path}") |
| 226 | # A distinct module name per path so two connectors never collide in |
| 227 | # sys.modules; not registered globally (kept local to this loader). |
| 228 | mod_name = f"_dbbasic_connector_{abs(hash(key)) & 0xFFFFFFFF:x}" |
| 229 | spec = importlib.util.spec_from_file_location(mod_name, str(path)) |
| 230 | if spec is None or spec.loader is None: |
| 231 | raise ConnectorLoadError(f"cannot load connector module: {path}") |
| 232 | module = importlib.util.module_from_spec(spec) |
| 233 | try: |
| 234 | spec.loader.exec_module(module) |
| 235 | except Exception as exc: # noqa: BLE001 -- surface any import-time failure |
| 236 | raise ConnectorLoadError(f"connector module {path} failed to import: {exc}") from exc |
| 237 | fn = getattr(module, entry, None) |
| 238 | if not callable(fn): |
| 239 | raise ConnectorLoadError(f"connector module {path} has no callable '{entry}'") |
| 240 | _MODULE_CACHE[key] = fn |
| 241 | return fn |