askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_packages.py · 85.6 KB · 2147 lines
| 1 | """DBBASIC package manifest discovery, dry-run planning, and gated installs. |
| 2 | |
| 3 | Packages are installable bundles of objects, schemas, permissions, seed data, |
| 4 | and migrations. Installs are intentionally conservative: object and schema files |
| 5 | can be created or replaced, seed files can be created, and permissions/migrations |
| 6 | wait for explicit merge/run semantics. |
| 7 | """ |
| 8 | |
| 9 | from __future__ import annotations |
| 10 | |
| 11 | import csv |
| 12 | import io |
| 13 | import json |
| 14 | import os |
| 15 | import re |
| 16 | import tempfile |
| 17 | from pathlib import Path |
| 18 | from typing import Any, Callable, Iterable, Mapping |
| 19 | |
| 20 | import object_collections |
| 21 | import object_namespace |
| 22 | import object_package_baselines |
| 23 | import object_permission_store |
| 24 | import object_permissions |
| 25 | import object_reconciles |
| 26 | import object_record_changes |
| 27 | import object_records |
| 28 | import object_schemas |
| 29 | import object_source |
| 30 | import object_state |
| 31 | from object_namespace import get_object_roots, object_id_from_path, resolve_object_id, validate_object_id |
| 32 | from object_versions import DEFAULT_DATA_DIR |
| 33 | |
| 34 | PACKAGES_DIR = "packages" |
| 35 | MANIFEST_FILE = "dbbasic-package.json" |
| 36 | PACKAGE_MIGRATIONS_DIR = "package_migrations" |
| 37 | |
| 38 | _PACKAGE_ID_RE = re.compile(r"^[a-z][a-z0-9_-]{0,63}$") |
| 39 | _VERSION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$") |
| 40 | _MIGRATION_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$") |
| 41 | |
| 42 | # A package's `schedules` become task_* rows in the scheduler trigger's |
| 43 | # state, which is where object_daemon.process_scheduler reads them from. |
| 44 | # Before this existed, every recurring pass on a running server had been |
| 45 | # hand-entered into that state and appeared NOWHERE in the repository, so |
| 46 | # rebuilding a box silently lost its daily work -- exactly what the |
| 47 | # scheduler object's own docstring warns about. A schedule is part of what |
| 48 | # an app IS, so it ships with the app. |
| 49 | SCHEDULER_OBJECT_ID = "scheduler" |
| 50 | _TASK_KEY_PREFIX = "task_" |
| 51 | _SCHEDULE_ID_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") |
| 52 | _SCHEDULE_TYPES = ("cron", "onetime") |
| 53 | _SCHEDULE_METHODS = ("POST", "GET", "PUT", "DELETE") |
| 54 | # Fields the daemon stamps back onto a task as it runs. An install must |
| 55 | # never touch them: they are the record of what actually happened, and a |
| 56 | # package upgrade is not a reason to forget it (doctrine #8 -- independent |
| 57 | # evidence is never edited). |
| 58 | _SCHEDULE_RUNTIME_FIELDS = ("next_run", "last_run", "run_count") |
| 59 | |
| 60 | # A package's `nav` entries become rows in the `nav_entries` collection, |
| 61 | # which is where every navigation surface reads its doors from. |
| 62 | # Before this existed there were THREE hand-maintained lists of the same |
| 63 | # apps -- a JS array in site_nav, a Python tuple in site_home, and a |
| 64 | # collection->URL map for search hits -- and they had already drifted: |
| 65 | # 25 entries, 21 entries, and no mention at all of shop, intake, billing, |
| 66 | # timers, banking, shipping, receiving or payments. A door is part of what |
| 67 | # an app IS, so it ships with the app, exactly like its schedule. |
| 68 | NAV_COLLECTION = "nav_entries" |
| 69 | _NAV_ID_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") |
| 70 | _NAV_SURFACES = ("public", "member", "operator", "hidden") |
| 71 | _NAV_DEFAULT_GROUP = "Apps" |
| 72 | _NAV_DEFAULT_ORDER = 100 |
| 73 | # What the package OWNS: reinstalling restates every one of these. |
| 74 | _NAV_PACKAGE_FIELDS = ("label", "path", "blurb", "group", "surface", "order") |
| 75 | # What the OPERATOR owns. The package says what an app is; the operator |
| 76 | # says what they want to see. An install never touches this column, for |
| 77 | # the same reason it never restarts a paused schedule (doctrine #8 -- |
| 78 | # somebody else's deliberate decision is not an install's to revert). |
| 79 | _NAV_OPERATOR_FIELDS = ("operator_hidden",) |
| 80 | |
| 81 | # A package's `attention` entries become rows in the `attention_sources` |
| 82 | # collection, which is where the daemon's rollup pass reads its list of |
| 83 | # providers from and how a surface learns that a queue exists at all. |
| 84 | # Third use of the pattern that fixed cron (`schedules`) and navigation |
| 85 | # (`nav`), and for the same reason: this server is built out of gates and |
| 86 | # derived states, every one of which produces a queue of things a machine |
| 87 | # deliberately refused to decide -- scans `extracted`, time `submitted`, |
| 88 | # invoices overdue, bank lines unmatched. It has been computing those |
| 89 | # queues all along and throwing them away at the end of each pass; |
| 90 | # `system_scan_processor` literally returns a field named |
| 91 | # `needing_a_human` and nothing reads it. Only the package that owns a |
| 92 | # domain can honestly say what "needs a human" means in it, so the |
| 93 | # declaration ships with the app. |
| 94 | ATTENTION_COLLECTION = "attention_sources" |
| 95 | _ATTENTION_ID_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") |
| 96 | _ATTENTION_SEVERITIES = ("normal", "warning", "urgent") |
| 97 | _ATTENTION_DEFAULT_GROUP = "Apps" |
| 98 | _ATTENTION_DEFAULT_SEVERITY = "normal" |
| 99 | # What the package OWNS: reinstalling restates every one of these. |
| 100 | _ATTENTION_PACKAGE_FIELDS = ("object_id", "label", "path", "nav_id", "group", "severity") |
| 101 | # What the OPERATOR owns. A deployment that does not care about a queue |
| 102 | # silences it here rather than by editing somebody else's manifest, and an |
| 103 | # install never writes this column -- same posture as nav's |
| 104 | # `operator_hidden` and a paused schedule (doctrine #8). |
| 105 | _ATTENTION_OPERATOR_FIELDS = ("operator_muted",) |
| 106 | |
| 107 | |
| 108 | class InvalidPackageIdError(ValueError): |
| 109 | """Raised when a package id is not safe for routes or storage.""" |
| 110 | |
| 111 | |
| 112 | class PackageNotFoundError(LookupError): |
| 113 | """Raised when a package directory or manifest is missing.""" |
| 114 | |
| 115 | |
| 116 | class InvalidPackageManifestError(ValueError): |
| 117 | """Raised when a package manifest is invalid.""" |
| 118 | |
| 119 | |
| 120 | class PackageInstallError(RuntimeError): |
| 121 | """Raised when a package install would be unsafe or unsupported.""" |
| 122 | |
| 123 | |
| 124 | def validate_package_id(package_id: str) -> bool: |
| 125 | """Return True when a package id is route-safe.""" |
| 126 | if not isinstance(package_id, str): |
| 127 | return False |
| 128 | return bool(_PACKAGE_ID_RE.fullmatch(package_id)) |
| 129 | |
| 130 | |
| 131 | def list_packages( |
| 132 | *, |
| 133 | root: Path | str = PACKAGES_DIR, |
| 134 | ) -> list[dict[str, Any]]: |
| 135 | """Return package summaries for all package manifests under root.""" |
| 136 | packages_root = Path(root) |
| 137 | if not packages_root.exists() or not packages_root.is_dir(): |
| 138 | return [] |
| 139 | |
| 140 | packages = [] |
| 141 | for path in sorted(packages_root.iterdir(), key=lambda item: item.name): |
| 142 | if not path.is_dir() or not validate_package_id(path.name): |
| 143 | continue |
| 144 | manifest_path = path / MANIFEST_FILE |
| 145 | if manifest_path.is_file(): |
| 146 | packages.append(_package_summary(_load_package(path.name, path))) |
| 147 | return packages |
| 148 | |
| 149 | |
| 150 | def get_package( |
| 151 | package_id: str, |
| 152 | *, |
| 153 | root: Path | str = PACKAGES_DIR, |
| 154 | ) -> dict[str, Any]: |
| 155 | """Return one normalized package manifest.""" |
| 156 | package_dir = _package_dir(package_id, root) |
| 157 | return _load_package(package_id, package_dir) |
| 158 | |
| 159 | |
| 160 | DATA_DIR_ENV = "DBBASIC_DATA_DIR" |
| 161 | |
| 162 | |
| 163 | def resolve_base_dir(base_dir: Path | str | None = None) -> Path: |
| 164 | """The data directory an install should write to. |
| 165 | |
| 166 | An explicit `base_dir` always wins. Otherwise the ENVIRONMENT decides, |
| 167 | because `DBBASIC_DATA_DIR` is the one authoritative statement of where |
| 168 | this server's data lives -- it is what the server itself reads, what |
| 169 | every package object reads, and what the systemd unit sets. |
| 170 | |
| 171 | This used to fall straight through to the literal "data", ignoring the |
| 172 | environment even when it was set to somewhere else entirely. The result |
| 173 | was the worst shape a bug can take: an install that reported |
| 174 | `"status": "written"` for every schema and seed, having written them |
| 175 | into a freshly created `./data` beside the checkout that nothing reads. |
| 176 | Nothing failed, nothing warned, and the package was simply absent from |
| 177 | the running server. Hit twice in one afternoon on a box whose layout was |
| 178 | well understood, which is the argument for fixing the default rather |
| 179 | than remembering harder. |
| 180 | |
| 181 | Callers that genuinely want a relative "data" can still ask for it by |
| 182 | name; what they can no longer do is get it by accident. |
| 183 | """ |
| 184 | if base_dir is not None: |
| 185 | return Path(base_dir) |
| 186 | return Path(os.environ.get(DATA_DIR_ENV) or DEFAULT_DATA_DIR) |
| 187 | |
| 188 | |
| 189 | def dry_run_package( |
| 190 | package_id: str, |
| 191 | *, |
| 192 | root: Path | str = PACKAGES_DIR, |
| 193 | base_dir: Path | str | None = None, |
| 194 | object_roots: Iterable[Path] | None = None, |
| 195 | ) -> dict[str, Any]: |
| 196 | """Return a non-mutating package install plan.""" |
| 197 | package_dir = _package_dir(package_id, root) |
| 198 | package = _load_package(package_id, package_dir) |
| 199 | base = resolve_base_dir(base_dir) |
| 200 | |
| 201 | warnings: list[str] = [] |
| 202 | objects = [ |
| 203 | _object_change(entry, package_dir=package_dir, object_roots=object_roots, warnings=warnings) |
| 204 | for entry in package["objects"] |
| 205 | ] |
| 206 | schemas = [ |
| 207 | _schema_change(entry, package_dir=package_dir, base_dir=base, warnings=warnings) |
| 208 | for entry in package["schemas"] |
| 209 | ] |
| 210 | permissions = [ |
| 211 | _permission_change( |
| 212 | entry, |
| 213 | package_id=package_id, |
| 214 | package_dir=package_dir, |
| 215 | base_dir=base, |
| 216 | warnings=warnings, |
| 217 | ) |
| 218 | for entry in package["permissions"] |
| 219 | ] |
| 220 | seed = [ |
| 221 | _seed_change(entry, package_dir=package_dir, base_dir=base, warnings=warnings) |
| 222 | for entry in package["seed"] |
| 223 | ] |
| 224 | migrations = [ |
| 225 | _migration_change(entry, package=package_id, package_dir=package_dir, base_dir=base, warnings=warnings) |
| 226 | for entry in package["migrations"] |
| 227 | ] |
| 228 | # A schedule may name an object this same install is about to write, so |
| 229 | # "does the target exist" is asked of the union of what is already |
| 230 | # resolvable and what this package provides -- not of the server as it |
| 231 | # stands right now. |
| 232 | existing_tasks = _load_scheduler_tasks(base) |
| 233 | installable_object_ids = {entry["id"] for entry in package["objects"]} |
| 234 | roots_for_lookup = list(object_roots) if object_roots is not None else get_object_roots() |
| 235 | for entry in package["schedules"] + package["attention"]: |
| 236 | if entry["object_id"] in installable_object_ids: |
| 237 | continue |
| 238 | if resolve_object_id(entry["object_id"], roots_for_lookup) is not None: |
| 239 | installable_object_ids.add(entry["object_id"]) |
| 240 | schedules = [ |
| 241 | _schedule_change( |
| 242 | entry, |
| 243 | existing=existing_tasks, |
| 244 | installable_object_ids=installable_object_ids, |
| 245 | warnings=warnings, |
| 246 | ) |
| 247 | for entry in package["schedules"] |
| 248 | ] |
| 249 | # Nav entries are compared against what is already registered, which |
| 250 | # is how "two packages claim one door" becomes visible before either |
| 251 | # of them lands. A missing nav_entries collection reads as "nothing |
| 252 | # registered yet", so the plan is create-everything rather than an |
| 253 | # error -- app-nav is not a dependency of being installable. |
| 254 | registered_nav = _load_nav_entries(base, roots=object_roots) or {} |
| 255 | nav = [ |
| 256 | _nav_change(entry, existing=registered_nav, package_id=package_id, warnings=warnings) |
| 257 | for entry in package["nav"] |
| 258 | ] |
| 259 | # Attention sources compare the same way nav entries do, and against |
| 260 | # the same missing-collection posture: app-nav absent reads as |
| 261 | # "nothing declared yet" rather than an error, because a package must |
| 262 | # not fail to install because the home screen is not installed. |
| 263 | registered_attention = _load_attention_sources(base, roots=object_roots) or {} |
| 264 | attention = [ |
| 265 | _attention_change( |
| 266 | entry, |
| 267 | existing=registered_attention, |
| 268 | package_id=package_id, |
| 269 | installable_object_ids=installable_object_ids, |
| 270 | warnings=warnings, |
| 271 | ) |
| 272 | for entry in package["attention"] |
| 273 | ] |
| 274 | |
| 275 | return { |
| 276 | "package": _package_summary(package), |
| 277 | "mode": "dry_run", |
| 278 | "install_enabled": False, |
| 279 | "safe_to_install": not warnings, |
| 280 | # WHERE, resolved and absolute. Every entry below reports a |
| 281 | # `destination` relative to this, and a status of "written" means |
| 282 | # nothing at all without knowing which tree it was written into -- |
| 283 | # an install can succeed completely into a directory the running |
| 284 | # server does not read. |
| 285 | "data_dir": str(base.resolve()), |
| 286 | "objects": objects, |
| 287 | "schemas": schemas, |
| 288 | "permissions": permissions, |
| 289 | "seed": seed, |
| 290 | "migrations": migrations, |
| 291 | "schedules": schedules, |
| 292 | "nav": nav, |
| 293 | "attention": attention, |
| 294 | "warnings": warnings, |
| 295 | } |
| 296 | |
| 297 | |
| 298 | def install_package( |
| 299 | package_id: str, |
| 300 | *, |
| 301 | root: Path | str = PACKAGES_DIR, |
| 302 | base_dir: Path | str | None = None, |
| 303 | object_roots: Iterable[Path] | None = None, |
| 304 | allow_replace: bool = False, |
| 305 | force: bool = False, |
| 306 | before_write: Callable[[Mapping[str, Any]], Mapping[str, Any] | None] | None = None, |
| 307 | ) -> dict[str, Any]: |
| 308 | """Install a package using the conservative public write contract. |
| 309 | |
| 310 | On an upgrade (an object/schema that already exists), writes are no |
| 311 | longer blind: each artifact is three-way compared against its recorded |
| 312 | baseline (see object_package_baselines) and its live content. Pristine |
| 313 | artifacts fast-forward, customized-but-unchanged artifacts are kept, and |
| 314 | genuine conflicts are parked as pending-reconcile records (never |
| 315 | overwritten) unless force=True. See docs/upgrade-and-customization.md |
| 316 | (Rule 1: Reconcile, Don't Replace). |
| 317 | """ |
| 318 | # force implies allow_replace: forcing only makes sense when you may |
| 319 | # touch existing artifacts, so the replace-without-allow_replace blocker |
| 320 | # still applies unless force is also set. |
| 321 | allow_replace = allow_replace or force |
| 322 | roots = list(object_roots) if object_roots is not None else get_object_roots() |
| 323 | if not roots: |
| 324 | raise PackageInstallError("Package installs require at least one object root") |
| 325 | |
| 326 | package_dir = _package_dir(package_id, root) |
| 327 | package = _load_package(package_id, package_dir) |
| 328 | base = resolve_base_dir(base_dir) |
| 329 | plan = dry_run_package(package_id, root=root, base_dir=base, object_roots=roots) |
| 330 | |
| 331 | blockers = _install_blockers(plan, package=package, allow_replace=allow_replace) |
| 332 | if blockers: |
| 333 | raise PackageInstallError("; ".join(blockers)) |
| 334 | |
| 335 | object_writes = [] |
| 336 | for entry, planned in zip(package["objects"], plan["objects"], strict=True): |
| 337 | source = _package_file(package_dir, entry["path"]) |
| 338 | existing = resolve_object_id(entry["id"], roots) |
| 339 | # An object that already exists stays where it lives -- a customized |
| 340 | # object must not be yanked out of the override root by an upgrade. |
| 341 | # |
| 342 | # A NEW object goes to the LAST root, which is the base/system root: |
| 343 | # get_object_roots() returns [override, base] in LOOKUP order, so |
| 344 | # roots[0] is the override root, and installing there was silently |
| 345 | # wrong. object_namespace.get_base_object_roots says so directly -- |
| 346 | # "the root packages install into and reconcile against ... the |
| 347 | # pristine, upgradeable copy" -- but this line used roots[0] and |
| 348 | # contradicted it. The consequence was not cosmetic: iter_object_sources |
| 349 | # labels anything under the override root kind="override", and |
| 350 | # object_handlers.build_index indexes ONLY kind=="system" (Decision 2, |
| 351 | # user-authored handlers deferred). So every package-installed object |
| 352 | # declaring HANDLES installed successfully, reported "written", and |
| 353 | # then never received a single event. |
| 354 | # |
| 355 | # With one root configured (no overrides) this is the same root it |
| 356 | # always was. |
| 357 | destination_root = (_root_for_path(existing, roots) if existing is not None |
| 358 | else roots[-1]) |
| 359 | if destination_root is None: |
| 360 | raise PackageInstallError(f"Existing object is outside configured object roots: {entry['id']}") |
| 361 | destination = existing or _object_destination(entry, destination_root) |
| 362 | _ensure_inside(destination, destination_root, label="object") |
| 363 | try: |
| 364 | mapped_id = object_id_from_path(destination, destination_root) |
| 365 | except ValueError as exc: |
| 366 | raise PackageInstallError( |
| 367 | f"Package object path is not a valid object destination: {entry['path']}" |
| 368 | ) from exc |
| 369 | if mapped_id != entry["id"]: |
| 370 | raise PackageInstallError( |
| 371 | f"Package object path does not map to object id {entry['id']}: {entry['path']}" |
| 372 | ) |
| 373 | object_writes.append((planned, destination, destination_root, source.read_bytes())) |
| 374 | |
| 375 | schema_writes = [] |
| 376 | for entry, planned in zip(package["schemas"], plan["schemas"], strict=True): |
| 377 | source = _package_file(package_dir, entry["path"]) |
| 378 | try: |
| 379 | payload = json.loads(source.read_text(encoding="utf-8")) |
| 380 | except json.JSONDecodeError as exc: |
| 381 | raise PackageInstallError(f"Package schema contains invalid JSON: {entry['path']}") from exc |
| 382 | try: |
| 383 | normalized = object_schemas.normalize_schema(entry["collection"], payload, source="manual") |
| 384 | except ValueError as exc: |
| 385 | raise PackageInstallError(f"Package schema is invalid: {entry['path']}") from exc |
| 386 | schema_writes.append((entry, planned, normalized)) |
| 387 | |
| 388 | seed_writes = [] |
| 389 | for entry, planned in zip(package["seed"], plan["seed"], strict=True): |
| 390 | source = _package_file(package_dir, entry["path"]) |
| 391 | destination = base / "collections" / entry["collection"] / "records.tsv" |
| 392 | _ensure_inside(destination, base / "collections", label="seed") |
| 393 | seed_writes.append((entry, planned, destination, source.read_bytes())) |
| 394 | |
| 395 | permission_writes = [] |
| 396 | for entry, planned in zip(package["permissions"], plan["permissions"], strict=True): |
| 397 | rules = _load_permission_rules(package_dir, entry, package_id=package_id) |
| 398 | permission_writes.append((planned, rules)) |
| 399 | |
| 400 | restore_point = before_write(plan) if before_write is not None else None |
| 401 | |
| 402 | existing_baseline = object_package_baselines.load_baseline(package_id, base_dir=base) or {} |
| 403 | base_objects = existing_baseline.get("objects") or {} |
| 404 | base_schemas = existing_baseline.get("schemas") or {} |
| 405 | base_schema_bodies = existing_baseline.get("schema_bodies") or {} |
| 406 | baseline_version = existing_baseline.get("version") |
| 407 | |
| 408 | reconciles: list[str] = [] |
| 409 | new_baseline_objects: dict[str, str] = {} |
| 410 | new_baseline_schemas: dict[str, str] = {} |
| 411 | new_baseline_schema_bodies: dict[str, Any] = {} |
| 412 | |
| 413 | installed_objects = [] |
| 414 | for planned, destination, destination_root, content in object_writes: |
| 415 | object_id = planned["id"] |
| 416 | extra: dict[str, Any] = {} |
| 417 | |
| 418 | if planned["action"] == "create": |
| 419 | _write_file_atomic_bytes(destination, content) |
| 420 | status = "written" |
| 421 | new_baseline_objects[object_id] = object_package_baselines.sha256_text(content.decode("utf-8")) |
| 422 | else: |
| 423 | new_text = content.decode("utf-8") |
| 424 | new_sha = object_package_baselines.sha256_text(new_text) |
| 425 | live_text = object_source.get_object_source(object_id, roots) |
| 426 | live_sha = object_package_baselines.sha256_text(live_text) |
| 427 | old_sha = base_objects.get(object_id) |
| 428 | |
| 429 | if live_sha == new_sha: |
| 430 | status = "unchanged" |
| 431 | new_baseline_objects[object_id] = new_sha |
| 432 | elif old_sha is not None and live_sha == old_sha: |
| 433 | _write_file_atomic_bytes(destination, content) |
| 434 | status = "updated" |
| 435 | new_baseline_objects[object_id] = new_sha |
| 436 | elif old_sha is not None and new_sha == old_sha: |
| 437 | status = "kept" |
| 438 | new_baseline_objects[object_id] = old_sha |
| 439 | elif force: |
| 440 | _write_file_atomic_bytes(destination, content) |
| 441 | status = "forced" |
| 442 | new_baseline_objects[object_id] = new_sha |
| 443 | else: |
| 444 | rec = object_reconciles.create_reconcile( |
| 445 | package=package_id, |
| 446 | target_version=package["version"], |
| 447 | baseline_version=baseline_version, |
| 448 | artifact={"kind": "object", "id": object_id}, |
| 449 | mine=live_text, |
| 450 | theirs=new_text, |
| 451 | base_sha=old_sha, |
| 452 | base_dir=base, |
| 453 | ) |
| 454 | reconciles.append(rec["id"]) |
| 455 | status = "conflict" |
| 456 | extra["reconcile_id"] = rec["id"] |
| 457 | new_baseline_objects[object_id] = old_sha if old_sha is not None else live_sha |
| 458 | |
| 459 | installed_objects.append( |
| 460 | { |
| 461 | **planned, |
| 462 | "status": status, |
| 463 | "destination": _relative_display_path(destination, destination_root), |
| 464 | **extra, |
| 465 | } |
| 466 | ) |
| 467 | |
| 468 | installed_schemas = [] |
| 469 | for entry, planned, normalized in schema_writes: |
| 470 | collection = entry["collection"] |
| 471 | new_sha = object_package_baselines.canonical_schema_hash(normalized) |
| 472 | extra: dict[str, Any] = {} |
| 473 | |
| 474 | live = None |
| 475 | if planned["action"] == "replace": |
| 476 | try: |
| 477 | live = object_schemas.get_schema(collection, base_dir=base, roots=roots) |
| 478 | except object_schemas.SchemaNotFoundError: |
| 479 | live = None |
| 480 | |
| 481 | if live is None: |
| 482 | object_schemas.replace_schema(collection, normalized, base_dir=base) |
| 483 | status = "written" |
| 484 | new_baseline_schemas[collection] = new_sha |
| 485 | new_baseline_schema_bodies[collection] = normalized |
| 486 | else: |
| 487 | live_sha = object_package_baselines.canonical_schema_hash(live) |
| 488 | old_sha = base_schemas.get(collection) |
| 489 | base_body = base_schema_bodies.get(collection) |
| 490 | |
| 491 | if live_sha == new_sha: |
| 492 | status = "unchanged" |
| 493 | new_baseline_schemas[collection] = new_sha |
| 494 | new_baseline_schema_bodies[collection] = normalized |
| 495 | elif old_sha is not None and live_sha == old_sha: |
| 496 | object_schemas.replace_schema(collection, normalized, base_dir=base) |
| 497 | status = "updated" |
| 498 | new_baseline_schemas[collection] = new_sha |
| 499 | new_baseline_schema_bodies[collection] = normalized |
| 500 | elif old_sha is not None and new_sha == old_sha: |
| 501 | status = "kept" |
| 502 | new_baseline_schemas[collection] = old_sha |
| 503 | new_baseline_schema_bodies[collection] = normalized |
| 504 | elif force: |
| 505 | object_schemas.replace_schema(collection, normalized, base_dir=base) |
| 506 | status = "forced" |
| 507 | new_baseline_schemas[collection] = new_sha |
| 508 | new_baseline_schema_bodies[collection] = normalized |
| 509 | elif base_body is not None: |
| 510 | # Customized AND shipped changed: try a field-union merge |
| 511 | # before parking a conflict (Rule 3: schemas are additive, |
| 512 | # so a same-named field changed on both sides is the only |
| 513 | # real conflict -- see docs/upgrade-and-customization.md). |
| 514 | merged, collisions = object_schemas.merge_schema_fields(base_body, live, normalized) |
| 515 | if not collisions: |
| 516 | object_schemas.replace_schema(collection, merged, base_dir=base) |
| 517 | status = "merged" |
| 518 | new_baseline_schemas[collection] = new_sha |
| 519 | new_baseline_schema_bodies[collection] = normalized |
| 520 | else: |
| 521 | rec = object_reconciles.create_reconcile( |
| 522 | package=package_id, |
| 523 | target_version=package["version"], |
| 524 | baseline_version=baseline_version, |
| 525 | artifact={"kind": "schema", "collection": collection}, |
| 526 | mine=json.dumps(live, indent=2, sort_keys=True), |
| 527 | theirs=json.dumps(normalized, indent=2, sort_keys=True), |
| 528 | base_sha=old_sha, |
| 529 | base_dir=base, |
| 530 | collisions=collisions, |
| 531 | ) |
| 532 | reconciles.append(rec["id"]) |
| 533 | status = "conflict" |
| 534 | extra["reconcile_id"] = rec["id"] |
| 535 | extra["collisions"] = collisions |
| 536 | new_baseline_schemas[collection] = old_sha if old_sha is not None else live_sha |
| 537 | new_baseline_schema_bodies[collection] = base_body |
| 538 | else: |
| 539 | # No base schema body on record (baseline predates this |
| 540 | # feature, or was never stamped): fall back to parking a |
| 541 | # conflict, unchanged from prior behavior. |
| 542 | rec = object_reconciles.create_reconcile( |
| 543 | package=package_id, |
| 544 | target_version=package["version"], |
| 545 | baseline_version=baseline_version, |
| 546 | artifact={"kind": "schema", "collection": collection}, |
| 547 | mine=json.dumps(live, indent=2, sort_keys=True), |
| 548 | theirs=json.dumps(normalized, indent=2, sort_keys=True), |
| 549 | base_sha=old_sha, |
| 550 | base_dir=base, |
| 551 | ) |
| 552 | reconciles.append(rec["id"]) |
| 553 | status = "conflict" |
| 554 | extra["reconcile_id"] = rec["id"] |
| 555 | new_baseline_schemas[collection] = old_sha if old_sha is not None else live_sha |
| 556 | if base_body is not None: |
| 557 | new_baseline_schema_bodies[collection] = base_body |
| 558 | |
| 559 | installed_schemas.append( |
| 560 | { |
| 561 | **planned, |
| 562 | "status": status, |
| 563 | "destination": f"schemas/{collection}.json", |
| 564 | **extra, |
| 565 | } |
| 566 | ) |
| 567 | |
| 568 | installed_seed = [] |
| 569 | for entry, planned, destination, content in seed_writes: |
| 570 | # Seed into a collection that already holds records: MERGE by id rather |
| 571 | # than skip. Add only the seed rows whose id isn't already present; |
| 572 | # existing rows (live data, possibly customized) are never touched. This |
| 573 | # is what lets several packages each seed a SHARED collection |
| 574 | # (views/site_routes) -- previously only the first-installed package's |
| 575 | # seed landed and every later one was skipped (needing manual inserts). |
| 576 | # A fresh collection (no records yet) still takes the whole seed file. |
| 577 | if planned.get("installed"): |
| 578 | added = _merge_seed_rows( |
| 579 | entry["collection"], content, base_dir=base, package_id=package_id |
| 580 | ) |
| 581 | installed_seed.append( |
| 582 | { |
| 583 | **planned, |
| 584 | "status": "merged" if added else "skipped", |
| 585 | "reason": ( |
| 586 | f"merged {added} new row(s); existing preserved" |
| 587 | if added else "collection already has these rows; preserved" |
| 588 | ), |
| 589 | "added": added, |
| 590 | "destination": f"collections/{entry['collection']}/records.tsv", |
| 591 | } |
| 592 | ) |
| 593 | continue |
| 594 | _write_file_atomic_bytes(destination, content) |
| 595 | _attribute_seed_records(entry["collection"], package_id=package_id, base_dir=base) |
| 596 | installed_seed.append( |
| 597 | { |
| 598 | **planned, |
| 599 | "status": "written", |
| 600 | "destination": f"collections/{entry['collection']}/records.tsv", |
| 601 | } |
| 602 | ) |
| 603 | |
| 604 | installed_permissions = [] |
| 605 | for planned, rules in permission_writes: |
| 606 | total, added = _merge_permission_rules(rules, base_dir=base) |
| 607 | installed_permissions.append( |
| 608 | { |
| 609 | **planned, |
| 610 | "status": "merged", |
| 611 | "rules": total, |
| 612 | "new_rules": added, |
| 613 | } |
| 614 | ) |
| 615 | |
| 616 | # Schedules land last: a task board that points at an object is only |
| 617 | # honest once that object is on disk. |
| 618 | installed_schedules = _apply_schedules( |
| 619 | package["schedules"], plan["schedules"], base_dir=base |
| 620 | ) |
| 621 | |
| 622 | # Nav lands after the seeds, because a package that ships the |
| 623 | # nav_entries schema itself (app-nav) has to have written it before |
| 624 | # its own doors can be registered. |
| 625 | installed_nav = _apply_nav( |
| 626 | package["nav"], plan["nav"], package_id=package_id, base_dir=base, roots=roots |
| 627 | ) |
| 628 | |
| 629 | # Attention sources land last of all, for the same reason schedules |
| 630 | # do: a counter that names an object is only honest once that object |
| 631 | # is on disk, and the row is what the daemon will start executing. |
| 632 | installed_attention = _apply_attention( |
| 633 | package["attention"], plan["attention"], package_id=package_id, |
| 634 | base_dir=base, roots=roots |
| 635 | ) |
| 636 | |
| 637 | object_package_baselines.record_baseline( |
| 638 | package_id, |
| 639 | version=package["version"], |
| 640 | objects=new_baseline_objects, |
| 641 | schemas=new_baseline_schemas, |
| 642 | schema_bodies=new_baseline_schema_bodies, |
| 643 | base_dir=base, |
| 644 | ) |
| 645 | |
| 646 | result = { |
| 647 | "package": plan["package"], |
| 648 | "mode": "install", |
| 649 | "install_enabled": True, |
| 650 | "allow_replace": allow_replace, |
| 651 | "safe_to_install": True, |
| 652 | # The resolved data directory everything below landed in. Reported |
| 653 | # because "written" is not a claim anybody can check without it -- |
| 654 | # see resolve_base_dir for the afternoon that made this necessary. |
| 655 | "data_dir": plan["data_dir"], |
| 656 | "objects": installed_objects, |
| 657 | "schemas": installed_schemas, |
| 658 | "permissions": installed_permissions, |
| 659 | "seed": installed_seed, |
| 660 | "migrations": plan["migrations"], |
| 661 | "schedules": installed_schedules, |
| 662 | "nav": installed_nav, |
| 663 | "attention": installed_attention, |
| 664 | "reconciles": reconciles, |
| 665 | "warnings": [], |
| 666 | } |
| 667 | if restore_point is not None: |
| 668 | result["restore_point"] = dict(restore_point) |
| 669 | return result |
| 670 | |
| 671 | |
| 672 | def package_status( |
| 673 | package_id: str, |
| 674 | *, |
| 675 | root: Path | str = PACKAGES_DIR, |
| 676 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 677 | object_roots: Iterable[Path] | None = None, |
| 678 | ) -> dict[str, Any]: |
| 679 | """Return install/customization status for a package's baselined artifacts.""" |
| 680 | roots = list(object_roots) if object_roots is not None else get_object_roots() |
| 681 | base = Path(base_dir) |
| 682 | |
| 683 | package = _load_package(package_id, _package_dir(package_id, root)) |
| 684 | summary = _package_summary(package) |
| 685 | summary["pending_reconciles"] = object_reconciles.count_pending(base_dir=base, package=package_id) |
| 686 | |
| 687 | baseline = object_package_baselines.load_baseline(package_id, base_dir=base) |
| 688 | if baseline is None: |
| 689 | summary["installed"] = False |
| 690 | summary["customized"] = False |
| 691 | summary["artifacts"] = [] |
| 692 | return summary |
| 693 | |
| 694 | artifacts: list[dict[str, Any]] = [] |
| 695 | any_customized = False |
| 696 | |
| 697 | for object_id, base_sha in (baseline.get("objects") or {}).items(): |
| 698 | try: |
| 699 | live = object_source.get_object_source(object_id, roots) |
| 700 | except (FileNotFoundError, LookupError, OSError, ValueError, object_source.ObjectSourceError): |
| 701 | state = "removed" |
| 702 | else: |
| 703 | state = "pristine" if object_package_baselines.sha256_text(live) == base_sha else "customized" |
| 704 | if state == "customized": |
| 705 | any_customized = True |
| 706 | artifacts.append( |
| 707 | { |
| 708 | "kind": "object", |
| 709 | "id": object_id, |
| 710 | "state": state, |
| 711 | "overridden": object_namespace.has_override(object_id), |
| 712 | } |
| 713 | ) |
| 714 | |
| 715 | for collection, base_sha in (baseline.get("schemas") or {}).items(): |
| 716 | try: |
| 717 | live = object_schemas.get_schema(collection, base_dir=base, roots=roots) |
| 718 | except (object_schemas.SchemaNotFoundError, LookupError, OSError, ValueError): |
| 719 | state = "removed" |
| 720 | else: |
| 721 | state = ( |
| 722 | "pristine" |
| 723 | if object_package_baselines.canonical_schema_hash(live) == base_sha |
| 724 | else "customized" |
| 725 | ) |
| 726 | if state == "customized": |
| 727 | any_customized = True |
| 728 | artifacts.append({"kind": "schema", "collection": collection, "state": state}) |
| 729 | |
| 730 | summary["installed"] = True |
| 731 | summary["installed_version"] = baseline.get("version") |
| 732 | summary["customized"] = any_customized |
| 733 | summary["artifacts"] = artifacts |
| 734 | return summary |
| 735 | |
| 736 | |
| 737 | def _package_dir(package_id: str, root: Path | str) -> Path: |
| 738 | if not validate_package_id(package_id): |
| 739 | raise InvalidPackageIdError(f"Invalid package id: {package_id}") |
| 740 | |
| 741 | package_dir = Path(root) / package_id |
| 742 | manifest_path = package_dir / MANIFEST_FILE |
| 743 | if not package_dir.is_dir() or not manifest_path.is_file(): |
| 744 | raise PackageNotFoundError(f"Package not found: {package_id}") |
| 745 | return package_dir |
| 746 | |
| 747 | |
| 748 | def _load_package(package_id: str, package_dir: Path) -> dict[str, Any]: |
| 749 | manifest_path = package_dir / MANIFEST_FILE |
| 750 | try: |
| 751 | payload = json.loads(manifest_path.read_text()) |
| 752 | except json.JSONDecodeError as exc: |
| 753 | raise InvalidPackageManifestError(f"Package manifest contains invalid JSON: {package_id}") from exc |
| 754 | |
| 755 | return _normalize_manifest(package_id, payload) |
| 756 | |
| 757 | |
| 758 | def _normalize_manifest(package_id: str, payload: Any) -> dict[str, Any]: |
| 759 | if not isinstance(payload, Mapping): |
| 760 | raise InvalidPackageManifestError(f"Package manifest must contain an object: {package_id}") |
| 761 | |
| 762 | manifest_id = _required_text(payload, "id", package_id=package_id) |
| 763 | if manifest_id != package_id: |
| 764 | raise InvalidPackageManifestError( |
| 765 | f"Package manifest id does not match directory: {package_id}" |
| 766 | ) |
| 767 | if not validate_package_id(manifest_id): |
| 768 | raise InvalidPackageIdError(f"Invalid package id: {manifest_id}") |
| 769 | |
| 770 | version = _required_text(payload, "version", package_id=package_id) |
| 771 | if not _VERSION_RE.fullmatch(version): |
| 772 | raise InvalidPackageManifestError(f"Invalid package version: {package_id}") |
| 773 | |
| 774 | return { |
| 775 | "id": manifest_id, |
| 776 | "name": _required_text(payload, "name", package_id=package_id), |
| 777 | "version": version, |
| 778 | "description": _optional_text(payload.get("description")), |
| 779 | "compatibility": _mapping_field(payload.get("compatibility"), package_id=package_id), |
| 780 | "dependencies": _normalize_dependencies(payload.get("dependencies", []), package_id=package_id), |
| 781 | "objects": _normalize_objects(payload.get("objects", []), package_id=package_id), |
| 782 | "schemas": _normalize_collection_paths( |
| 783 | payload.get("schemas", []), |
| 784 | package_id=package_id, |
| 785 | collection_key="collection", |
| 786 | section="schemas", |
| 787 | ), |
| 788 | "permissions": _normalize_path_entries( |
| 789 | payload.get("permissions", []), |
| 790 | package_id=package_id, |
| 791 | section="permissions", |
| 792 | ), |
| 793 | "seed": _normalize_collection_paths( |
| 794 | payload.get("seed", []), |
| 795 | package_id=package_id, |
| 796 | collection_key="collection", |
| 797 | section="seed", |
| 798 | ), |
| 799 | "migrations": _normalize_migrations(payload.get("migrations", []), package_id=package_id), |
| 800 | "connectors": _normalize_connectors(payload.get("connectors", []), package_id=package_id), |
| 801 | "schedules": _normalize_schedules(payload.get("schedules", []), package_id=package_id), |
| 802 | "nav": _normalize_nav(payload.get("nav", []), package_id=package_id), |
| 803 | "attention": _normalize_attention(payload.get("attention", []), package_id=package_id), |
| 804 | # The documented opt-out for a package that ships a site_* object |
| 805 | # which is not a door: a JS widget (site_thread), a fragment |
| 806 | # (site_materialize_run_button), or a per-record document reached |
| 807 | # from the record itself (site_packing_slip). Saying so in the |
| 808 | # manifest is the point -- "this app has no front page" becomes a |
| 809 | # reviewable claim rather than an omission nobody notices. |
| 810 | "nav_optional": bool(payload.get("nav_optional", False)), |
| 811 | } |
| 812 | |
| 813 | |
| 814 | def _package_summary(package: Mapping[str, Any]) -> dict[str, Any]: |
| 815 | return { |
| 816 | "id": package["id"], |
| 817 | "name": package["name"], |
| 818 | "version": package["version"], |
| 819 | "description": package.get("description"), |
| 820 | "status": "available", |
| 821 | "object_count": len(package["objects"]), |
| 822 | "schema_count": len(package["schemas"]), |
| 823 | "permission_count": len(package["permissions"]), |
| 824 | "seed_count": len(package["seed"]), |
| 825 | "migration_count": len(package["migrations"]), |
| 826 | "dependency_count": len(package["dependencies"]), |
| 827 | "connector_count": len(package.get("connectors", [])), |
| 828 | "schedule_count": len(package.get("schedules", [])), |
| 829 | "nav_count": len(package.get("nav", [])), |
| 830 | "attention_count": len(package.get("attention", [])), |
| 831 | } |
| 832 | |
| 833 | |
| 834 | def _normalize_objects(payload: Any, *, package_id: str) -> list[dict[str, str]]: |
| 835 | entries = _list_field(payload, package_id=package_id, section="objects") |
| 836 | normalized = [] |
| 837 | for entry in entries: |
| 838 | mapping = _entry_mapping(entry, package_id=package_id, section="objects") |
| 839 | object_id = _required_text(mapping, "id", package_id=package_id) |
| 840 | if not validate_object_id(object_id): |
| 841 | raise InvalidPackageManifestError(f"Invalid package object id: {object_id}") |
| 842 | normalized.append( |
| 843 | { |
| 844 | "id": object_id, |
| 845 | "path": _safe_relative_path( |
| 846 | _required_text(mapping, "path", package_id=package_id), |
| 847 | package_id=package_id, |
| 848 | section="objects", |
| 849 | ), |
| 850 | } |
| 851 | ) |
| 852 | return normalized |
| 853 | |
| 854 | |
| 855 | def _normalize_collection_paths( |
| 856 | payload: Any, |
| 857 | *, |
| 858 | package_id: str, |
| 859 | collection_key: str, |
| 860 | section: str, |
| 861 | ) -> list[dict[str, str]]: |
| 862 | entries = _list_field(payload, package_id=package_id, section=section) |
| 863 | normalized = [] |
| 864 | for entry in entries: |
| 865 | mapping = _entry_mapping(entry, package_id=package_id, section=section) |
| 866 | collection = _required_text(mapping, collection_key, package_id=package_id) |
| 867 | if not object_collections.validate_collection_name(collection): |
| 868 | raise InvalidPackageManifestError(f"Invalid package collection: {collection}") |
| 869 | normalized.append( |
| 870 | { |
| 871 | collection_key: collection, |
| 872 | "path": _safe_relative_path( |
| 873 | _required_text(mapping, "path", package_id=package_id), |
| 874 | package_id=package_id, |
| 875 | section=section, |
| 876 | ), |
| 877 | } |
| 878 | ) |
| 879 | return normalized |
| 880 | |
| 881 | |
| 882 | def _normalize_path_entries( |
| 883 | payload: Any, |
| 884 | *, |
| 885 | package_id: str, |
| 886 | section: str, |
| 887 | ) -> list[dict[str, str]]: |
| 888 | entries = _list_field(payload, package_id=package_id, section=section) |
| 889 | normalized = [] |
| 890 | for entry in entries: |
| 891 | mapping = _entry_mapping(entry, package_id=package_id, section=section) |
| 892 | normalized.append( |
| 893 | { |
| 894 | "path": _safe_relative_path( |
| 895 | _required_text(mapping, "path", package_id=package_id), |
| 896 | package_id=package_id, |
| 897 | section=section, |
| 898 | ) |
| 899 | } |
| 900 | ) |
| 901 | return normalized |
| 902 | |
| 903 | |
| 904 | def _normalize_migrations(payload: Any, *, package_id: str) -> list[dict[str, str]]: |
| 905 | entries = _list_field(payload, package_id=package_id, section="migrations") |
| 906 | normalized = [] |
| 907 | for entry in entries: |
| 908 | mapping = _entry_mapping(entry, package_id=package_id, section="migrations") |
| 909 | migration_id = _required_text(mapping, "id", package_id=package_id) |
| 910 | if not _MIGRATION_ID_RE.fullmatch(migration_id): |
| 911 | raise InvalidPackageManifestError(f"Invalid package migration id: {migration_id}") |
| 912 | normalized.append( |
| 913 | { |
| 914 | "id": migration_id, |
| 915 | "path": _safe_relative_path( |
| 916 | _required_text(mapping, "path", package_id=package_id), |
| 917 | package_id=package_id, |
| 918 | section="migrations", |
| 919 | ), |
| 920 | } |
| 921 | ) |
| 922 | return normalized |
| 923 | |
| 924 | |
| 925 | def _normalize_connectors(payload: Any, *, package_id: str) -> list[dict[str, str]]: |
| 926 | """A `connectors` entry declares that a collection is reconciled against an |
| 927 | external system by a connector module inside this package (see |
| 928 | object_connectors / plan/vocabulary/03-external-connectors-spec.md): |
| 929 | `{collection, module, entry?}`. `module` is validated as a package-relative |
| 930 | path (escape-checked again at load time); `entry` defaults to `reconcile`.""" |
| 931 | entries = _list_field(payload, package_id=package_id, section="connectors") |
| 932 | normalized = [] |
| 933 | for entry in entries: |
| 934 | mapping = _entry_mapping(entry, package_id=package_id, section="connectors") |
| 935 | collection = _required_text(mapping, "collection", package_id=package_id) |
| 936 | if not object_collections.validate_collection_name(collection): |
| 937 | raise InvalidPackageManifestError(f"Invalid package connector collection: {collection}") |
| 938 | module = _safe_relative_path( |
| 939 | _required_text(mapping, "module", package_id=package_id), |
| 940 | package_id=package_id, |
| 941 | section="connectors", |
| 942 | ) |
| 943 | entry_name = _optional_text(mapping.get("entry")) or "reconcile" |
| 944 | if not entry_name.isidentifier(): |
| 945 | raise InvalidPackageManifestError(f"Invalid connector entry: {entry_name}") |
| 946 | normalized.append({"collection": collection, "module": module, "entry": entry_name}) |
| 947 | return normalized |
| 948 | |
| 949 | |
| 950 | def _normalize_schedules(payload: Any, *, package_id: str) -> list[dict[str, Any]]: |
| 951 | """A `schedules` entry declares a recurring pass the app needs to work: |
| 952 | `{id, object_id, schedule, type?, method?, payload?, description?}`. |
| 953 | |
| 954 | Time-driven work belongs to the daemon (docs/logic-decisions.md #2), |
| 955 | and until now the declaration of WHICH work lived only in a running |
| 956 | server's state. That made a schedule invisible to review, absent from |
| 957 | a fresh install, and lost on a rebuild. Declaring it here makes the |
| 958 | recurring pass part of the package, like the object it calls. |
| 959 | |
| 960 | The id becomes a `task_<id>` state key, so it is restricted to the |
| 961 | characters a key can safely hold; every other field is validated here |
| 962 | rather than at 3am inside the daemon, where a bad cron string is a |
| 963 | pass that quietly does nothing. |
| 964 | """ |
| 965 | entries = _list_field(payload, package_id=package_id, section="schedules") |
| 966 | normalized: list[dict[str, Any]] = [] |
| 967 | seen: set[str] = set() |
| 968 | for entry in entries: |
| 969 | mapping = _entry_mapping(entry, package_id=package_id, section="schedules") |
| 970 | schedule_id = _required_text(mapping, "id", package_id=package_id) |
| 971 | if not _SCHEDULE_ID_RE.fullmatch(schedule_id): |
| 972 | raise InvalidPackageManifestError(f"Invalid package schedule id: {schedule_id}") |
| 973 | if schedule_id in seen: |
| 974 | raise InvalidPackageManifestError( |
| 975 | f"Duplicate package schedule id: {schedule_id}" |
| 976 | ) |
| 977 | seen.add(schedule_id) |
| 978 | |
| 979 | object_id = _required_text(mapping, "object_id", package_id=package_id) |
| 980 | if not validate_object_id(object_id): |
| 981 | raise InvalidPackageManifestError( |
| 982 | f"Invalid package schedule object_id: {object_id}" |
| 983 | ) |
| 984 | |
| 985 | schedule_type = (_optional_text(mapping.get("type")) or "cron").lower() |
| 986 | if schedule_type not in _SCHEDULE_TYPES: |
| 987 | raise InvalidPackageManifestError( |
| 988 | f"Package schedule type must be cron|onetime: {schedule_id}" |
| 989 | ) |
| 990 | |
| 991 | expression = _required_text(mapping, "schedule", package_id=package_id) |
| 992 | _validate_schedule_expression(expression, schedule_type, schedule_id=schedule_id) |
| 993 | |
| 994 | method = (_optional_text(mapping.get("method")) or "POST").upper() |
| 995 | if method not in _SCHEDULE_METHODS: |
| 996 | raise InvalidPackageManifestError( |
| 997 | f"Invalid package schedule method: {schedule_id}" |
| 998 | ) |
| 999 | |
| 1000 | task_payload = mapping.get("payload", {}) |
| 1001 | if task_payload in (None, ""): |
| 1002 | task_payload = {} |
| 1003 | if not isinstance(task_payload, Mapping): |
| 1004 | raise InvalidPackageManifestError( |
| 1005 | f"Package schedule payload must be an object: {schedule_id}" |
| 1006 | ) |
| 1007 | |
| 1008 | normalized.append({ |
| 1009 | "id": schedule_id, |
| 1010 | "object_id": object_id, |
| 1011 | "method": method, |
| 1012 | "payload": dict(task_payload), |
| 1013 | "schedule": expression, |
| 1014 | "type": schedule_type, |
| 1015 | "description": _optional_text(mapping.get("description")), |
| 1016 | }) |
| 1017 | return normalized |
| 1018 | |
| 1019 | |
| 1020 | def _validate_schedule_expression( |
| 1021 | expression: str, |
| 1022 | schedule_type: str, |
| 1023 | *, |
| 1024 | schedule_id: str, |
| 1025 | ) -> None: |
| 1026 | """Reject a schedule that could never fire, at install time. |
| 1027 | |
| 1028 | The daemon treats an unparseable expression as "no next run" and moves |
| 1029 | on in silence, so a typo here is a pass that never runs and never |
| 1030 | complains -- the failure this whole feature exists to end. Cron is |
| 1031 | checked with croniter where it is installed, and structurally where it |
| 1032 | is not, so validation degrades rather than disappearing. |
| 1033 | """ |
| 1034 | if "\t" in expression or "\n" in expression: |
| 1035 | raise InvalidPackageManifestError( |
| 1036 | f"Package schedule expression may not contain tabs or newlines: {schedule_id}" |
| 1037 | ) |
| 1038 | if schedule_type == "onetime": |
| 1039 | from datetime import datetime |
| 1040 | |
| 1041 | try: |
| 1042 | datetime.fromisoformat(expression.replace("Z", "+00:00")) |
| 1043 | except ValueError as exc: |
| 1044 | raise InvalidPackageManifestError( |
| 1045 | f"Package onetime schedule must be an ISO timestamp: {schedule_id}" |
| 1046 | ) from exc |
| 1047 | return |
| 1048 | |
| 1049 | try: |
| 1050 | from croniter import croniter |
| 1051 | except ImportError: |
| 1052 | croniter = None |
| 1053 | if croniter is not None: |
| 1054 | if not croniter.is_valid(expression): |
| 1055 | raise InvalidPackageManifestError( |
| 1056 | f"Invalid package cron expression: {schedule_id}" |
| 1057 | ) |
| 1058 | return |
| 1059 | if len(expression.split()) not in (5, 6): |
| 1060 | raise InvalidPackageManifestError( |
| 1061 | f"Package cron expression must have 5 or 6 fields: {schedule_id}" |
| 1062 | ) |
| 1063 | |
| 1064 | |
| 1065 | def _load_scheduler_tasks(base_dir: Path) -> dict[str, dict[str, Any]]: |
| 1066 | """Every task currently on the daemon's board, keyed by schedule id. |
| 1067 | |
| 1068 | Unreadable rows are skipped rather than raising: one hand-edited task |
| 1069 | must not make a package uninstallable. |
| 1070 | """ |
| 1071 | try: |
| 1072 | state = object_state.get_object_state(SCHEDULER_OBJECT_ID, base_dir=base_dir) |
| 1073 | except Exception: |
| 1074 | return {} |
| 1075 | tasks: dict[str, dict[str, Any]] = {} |
| 1076 | for key, value in state.items(): |
| 1077 | if not str(key).startswith(_TASK_KEY_PREFIX): |
| 1078 | continue |
| 1079 | try: |
| 1080 | task = json.loads(value) |
| 1081 | except (TypeError, ValueError): |
| 1082 | continue |
| 1083 | if isinstance(task, Mapping): |
| 1084 | tasks[str(key)[len(_TASK_KEY_PREFIX):]] = dict(task) |
| 1085 | return tasks |
| 1086 | |
| 1087 | |
| 1088 | def _schedule_change( |
| 1089 | entry: Mapping[str, Any], |
| 1090 | *, |
| 1091 | existing: Mapping[str, dict[str, Any]], |
| 1092 | installable_object_ids: set[str], |
| 1093 | warnings: list[str], |
| 1094 | ) -> dict[str, Any]: |
| 1095 | live = existing.get(entry["id"]) |
| 1096 | if live is None: |
| 1097 | action = "create" |
| 1098 | elif all(live.get(field) == entry[field] |
| 1099 | for field in ("object_id", "method", "payload", "schedule", "type")) \ |
| 1100 | and (live.get("description") or None) == entry["description"]: |
| 1101 | # The description is compared too, so a plan that says "unchanged" |
| 1102 | # is one where nothing at all is about to be written -- a dry run |
| 1103 | # that under-reports is worse than no dry run. |
| 1104 | action = "unchanged" |
| 1105 | else: |
| 1106 | action = "update" |
| 1107 | |
| 1108 | if entry["object_id"] not in installable_object_ids: |
| 1109 | # A schedule aimed at an object that will not exist is the silent |
| 1110 | # failure this feature exists to end, so it blocks the install |
| 1111 | # rather than landing as a task that can never run. |
| 1112 | warnings.append( |
| 1113 | f"Schedule {entry['id']} targets an object this install will not " |
| 1114 | f"provide: {entry['object_id']}" |
| 1115 | ) |
| 1116 | |
| 1117 | change = { |
| 1118 | "id": entry["id"], |
| 1119 | "object_id": entry["object_id"], |
| 1120 | "schedule": entry["schedule"], |
| 1121 | "type": entry["type"], |
| 1122 | "exists": live is not None, |
| 1123 | "action": action, |
| 1124 | } |
| 1125 | if live is not None: |
| 1126 | change["status"] = live.get("status", "active") |
| 1127 | change["run_count"] = live.get("run_count", 0) |
| 1128 | return change |
| 1129 | |
| 1130 | |
| 1131 | def _apply_schedules( |
| 1132 | schedules: Iterable[Mapping[str, Any]], |
| 1133 | planned: Iterable[Mapping[str, Any]], |
| 1134 | *, |
| 1135 | base_dir: Path, |
| 1136 | ) -> list[dict[str, Any]]: |
| 1137 | """Write the package's schedules onto the daemon's task board. |
| 1138 | |
| 1139 | Two things are deliberately preserved across an install: |
| 1140 | |
| 1141 | **Run history** -- last_run/run_count/next_run belong to the daemon, |
| 1142 | not the package. Resetting them on an upgrade would make an operator's |
| 1143 | "when did this last work?" unanswerable. |
| 1144 | |
| 1145 | **A pause** -- if someone paused a task, it stays paused. The package |
| 1146 | declares what SHOULD run; an operator decides what DOES right now, and |
| 1147 | a reinstall that silently restarts a deliberately-stopped nightly pass |
| 1148 | is how an upgrade becomes an incident. |
| 1149 | |
| 1150 | A changed expression clears next_run so the daemon recomputes it from |
| 1151 | the new one; an unchanged expression keeps the pending firing exactly |
| 1152 | where it was, so reinstalling an app does not skip tonight's run. |
| 1153 | """ |
| 1154 | entries = list(schedules) |
| 1155 | plans = list(planned) |
| 1156 | if not entries: |
| 1157 | return [] |
| 1158 | |
| 1159 | manager = object_state.ObjectStateManager(SCHEDULER_OBJECT_ID, base_dir=base_dir) |
| 1160 | applied = [] |
| 1161 | for entry, plan in zip(entries, plans, strict=True): |
| 1162 | key = f"{_TASK_KEY_PREFIX}{entry['id']}" |
| 1163 | raw = manager.get(key) |
| 1164 | live: dict[str, Any] = {} |
| 1165 | if raw is not None: |
| 1166 | try: |
| 1167 | parsed = json.loads(raw) |
| 1168 | except (TypeError, ValueError): |
| 1169 | parsed = None |
| 1170 | if isinstance(parsed, Mapping): |
| 1171 | live = dict(parsed) |
| 1172 | |
| 1173 | task = { |
| 1174 | "id": entry["id"], |
| 1175 | "object_id": entry["object_id"], |
| 1176 | "method": entry["method"], |
| 1177 | "payload": entry["payload"], |
| 1178 | "schedule": entry["schedule"], |
| 1179 | "type": entry["type"], |
| 1180 | "status": live.get("status", "active"), |
| 1181 | } |
| 1182 | if entry.get("description"): |
| 1183 | task["description"] = entry["description"] |
| 1184 | for field in _SCHEDULE_RUNTIME_FIELDS: |
| 1185 | if field in live: |
| 1186 | task[field] = live[field] |
| 1187 | if live and live.get("schedule") != entry["schedule"]: |
| 1188 | task["next_run"] = None |
| 1189 | |
| 1190 | if live and json.dumps(live, sort_keys=True) == json.dumps(task, sort_keys=True): |
| 1191 | status = "unchanged" |
| 1192 | else: |
| 1193 | manager.set(key, json.dumps(task)) |
| 1194 | status = "updated" if live else "written" |
| 1195 | |
| 1196 | applied.append({**plan, "status": status, "task_status": task["status"], |
| 1197 | "destination": f"state/{SCHEDULER_OBJECT_ID}/{key}"}) |
| 1198 | return applied |
| 1199 | |
| 1200 | |
| 1201 | def _normalize_nav(payload: Any, *, package_id: str) -> list[dict[str, Any]]: |
| 1202 | """A `nav` entry declares a door the app puts on the site: |
| 1203 | `{id, label, path, blurb?, surface?, group?, order?}`. |
| 1204 | |
| 1205 | Navigation used to be a hand-maintained list, and there were three of |
| 1206 | them -- the app switcher's JS array, the home page's tile list, and |
| 1207 | the search result URL map -- each edited by whoever remembered. They |
| 1208 | disagreed by four entries and between them knew about none of the |
| 1209 | eight newest apps, so the front door of the server advertised a |
| 1210 | server that no longer existed. A menu built by hand rots the moment |
| 1211 | somebody ships without editing it; a menu FOLDED over what every |
| 1212 | installed package declares cannot. |
| 1213 | |
| 1214 | The id becomes the `nav_entries` record id, so it is restricted to |
| 1215 | the characters a record id can safely hold. `path` is validated for |
| 1216 | shape only -- whether anything answers it is a routing question this |
| 1217 | module deliberately does not pretend to answer (see `_nav_change`). |
| 1218 | """ |
| 1219 | entries = _list_field(payload, package_id=package_id, section="nav") |
| 1220 | normalized: list[dict[str, Any]] = [] |
| 1221 | seen: set[str] = set() |
| 1222 | for entry in entries: |
| 1223 | mapping = _entry_mapping(entry, package_id=package_id, section="nav") |
| 1224 | nav_id = _required_text(mapping, "id", package_id=package_id) |
| 1225 | if not _NAV_ID_RE.fullmatch(nav_id): |
| 1226 | raise InvalidPackageManifestError(f"Invalid package nav id: {nav_id}") |
| 1227 | if nav_id in seen: |
| 1228 | raise InvalidPackageManifestError(f"Duplicate package nav id: {nav_id}") |
| 1229 | seen.add(nav_id) |
| 1230 | |
| 1231 | label = _required_text(mapping, "label", package_id=package_id) |
| 1232 | path = _required_text(mapping, "path", package_id=package_id) |
| 1233 | if not path.startswith("/"): |
| 1234 | raise InvalidPackageManifestError( |
| 1235 | f"Package nav path must start with '/': {nav_id}" |
| 1236 | ) |
| 1237 | if any(character.isspace() for character in path): |
| 1238 | # A path with a space or a newline in it is a link that never |
| 1239 | # resolves and a TSV cell that can break the row it lives in. |
| 1240 | raise InvalidPackageManifestError( |
| 1241 | f"Package nav path may not contain whitespace: {nav_id}" |
| 1242 | ) |
| 1243 | |
| 1244 | surface = (_optional_text(mapping.get("surface")) or "member").lower() |
| 1245 | if surface not in _NAV_SURFACES: |
| 1246 | raise InvalidPackageManifestError( |
| 1247 | f"Package nav surface must be public|member|operator|hidden: {nav_id}" |
| 1248 | ) |
| 1249 | |
| 1250 | order = mapping.get("order", _NAV_DEFAULT_ORDER) |
| 1251 | if order in (None, ""): |
| 1252 | order = _NAV_DEFAULT_ORDER |
| 1253 | if isinstance(order, bool) or not isinstance(order, int): |
| 1254 | raise InvalidPackageManifestError( |
| 1255 | f"Package nav order must be an integer: {nav_id}" |
| 1256 | ) |
| 1257 | |
| 1258 | normalized.append({ |
| 1259 | "id": nav_id, |
| 1260 | "label": label, |
| 1261 | "path": path, |
| 1262 | "blurb": _optional_text(mapping.get("blurb")) or "", |
| 1263 | "group": _optional_text(mapping.get("group")) or _NAV_DEFAULT_GROUP, |
| 1264 | "surface": surface, |
| 1265 | "order": order, |
| 1266 | }) |
| 1267 | return normalized |
| 1268 | |
| 1269 | |
| 1270 | def _load_nav_entries( |
| 1271 | base_dir: Path, |
| 1272 | *, |
| 1273 | roots: Iterable[Path] | None = None, |
| 1274 | ) -> dict[str, dict[str, str]] | None: |
| 1275 | """Every nav entry currently registered, keyed by record id. |
| 1276 | |
| 1277 | ``None`` means the `nav_entries` collection is not installed at all |
| 1278 | (app-nav absent). That is not an error: a package must never fail to |
| 1279 | install because the navigation app is missing, any more than an app |
| 1280 | should fail because nobody has opened the menu yet. |
| 1281 | """ |
| 1282 | try: |
| 1283 | rows = object_records.read_collection_records( |
| 1284 | NAV_COLLECTION, base_dir=base_dir, roots=roots) |
| 1285 | except (object_collections.CollectionNotFoundError, |
| 1286 | object_collections.InvalidCollectionNameError): |
| 1287 | return None |
| 1288 | except (OSError, ValueError): |
| 1289 | # A hand-edited row must not make a package uninstallable, the |
| 1290 | # same posture _load_scheduler_tasks takes. |
| 1291 | return {} |
| 1292 | return {str(row.get("id") or ""): dict(row) for row in rows if row.get("id")} |
| 1293 | |
| 1294 | |
| 1295 | def _nav_change( |
| 1296 | entry: Mapping[str, Any], |
| 1297 | *, |
| 1298 | existing: Mapping[str, Mapping[str, str]], |
| 1299 | package_id: str, |
| 1300 | warnings: list[str], |
| 1301 | ) -> dict[str, Any]: |
| 1302 | live = existing.get(entry["id"]) |
| 1303 | if live is None: |
| 1304 | action = "create" |
| 1305 | elif all(str(live.get(field) or "") == str(entry[field]) for field in _NAV_PACKAGE_FIELDS): |
| 1306 | # Every package-owned field is compared, blurb and order |
| 1307 | # included, so a plan that says "unchanged" is one where nothing |
| 1308 | # at all is about to be written -- a dry run that under-reports |
| 1309 | # is worse than no dry run. |
| 1310 | action = "unchanged" |
| 1311 | else: |
| 1312 | action = "update" |
| 1313 | |
| 1314 | # Whether a path is actually SERVED is not knowable here: routing has |
| 1315 | # three sources (convention, site_routes records, views records) and |
| 1316 | # two of them are data this install may be about to seed. Guessing |
| 1317 | # would produce false blockers, so the only thing refused is the one |
| 1318 | # collision that is unambiguous and destructive: two packages fighting |
| 1319 | # over one door, where whichever installs last silently wins. |
| 1320 | owner = str((live or {}).get("package") or "") |
| 1321 | if owner and owner != package_id: |
| 1322 | warnings.append( |
| 1323 | f"Nav entry {entry['id']} is already registered by another package: {owner}" |
| 1324 | ) |
| 1325 | for other_id, other in existing.items(): |
| 1326 | if other_id == entry["id"]: |
| 1327 | continue |
| 1328 | other_owner = str(other.get("package") or "") |
| 1329 | if other_owner and other_owner != package_id and other.get("path") == entry["path"]: |
| 1330 | warnings.append( |
| 1331 | f"Nav entry {entry['id']} claims a path already registered by " |
| 1332 | f"{other_owner}: {entry['path']}" |
| 1333 | ) |
| 1334 | |
| 1335 | return { |
| 1336 | "id": entry["id"], |
| 1337 | "label": entry["label"], |
| 1338 | "path": entry["path"], |
| 1339 | "group": entry["group"], |
| 1340 | "surface": entry["surface"], |
| 1341 | "exists": live is not None, |
| 1342 | "action": action, |
| 1343 | } |
| 1344 | |
| 1345 | |
| 1346 | def _apply_nav( |
| 1347 | nav: Iterable[Mapping[str, Any]], |
| 1348 | planned: Iterable[Mapping[str, Any]], |
| 1349 | *, |
| 1350 | package_id: str, |
| 1351 | base_dir: Path, |
| 1352 | roots: Iterable[Path] | None = None, |
| 1353 | ) -> list[dict[str, Any]]: |
| 1354 | """Register the package's doors in the `nav_entries` collection. |
| 1355 | |
| 1356 | Two things are deliberately preserved across an install: |
| 1357 | |
| 1358 | **The operator's `hidden`** -- `operator_hidden` is the one column an |
| 1359 | install never writes on an existing row. The package declares what |
| 1360 | the app IS; the operator decides what they want to SEE, and an |
| 1361 | upgrade that silently un-hides a page somebody deliberately took off |
| 1362 | the menu is the same class of incident as one that restarts a paused |
| 1363 | nightly pass. |
| 1364 | |
| 1365 | **Somebody else's row** -- entries this package did not register are |
| 1366 | not touched and not deleted. Removing a `nav` entry from a manifest |
| 1367 | leaves the row alone, matching the rest of install: deregistering |
| 1368 | never destroys. |
| 1369 | |
| 1370 | A missing `nav_entries` collection is reported and skipped rather |
| 1371 | than raised. The nav app is not a dependency of being installable. |
| 1372 | """ |
| 1373 | entries = list(nav) |
| 1374 | plans = list(planned) |
| 1375 | if not entries: |
| 1376 | return [] |
| 1377 | |
| 1378 | existing = _load_nav_entries(base_dir, roots=roots) |
| 1379 | if existing is None: |
| 1380 | return [ |
| 1381 | {**plan, "status": "skipped", |
| 1382 | "reason": f"{NAV_COLLECTION} collection is not installed"} |
| 1383 | for plan in plans |
| 1384 | ] |
| 1385 | |
| 1386 | applied = [] |
| 1387 | for entry, plan in zip(entries, plans, strict=True): |
| 1388 | live = existing.get(entry["id"]) |
| 1389 | row = { |
| 1390 | "package": package_id, |
| 1391 | "label": entry["label"], |
| 1392 | "path": entry["path"], |
| 1393 | "blurb": entry["blurb"], |
| 1394 | "group": entry["group"], |
| 1395 | "surface": entry["surface"], |
| 1396 | "order": str(entry["order"]), |
| 1397 | } |
| 1398 | if live is None: |
| 1399 | object_records.create_collection_record( |
| 1400 | NAV_COLLECTION, |
| 1401 | {"id": entry["id"], "operator_hidden": "false", **row}, |
| 1402 | base_dir=base_dir, roots=roots, actor=f"package:{package_id}", |
| 1403 | ) |
| 1404 | status = "written" |
| 1405 | elif all(str(live.get(field) or "") == row[field] for field in row): |
| 1406 | status = "unchanged" |
| 1407 | else: |
| 1408 | # `row` carries no operator_hidden key at all, so the update |
| 1409 | # cannot reach that column even by accident. |
| 1410 | object_records.update_collection_record( |
| 1411 | NAV_COLLECTION, entry["id"], row, |
| 1412 | base_dir=base_dir, roots=roots, actor=f"package:{package_id}", |
| 1413 | ) |
| 1414 | status = "updated" |
| 1415 | |
| 1416 | applied.append({**plan, "status": status, |
| 1417 | "destination": f"collections/{NAV_COLLECTION}/{entry['id']}"}) |
| 1418 | return applied |
| 1419 | |
| 1420 | |
| 1421 | def _normalize_attention(payload: Any, *, package_id: str) -> list[dict[str, Any]]: |
| 1422 | """An `attention` entry declares one queue of things that need a human: |
| 1423 | `{id, object_id, label, path, nav_id?, group?, severity?}`. |
| 1424 | |
| 1425 | Every gate and derived state on this server ends in a pile somebody |
| 1426 | has to look at -- receipts waiting to be confirmed into expenses, time |
| 1427 | entries waiting for an approver, invoices past their due date, bank |
| 1428 | lines nothing matched. The system computes those piles constantly and |
| 1429 | then discards them: `system_scan_processor` returns a field called |
| 1430 | `needing_a_human` that no surface has ever read. What was missing was |
| 1431 | not the arithmetic, it was a place to declare that the arithmetic |
| 1432 | MEANS something -- and the only honest place for that definition is |
| 1433 | the package that owns the domain, because nobody else knows that |
| 1434 | `extracted` is a waiting room and `confirmed` is not. |
| 1435 | |
| 1436 | `object_id` names the provider, an object exposing `COUNT(request) -> |
| 1437 | {"count": int, "detail"?: str}`. `path` is validated for shape only, |
| 1438 | exactly as `nav` paths are: whether anything answers it is a routing |
| 1439 | question this module deliberately does not pretend to answer. |
| 1440 | `nav_id` is the door this count decorates, so the app list can read |
| 1441 | "Invoices - 5 overdue" without a second surface to maintain; it is |
| 1442 | optional because a queue can belong to a package that ships no page |
| 1443 | at all (app-intake is one). |
| 1444 | """ |
| 1445 | entries = _list_field(payload, package_id=package_id, section="attention") |
| 1446 | normalized: list[dict[str, Any]] = [] |
| 1447 | seen: set[str] = set() |
| 1448 | for entry in entries: |
| 1449 | mapping = _entry_mapping(entry, package_id=package_id, section="attention") |
| 1450 | attention_id = _required_text(mapping, "id", package_id=package_id) |
| 1451 | if not _ATTENTION_ID_RE.fullmatch(attention_id): |
| 1452 | raise InvalidPackageManifestError( |
| 1453 | f"Invalid package attention id: {attention_id}") |
| 1454 | if attention_id in seen: |
| 1455 | raise InvalidPackageManifestError( |
| 1456 | f"Duplicate package attention id: {attention_id}") |
| 1457 | seen.add(attention_id) |
| 1458 | |
| 1459 | object_id = _required_text(mapping, "object_id", package_id=package_id) |
| 1460 | if not validate_object_id(object_id): |
| 1461 | raise InvalidPackageManifestError( |
| 1462 | f"Invalid package attention object_id: {object_id}") |
| 1463 | |
| 1464 | label = _required_text(mapping, "label", package_id=package_id) |
| 1465 | path = _required_text(mapping, "path", package_id=package_id) |
| 1466 | if not path.startswith("/"): |
| 1467 | raise InvalidPackageManifestError( |
| 1468 | f"Package attention path must start with '/': {attention_id}" |
| 1469 | ) |
| 1470 | if any(character.isspace() for character in path): |
| 1471 | # A count nobody can click is a number on a poster. A path |
| 1472 | # with a space or a newline in it is both a link that never |
| 1473 | # resolves and a TSV cell that can break the row it lives in. |
| 1474 | raise InvalidPackageManifestError( |
| 1475 | f"Package attention path may not contain whitespace: {attention_id}" |
| 1476 | ) |
| 1477 | |
| 1478 | nav_id = _optional_text(mapping.get("nav_id")) or "" |
| 1479 | if nav_id and not _NAV_ID_RE.fullmatch(nav_id): |
| 1480 | raise InvalidPackageManifestError( |
| 1481 | f"Invalid package attention nav_id: {attention_id}") |
| 1482 | |
| 1483 | severity = (_optional_text(mapping.get("severity")) |
| 1484 | or _ATTENTION_DEFAULT_SEVERITY).lower() |
| 1485 | if severity not in _ATTENTION_SEVERITIES: |
| 1486 | raise InvalidPackageManifestError( |
| 1487 | f"Package attention severity must be normal|warning|urgent: {attention_id}" |
| 1488 | ) |
| 1489 | |
| 1490 | normalized.append({ |
| 1491 | "id": attention_id, |
| 1492 | "object_id": object_id, |
| 1493 | "label": label, |
| 1494 | "path": path, |
| 1495 | "nav_id": nav_id, |
| 1496 | "group": _optional_text(mapping.get("group")) or _ATTENTION_DEFAULT_GROUP, |
| 1497 | "severity": severity, |
| 1498 | }) |
| 1499 | return normalized |
| 1500 | |
| 1501 | |
| 1502 | def _load_attention_sources( |
| 1503 | base_dir: Path, |
| 1504 | *, |
| 1505 | roots: Iterable[Path] | None = None, |
| 1506 | ) -> dict[str, dict[str, str]] | None: |
| 1507 | """Every attention source currently registered, keyed by record id. |
| 1508 | |
| 1509 | ``None`` means the `attention_sources` collection is not installed at |
| 1510 | all (app-nav absent). Same posture as `_load_nav_entries`: a package |
| 1511 | must never fail to install because the home screen is missing, any |
| 1512 | more than an app should fail because nobody has looked at it yet. |
| 1513 | """ |
| 1514 | try: |
| 1515 | rows = object_records.read_collection_records( |
| 1516 | ATTENTION_COLLECTION, base_dir=base_dir, roots=roots) |
| 1517 | except (object_collections.CollectionNotFoundError, |
| 1518 | object_collections.InvalidCollectionNameError): |
| 1519 | return None |
| 1520 | except (OSError, ValueError): |
| 1521 | # A hand-edited row must not make a package uninstallable, the |
| 1522 | # same posture _load_nav_entries takes. |
| 1523 | return {} |
| 1524 | return {str(row.get("id") or ""): dict(row) for row in rows if row.get("id")} |
| 1525 | |
| 1526 | |
| 1527 | def _attention_change( |
| 1528 | entry: Mapping[str, Any], |
| 1529 | *, |
| 1530 | existing: Mapping[str, Mapping[str, str]], |
| 1531 | package_id: str, |
| 1532 | installable_object_ids: set[str], |
| 1533 | warnings: list[str], |
| 1534 | ) -> dict[str, Any]: |
| 1535 | live = existing.get(entry["id"]) |
| 1536 | if live is None: |
| 1537 | action = "create" |
| 1538 | elif all(str(live.get(field) or "") == str(entry[field]) |
| 1539 | for field in _ATTENTION_PACKAGE_FIELDS): |
| 1540 | # Every package-owned field is compared, nav_id and severity |
| 1541 | # included, so a plan that says "unchanged" is one where nothing |
| 1542 | # at all is about to be written. |
| 1543 | action = "unchanged" |
| 1544 | else: |
| 1545 | action = "update" |
| 1546 | |
| 1547 | if entry["object_id"] not in installable_object_ids: |
| 1548 | # A counter aimed at an object that will not exist reads as zero |
| 1549 | # forever, and a queue that reads zero forever is indistinguishable |
| 1550 | # from a queue that is empty. That silent failure is the entire |
| 1551 | # reason this section exists, so it blocks the install. |
| 1552 | warnings.append( |
| 1553 | f"Attention source {entry['id']} targets an object this install " |
| 1554 | f"will not provide: {entry['object_id']}" |
| 1555 | ) |
| 1556 | |
| 1557 | # Two packages writing one attention_sources row is two definitions of |
| 1558 | # what needs a human, where whichever installed last silently wins. |
| 1559 | # Unlike nav there is no path-collision rule: two queues legitimately |
| 1560 | # point at one list (an orders page can be both "to pick" and "to |
| 1561 | # invoice"), and blocking that would be a false blocker. |
| 1562 | owner = str((live or {}).get("package") or "") |
| 1563 | if owner and owner != package_id: |
| 1564 | warnings.append( |
| 1565 | f"Attention source {entry['id']} is already registered by another " |
| 1566 | f"package: {owner}" |
| 1567 | ) |
| 1568 | |
| 1569 | return { |
| 1570 | "id": entry["id"], |
| 1571 | "object_id": entry["object_id"], |
| 1572 | "label": entry["label"], |
| 1573 | "path": entry["path"], |
| 1574 | "nav_id": entry["nav_id"], |
| 1575 | "group": entry["group"], |
| 1576 | "severity": entry["severity"], |
| 1577 | "exists": live is not None, |
| 1578 | "action": action, |
| 1579 | } |
| 1580 | |
| 1581 | |
| 1582 | def _apply_attention( |
| 1583 | attention: Iterable[Mapping[str, Any]], |
| 1584 | planned: Iterable[Mapping[str, Any]], |
| 1585 | *, |
| 1586 | package_id: str, |
| 1587 | base_dir: Path, |
| 1588 | roots: Iterable[Path] | None = None, |
| 1589 | ) -> list[dict[str, Any]]: |
| 1590 | """Register the package's queues in the `attention_sources` collection. |
| 1591 | |
| 1592 | Two things are deliberately preserved across an install: |
| 1593 | |
| 1594 | **The operator's `operator_muted`** -- the one column an install never |
| 1595 | writes on an existing row. The package declares what a queue MEANS; |
| 1596 | the operator decides whether this deployment wants to be told about |
| 1597 | it, and an upgrade that silently un-mutes a counter somebody turned |
| 1598 | off is the same class of incident as one that restarts a paused |
| 1599 | nightly pass. |
| 1600 | |
| 1601 | **Somebody else's row** -- sources this package did not register are |
| 1602 | not touched and not deleted. Removing an `attention` entry from a |
| 1603 | manifest leaves the row alone, matching the rest of install: |
| 1604 | deregistering never destroys. (The rollup pass is what stops feeding |
| 1605 | a stale row, because it only executes providers it can still resolve.) |
| 1606 | |
| 1607 | A missing `attention_sources` collection is reported and skipped |
| 1608 | rather than raised. The home screen is not a dependency of being |
| 1609 | installable. |
| 1610 | """ |
| 1611 | entries = list(attention) |
| 1612 | plans = list(planned) |
| 1613 | if not entries: |
| 1614 | return [] |
| 1615 | |
| 1616 | existing = _load_attention_sources(base_dir, roots=roots) |
| 1617 | if existing is None: |
| 1618 | return [ |
| 1619 | {**plan, "status": "skipped", |
| 1620 | "reason": f"{ATTENTION_COLLECTION} collection is not installed"} |
| 1621 | for plan in plans |
| 1622 | ] |
| 1623 | |
| 1624 | applied = [] |
| 1625 | for entry, plan in zip(entries, plans, strict=True): |
| 1626 | live = existing.get(entry["id"]) |
| 1627 | row = { |
| 1628 | "package": package_id, |
| 1629 | "object_id": entry["object_id"], |
| 1630 | "label": entry["label"], |
| 1631 | "path": entry["path"], |
| 1632 | "nav_id": entry["nav_id"], |
| 1633 | "group": entry["group"], |
| 1634 | "severity": entry["severity"], |
| 1635 | } |
| 1636 | if live is None: |
| 1637 | object_records.create_collection_record( |
| 1638 | ATTENTION_COLLECTION, |
| 1639 | {"id": entry["id"], "operator_muted": "false", **row}, |
| 1640 | base_dir=base_dir, roots=roots, actor=f"package:{package_id}", |
| 1641 | ) |
| 1642 | status = "written" |
| 1643 | elif all(str(live.get(field) or "") == row[field] for field in row): |
| 1644 | status = "unchanged" |
| 1645 | else: |
| 1646 | # `row` carries no operator_muted key at all, so the update |
| 1647 | # cannot reach that column even by accident. |
| 1648 | object_records.update_collection_record( |
| 1649 | ATTENTION_COLLECTION, entry["id"], row, |
| 1650 | base_dir=base_dir, roots=roots, actor=f"package:{package_id}", |
| 1651 | ) |
| 1652 | status = "updated" |
| 1653 | |
| 1654 | applied.append({**plan, "status": status, |
| 1655 | "destination": f"collections/{ATTENTION_COLLECTION}/{entry['id']}"}) |
| 1656 | return applied |
| 1657 | |
| 1658 | |
| 1659 | def iter_connectors(*, root: Path | str = PACKAGES_DIR) -> list[dict[str, str]]: |
| 1660 | """Every connector declaration under a package root, each with its module |
| 1661 | resolved to an absolute path proven to live INSIDE its package. Robust to a |
| 1662 | single bad package: a manifest that won't parse, or a module that is missing |
| 1663 | or escapes the package dir, is skipped (never raises) so one bad package |
| 1664 | can't blind the whole reconcile pass.""" |
| 1665 | packages_root = Path(root) |
| 1666 | if not packages_root.is_dir(): |
| 1667 | return [] |
| 1668 | out: list[dict[str, str]] = [] |
| 1669 | for path in sorted(packages_root.iterdir(), key=lambda item: item.name): |
| 1670 | if not path.is_dir() or not validate_package_id(path.name): |
| 1671 | continue |
| 1672 | if not (path / MANIFEST_FILE).is_file(): |
| 1673 | continue |
| 1674 | try: |
| 1675 | package = _load_package(path.name, path) |
| 1676 | except (InvalidPackageManifestError, InvalidPackageIdError, OSError, ValueError): |
| 1677 | continue |
| 1678 | for decl in package.get("connectors", []): |
| 1679 | status = _package_file_status(path, decl["module"]) |
| 1680 | if not status["exists"]: # missing module, or escapes the package |
| 1681 | continue |
| 1682 | out.append({ |
| 1683 | "package_id": package["id"], |
| 1684 | "collection": decl["collection"], |
| 1685 | "module": str((path / decl["module"]).resolve()), |
| 1686 | "entry": decl["entry"], |
| 1687 | }) |
| 1688 | return out |
| 1689 | |
| 1690 | |
| 1691 | def _normalize_dependencies(payload: Any, *, package_id: str) -> list[dict[str, str | None]]: |
| 1692 | entries = _list_field(payload, package_id=package_id, section="dependencies") |
| 1693 | normalized = [] |
| 1694 | for entry in entries: |
| 1695 | if isinstance(entry, str): |
| 1696 | dependency_id = entry |
| 1697 | version = None |
| 1698 | else: |
| 1699 | mapping = _entry_mapping(entry, package_id=package_id, section="dependencies") |
| 1700 | dependency_id = _required_text(mapping, "id", package_id=package_id) |
| 1701 | version = _optional_text(mapping.get("version")) |
| 1702 | if not validate_package_id(dependency_id): |
| 1703 | raise InvalidPackageManifestError(f"Invalid package dependency id: {dependency_id}") |
| 1704 | normalized.append({"id": dependency_id, "version": version}) |
| 1705 | return normalized |
| 1706 | |
| 1707 | |
| 1708 | def _object_change( |
| 1709 | entry: Mapping[str, str], |
| 1710 | *, |
| 1711 | package_dir: Path, |
| 1712 | object_roots: Iterable[Path] | None, |
| 1713 | warnings: list[str], |
| 1714 | ) -> dict[str, Any]: |
| 1715 | file_status = _package_file_status(package_dir, entry["path"]) |
| 1716 | if not file_status["exists"]: |
| 1717 | warnings.append(f"Missing package object file: {entry['path']}") |
| 1718 | installed = resolve_object_id(entry["id"], object_roots) is not None |
| 1719 | return { |
| 1720 | "id": entry["id"], |
| 1721 | "path": entry["path"], |
| 1722 | "exists": file_status["exists"], |
| 1723 | "action": "replace" if installed else "create", |
| 1724 | "installed": installed, |
| 1725 | } |
| 1726 | |
| 1727 | |
| 1728 | def _schema_change( |
| 1729 | entry: Mapping[str, str], |
| 1730 | *, |
| 1731 | package_dir: Path, |
| 1732 | base_dir: Path, |
| 1733 | warnings: list[str], |
| 1734 | ) -> dict[str, Any]: |
| 1735 | file_status = _package_file_status(package_dir, entry["path"]) |
| 1736 | if not file_status["exists"]: |
| 1737 | warnings.append(f"Missing package schema file: {entry['path']}") |
| 1738 | installed = (base_dir / "schemas" / f"{entry['collection']}.json").is_file() |
| 1739 | return { |
| 1740 | "collection": entry["collection"], |
| 1741 | "path": entry["path"], |
| 1742 | "exists": file_status["exists"], |
| 1743 | "action": "replace" if installed else "create", |
| 1744 | "installed": installed, |
| 1745 | } |
| 1746 | |
| 1747 | |
| 1748 | def _seed_change( |
| 1749 | entry: Mapping[str, str], |
| 1750 | *, |
| 1751 | package_dir: Path, |
| 1752 | base_dir: Path, |
| 1753 | warnings: list[str], |
| 1754 | ) -> dict[str, Any]: |
| 1755 | file_status = _package_file_status(package_dir, entry["path"]) |
| 1756 | if not file_status["exists"]: |
| 1757 | warnings.append(f"Missing package seed file: {entry['path']}") |
| 1758 | installed = (base_dir / "collections" / entry["collection"] / "records.tsv").is_file() |
| 1759 | return { |
| 1760 | "collection": entry["collection"], |
| 1761 | "path": entry["path"], |
| 1762 | "exists": file_status["exists"], |
| 1763 | # Seed is install-once: on an upgrade where the collection already has |
| 1764 | # records, seeding is skipped so live data is preserved (not merged). |
| 1765 | "action": "skip" if installed else "create", |
| 1766 | "installed": installed, |
| 1767 | } |
| 1768 | |
| 1769 | |
| 1770 | def _migration_change( |
| 1771 | entry: Mapping[str, str], |
| 1772 | *, |
| 1773 | package: str, |
| 1774 | package_dir: Path, |
| 1775 | base_dir: Path, |
| 1776 | warnings: list[str], |
| 1777 | ) -> dict[str, Any]: |
| 1778 | file_status = _package_file_status(package_dir, entry["path"]) |
| 1779 | if not file_status["exists"]: |
| 1780 | warnings.append(f"Missing package migration file: {entry['path']}") |
| 1781 | marker = base_dir / PACKAGE_MIGRATIONS_DIR / package / f"{entry['id']}.json" |
| 1782 | applied = marker.is_file() |
| 1783 | return { |
| 1784 | "id": entry["id"], |
| 1785 | "path": entry["path"], |
| 1786 | "exists": file_status["exists"], |
| 1787 | "action": "skip" if applied else "apply", |
| 1788 | "applied": applied, |
| 1789 | } |
| 1790 | |
| 1791 | |
| 1792 | def _permission_change( |
| 1793 | entry: Mapping[str, str], |
| 1794 | *, |
| 1795 | package_id: str, |
| 1796 | package_dir: Path, |
| 1797 | base_dir: Path, |
| 1798 | warnings: list[str], |
| 1799 | ) -> dict[str, Any]: |
| 1800 | change = { |
| 1801 | "path": entry["path"], |
| 1802 | "action": "merge", |
| 1803 | "exists": False, |
| 1804 | "rules": 0, |
| 1805 | "new_rules": 0, |
| 1806 | } |
| 1807 | file_status = _package_file_status(package_dir, entry["path"]) |
| 1808 | if not file_status["exists"]: |
| 1809 | warnings.append(f"Missing package permissions file: {entry['path']}") |
| 1810 | return change |
| 1811 | change["exists"] = True |
| 1812 | |
| 1813 | try: |
| 1814 | rules = _load_permission_rules(package_dir, entry, package_id=package_id) |
| 1815 | except PackageInstallError as exc: |
| 1816 | warnings.append(str(exc)) |
| 1817 | return change |
| 1818 | |
| 1819 | change["rules"] = len(rules) |
| 1820 | try: |
| 1821 | policy = object_permission_store.load_policy(base_dir) |
| 1822 | existing_keys = {_rule_merge_key(rule) for rule in policy.rules} |
| 1823 | except ValueError: |
| 1824 | existing_keys = set() |
| 1825 | change["new_rules"] = sum(1 for rule in rules if _rule_merge_key(rule) not in existing_keys) |
| 1826 | return change |
| 1827 | |
| 1828 | |
| 1829 | def _load_permission_rules( |
| 1830 | package_dir: Path, |
| 1831 | entry: Mapping[str, str], |
| 1832 | *, |
| 1833 | package_id: str, |
| 1834 | ) -> list[object_permissions.PermissionRule]: |
| 1835 | source = _package_file(package_dir, entry["path"]) |
| 1836 | try: |
| 1837 | payload = json.loads(source.read_text(encoding="utf-8")) |
| 1838 | except (OSError, json.JSONDecodeError) as exc: |
| 1839 | raise PackageInstallError( |
| 1840 | f"Package permissions file contains invalid JSON: {entry['path']}" |
| 1841 | ) from exc |
| 1842 | |
| 1843 | rules_payload = payload.get("rules") if isinstance(payload, dict) else payload |
| 1844 | if not isinstance(rules_payload, list): |
| 1845 | raise PackageInstallError( |
| 1846 | f"Package permissions file must contain a rules list: {entry['path']}" |
| 1847 | ) |
| 1848 | |
| 1849 | rules = [] |
| 1850 | for rule_payload in rules_payload: |
| 1851 | if not isinstance(rule_payload, dict): |
| 1852 | raise PackageInstallError( |
| 1853 | f"Package permission rules must be objects: {entry['path']}" |
| 1854 | ) |
| 1855 | merged_payload = {**rule_payload, "package": package_id} |
| 1856 | try: |
| 1857 | rules.append(object_permissions.rule_from_dict(merged_payload)) |
| 1858 | except ValueError as exc: |
| 1859 | raise PackageInstallError( |
| 1860 | f"Package permission rule is invalid in {entry['path']}: {exc}" |
| 1861 | ) from exc |
| 1862 | return rules |
| 1863 | |
| 1864 | |
| 1865 | def _rule_merge_key(rule: object_permissions.PermissionRule) -> str: |
| 1866 | payload = object_permissions.rule_to_dict(rule) |
| 1867 | payload.pop("reason", None) |
| 1868 | payload.pop("package", None) |
| 1869 | return json.dumps(payload, sort_keys=True) |
| 1870 | |
| 1871 | |
| 1872 | def _merge_permission_rules( |
| 1873 | rules: list[object_permissions.PermissionRule], |
| 1874 | *, |
| 1875 | base_dir: Path, |
| 1876 | ) -> tuple[int, int]: |
| 1877 | """Append new package rules to the policy; return (total, newly added).""" |
| 1878 | policy = object_permission_store.load_policy(base_dir) |
| 1879 | existing_keys = {_rule_merge_key(rule) for rule in policy.rules} |
| 1880 | |
| 1881 | added = [] |
| 1882 | for rule in rules: |
| 1883 | if _rule_merge_key(rule) in existing_keys: |
| 1884 | continue |
| 1885 | added.append(rule) |
| 1886 | existing_keys.add(_rule_merge_key(rule)) |
| 1887 | |
| 1888 | if added: |
| 1889 | merged = object_permissions.PermissionPolicy( |
| 1890 | access_mode=policy.access_mode, |
| 1891 | rules=tuple(policy.rules) + tuple(added), |
| 1892 | roles=policy.roles, |
| 1893 | user_roles=policy.user_roles, |
| 1894 | admin_roles=policy.admin_roles, |
| 1895 | ) |
| 1896 | object_permission_store.save_policy(merged, base_dir=base_dir) |
| 1897 | |
| 1898 | return len(rules), len(added) |
| 1899 | |
| 1900 | |
| 1901 | def _path_change( |
| 1902 | entry: Mapping[str, str], |
| 1903 | *, |
| 1904 | package_dir: Path, |
| 1905 | section: str, |
| 1906 | action: str, |
| 1907 | warnings: list[str], |
| 1908 | ) -> dict[str, Any]: |
| 1909 | file_status = _package_file_status(package_dir, entry["path"]) |
| 1910 | if not file_status["exists"]: |
| 1911 | warnings.append(f"Missing package {section} file: {entry['path']}") |
| 1912 | return { |
| 1913 | "path": entry["path"], |
| 1914 | "exists": file_status["exists"], |
| 1915 | "action": action, |
| 1916 | } |
| 1917 | |
| 1918 | |
| 1919 | def _package_file_status(package_dir: Path, relative_path: str) -> dict[str, bool]: |
| 1920 | package_root = package_dir.resolve() |
| 1921 | candidate = package_dir / relative_path |
| 1922 | resolved = candidate.resolve(strict=False) |
| 1923 | inside = resolved == package_root or package_root in resolved.parents |
| 1924 | return { |
| 1925 | "exists": inside and candidate.is_file(), |
| 1926 | "inside_package": inside, |
| 1927 | } |
| 1928 | |
| 1929 | |
| 1930 | def _install_blockers( |
| 1931 | plan: Mapping[str, Any], |
| 1932 | *, |
| 1933 | package: Mapping[str, Any], |
| 1934 | allow_replace: bool, |
| 1935 | ) -> list[str]: |
| 1936 | blockers = [str(warning) for warning in plan.get("warnings", [])] |
| 1937 | |
| 1938 | if package["migrations"]: |
| 1939 | blockers.append("Package migration execution is not implemented yet") |
| 1940 | |
| 1941 | if not allow_replace: |
| 1942 | for entry in plan["objects"]: |
| 1943 | if entry["action"] == "replace": |
| 1944 | blockers.append(f"Object already exists; set allow_replace=true: {entry['id']}") |
| 1945 | for entry in plan["schemas"]: |
| 1946 | if entry["action"] == "replace": |
| 1947 | blockers.append( |
| 1948 | f"Schema already exists; set allow_replace=true: {entry['collection']}" |
| 1949 | ) |
| 1950 | |
| 1951 | # Existing seed data is NOT a blocker: seed is install-once, so an upgrade |
| 1952 | # of an already-installed package skips seeding and preserves live records |
| 1953 | # (see the seed apply step). This is what makes "upgrade the app, keep the |
| 1954 | # data" work — the package ships code + schema + a seed template, while the |
| 1955 | # records live outside the package in the server's data dir. |
| 1956 | |
| 1957 | return blockers |
| 1958 | |
| 1959 | |
| 1960 | def _package_file(package_dir: Path, relative_path: str) -> Path: |
| 1961 | status = _package_file_status(package_dir, relative_path) |
| 1962 | if not status["inside_package"]: |
| 1963 | raise PackageInstallError(f"Package file escapes package directory: {relative_path}") |
| 1964 | if not status["exists"]: |
| 1965 | raise PackageInstallError(f"Package file does not exist: {relative_path}") |
| 1966 | return package_dir / relative_path |
| 1967 | |
| 1968 | |
| 1969 | def _object_destination(entry: Mapping[str, str], object_root: Path) -> Path: |
| 1970 | relative = Path(entry["path"]) |
| 1971 | if relative.parts and relative.parts[0] == "objects": |
| 1972 | relative = Path(*relative.parts[1:]) |
| 1973 | if not relative.parts or relative.suffix != ".py": |
| 1974 | raise PackageInstallError(f"Package object path must point to a Python file: {entry['path']}") |
| 1975 | destination = object_root / relative |
| 1976 | _ensure_inside(destination, object_root, label="object") |
| 1977 | return destination |
| 1978 | |
| 1979 | |
| 1980 | def _ensure_inside(path: Path, root: Path, *, label: str) -> None: |
| 1981 | resolved_path = path.resolve(strict=False) |
| 1982 | resolved_root = root.resolve(strict=False) |
| 1983 | try: |
| 1984 | resolved_path.relative_to(resolved_root) |
| 1985 | except ValueError as exc: |
| 1986 | raise PackageInstallError(f"Package {label} destination escapes its root: {path}") from exc |
| 1987 | |
| 1988 | |
| 1989 | def _root_for_path(path: Path | None, roots: Iterable[Path]) -> Path | None: |
| 1990 | if path is None: |
| 1991 | return None |
| 1992 | for root in roots: |
| 1993 | try: |
| 1994 | _ensure_inside(path, root, label="object") |
| 1995 | except PackageInstallError: |
| 1996 | continue |
| 1997 | return root |
| 1998 | return None |
| 1999 | |
| 2000 | |
| 2001 | def _merge_seed_rows( |
| 2002 | collection: str, content: bytes, *, base_dir: Path, package_id: str |
| 2003 | ) -> int: |
| 2004 | """Add the seed rows whose id isn't already in ``collection``; leave every |
| 2005 | existing row untouched. Returns how many were added. Seed rows carry |
| 2006 | explicit ids and created_at, which ``create_collection_record`` honors |
| 2007 | (require_id + preserve_read_only), so a re-run adds nothing. One bad row is |
| 2008 | skipped, never fails the install.""" |
| 2009 | try: |
| 2010 | text = content.decode("utf-8") |
| 2011 | except UnicodeDecodeError: |
| 2012 | return 0 |
| 2013 | rows = list(csv.DictReader(io.StringIO(text), delimiter="\t")) |
| 2014 | if not rows: |
| 2015 | return 0 |
| 2016 | try: |
| 2017 | existing = { |
| 2018 | r.get("id") |
| 2019 | for r in object_records.read_collection_records(collection, base_dir=base_dir) |
| 2020 | } |
| 2021 | except (object_collections.CollectionNotFoundError, |
| 2022 | object_collections.InvalidCollectionNameError, OSError, ValueError): |
| 2023 | existing = set() |
| 2024 | added = 0 |
| 2025 | for row in rows: |
| 2026 | record_id = (row.get("id") or "").strip() |
| 2027 | if not record_id or record_id in existing: |
| 2028 | continue |
| 2029 | try: |
| 2030 | object_records.create_collection_record( |
| 2031 | collection, {k: v for k, v in row.items() if v is not None}, |
| 2032 | base_dir=base_dir, actor=f"package:{package_id}", preserve_read_only=True, |
| 2033 | ) |
| 2034 | existing.add(record_id) |
| 2035 | added += 1 |
| 2036 | except Exception: # noqa: BLE001 -- one malformed seed row never fails the install |
| 2037 | continue |
| 2038 | return added |
| 2039 | |
| 2040 | |
| 2041 | def _write_file_atomic_bytes(path: Path, content: bytes) -> None: |
| 2042 | path.parent.mkdir(parents=True, exist_ok=True) |
| 2043 | tmp_name = "" |
| 2044 | try: |
| 2045 | with tempfile.NamedTemporaryFile( |
| 2046 | "wb", |
| 2047 | dir=path.parent, |
| 2048 | prefix=f".{path.name}.", |
| 2049 | suffix=".tmp", |
| 2050 | delete=False, |
| 2051 | ) as tmp: |
| 2052 | tmp_name = tmp.name |
| 2053 | tmp.write(content) |
| 2054 | tmp.flush() |
| 2055 | os.fsync(tmp.fileno()) |
| 2056 | os.replace(tmp_name, path) |
| 2057 | finally: |
| 2058 | if tmp_name: |
| 2059 | try: |
| 2060 | Path(tmp_name).unlink() |
| 2061 | except FileNotFoundError: |
| 2062 | pass |
| 2063 | |
| 2064 | |
| 2065 | def _attribute_seed_records(collection: str, *, package_id: str, base_dir: Path) -> None: |
| 2066 | """Emit an attributed "create" change for every row a seed file just wrote. |
| 2067 | |
| 2068 | Seed writes land the whole records.tsv in one atomic byte-for-byte |
| 2069 | copy (see the seed_writes loop above), bypassing |
| 2070 | object_records.create_collection_record entirely -- so, unlike a |
| 2071 | normal record write, nothing emits a change automatically. This is |
| 2072 | the seed-install side of universal attribution: read the file back |
| 2073 | (install-once, so every row it now holds is a fresh create) and log |
| 2074 | one change per row, attributed to the installing package rather than |
| 2075 | left unattributed. |
| 2076 | """ |
| 2077 | try: |
| 2078 | rows = object_records.read_collection_records(collection, base_dir=base_dir) |
| 2079 | except (object_collections.CollectionNotFoundError, object_collections.InvalidCollectionNameError): |
| 2080 | return |
| 2081 | for row in rows: |
| 2082 | object_record_changes.append_record_change( |
| 2083 | collection=collection, |
| 2084 | record_id=row["id"], |
| 2085 | action="create", |
| 2086 | before=None, |
| 2087 | after=row, |
| 2088 | actor=f"package-install:{package_id}", |
| 2089 | base_dir=base_dir, |
| 2090 | ) |
| 2091 | |
| 2092 | |
| 2093 | def _relative_display_path(path: Path, root: Path) -> str: |
| 2094 | try: |
| 2095 | return path.resolve(strict=False).relative_to(root.resolve(strict=False)).as_posix() |
| 2096 | except ValueError: |
| 2097 | return path.name |
| 2098 | |
| 2099 | |
| 2100 | def _safe_relative_path(value: str, *, package_id: str, section: str) -> str: |
| 2101 | if "\x00" in value: |
| 2102 | raise InvalidPackageManifestError(f"Package {section} path contains a null byte: {package_id}") |
| 2103 | path = Path(value) |
| 2104 | if path.is_absolute() or not path.parts: |
| 2105 | raise InvalidPackageManifestError(f"Package {section} path must be relative: {package_id}") |
| 2106 | if any(part in {"", ".", ".."} for part in path.parts): |
| 2107 | raise InvalidPackageManifestError(f"Package {section} path is not safe: {package_id}") |
| 2108 | return path.as_posix() |
| 2109 | |
| 2110 | |
| 2111 | def _required_text(payload: Mapping[str, Any], key: str, *, package_id: str) -> str: |
| 2112 | value = payload.get(key) |
| 2113 | text = _optional_text(value) |
| 2114 | if text is None: |
| 2115 | raise InvalidPackageManifestError(f"Package manifest requires '{key}': {package_id}") |
| 2116 | return text |
| 2117 | |
| 2118 | |
| 2119 | def _optional_text(value: Any) -> str | None: |
| 2120 | if value is None: |
| 2121 | return None |
| 2122 | if not isinstance(value, str): |
| 2123 | return None |
| 2124 | text = value.strip() |
| 2125 | return text or None |
| 2126 | |
| 2127 | |
| 2128 | def _mapping_field(value: Any, *, package_id: str) -> dict[str, Any]: |
| 2129 | if value is None: |
| 2130 | return {} |
| 2131 | if not isinstance(value, Mapping): |
| 2132 | raise InvalidPackageManifestError(f"Package compatibility must be an object: {package_id}") |
| 2133 | return dict(value) |
| 2134 | |
| 2135 | |
| 2136 | def _list_field(value: Any, *, package_id: str, section: str) -> list[Any]: |
| 2137 | if value is None: |
| 2138 | return [] |
| 2139 | if not isinstance(value, list): |
| 2140 | raise InvalidPackageManifestError(f"Package {section} must be a list: {package_id}") |
| 2141 | return value |
| 2142 | |
| 2143 | |
| 2144 | def _entry_mapping(value: Any, *, package_id: str, section: str) -> Mapping[str, Any]: |
| 2145 | if not isinstance(value, Mapping): |
| 2146 | raise InvalidPackageManifestError(f"Package {section} entries must be objects: {package_id}") |
| 2147 | return value |