Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_packages.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_packages.py · 85.6 KB · 2147 lines
1"""DBBASIC package manifest discovery, dry-run planning, and gated installs.
2
3Packages are installable bundles of objects, schemas, permissions, seed data,
4and migrations. Installs are intentionally conservative: object and schema files
5can be created or replaced, seed files can be created, and permissions/migrations
6wait for explicit merge/run semantics.
7"""
8
9from __future__ import annotations
10
11import csv
12import io
13import json
14import os
15import re
16import tempfile
17from pathlib import Path
18from typing import Any, Callable, Iterable, Mapping
19
20import object_collections
21import object_namespace
22import object_package_baselines
23import object_permission_store
24import object_permissions
25import object_reconciles
26import object_record_changes
27import object_records
28import object_schemas
29import object_source
30import object_state
31from object_namespace import get_object_roots, object_id_from_path, resolve_object_id, validate_object_id
32from object_versions import DEFAULT_DATA_DIR
33
34PACKAGES_DIR = "packages"
35MANIFEST_FILE = "dbbasic-package.json"
36PACKAGE_MIGRATIONS_DIR = "package_migrations"
37
38_PACKAGE_ID_RE = re.compile(r"^[a-z][a-z0-9_-]{0,63}$")
39_VERSION_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$")
40_MIGRATION_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$")
41
42# A package's `schedules` become task_* rows in the scheduler trigger's
43# state, which is where object_daemon.process_scheduler reads them from.
44# Before this existed, every recurring pass on a running server had been
45# hand-entered into that state and appeared NOWHERE in the repository, so
46# rebuilding a box silently lost its daily work -- exactly what the
47# scheduler object's own docstring warns about. A schedule is part of what
48# an app IS, so it ships with the app.
49SCHEDULER_OBJECT_ID = "scheduler"
50_TASK_KEY_PREFIX = "task_"
51_SCHEDULE_ID_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$")
52_SCHEDULE_TYPES = ("cron", "onetime")
53_SCHEDULE_METHODS = ("POST", "GET", "PUT", "DELETE")
54# Fields the daemon stamps back onto a task as it runs. An install must
55# never touch them: they are the record of what actually happened, and a
56# package upgrade is not a reason to forget it (doctrine #8 -- independent
57# evidence is never edited).
58_SCHEDULE_RUNTIME_FIELDS = ("next_run", "last_run", "run_count")
59
60# A package's `nav` entries become rows in the `nav_entries` collection,
61# which is where every navigation surface reads its doors from.
62# Before this existed there were THREE hand-maintained lists of the same
63# apps -- a JS array in site_nav, a Python tuple in site_home, and a
64# collection->URL map for search hits -- and they had already drifted:
65# 25 entries, 21 entries, and no mention at all of shop, intake, billing,
66# timers, banking, shipping, receiving or payments. A door is part of what
67# an app IS, so it ships with the app, exactly like its schedule.
68NAV_COLLECTION = "nav_entries"
69_NAV_ID_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$")
70_NAV_SURFACES = ("public", "member", "operator", "hidden")
71_NAV_DEFAULT_GROUP = "Apps"
72_NAV_DEFAULT_ORDER = 100
73# What the package OWNS: reinstalling restates every one of these.
74_NAV_PACKAGE_FIELDS = ("label", "path", "blurb", "group", "surface", "order")
75# What the OPERATOR owns. The package says what an app is; the operator
76# says what they want to see. An install never touches this column, for
77# the same reason it never restarts a paused schedule (doctrine #8 --
78# somebody else's deliberate decision is not an install's to revert).
79_NAV_OPERATOR_FIELDS = ("operator_hidden",)
80
81# A package's `attention` entries become rows in the `attention_sources`
82# collection, which is where the daemon's rollup pass reads its list of
83# providers from and how a surface learns that a queue exists at all.
84# Third use of the pattern that fixed cron (`schedules`) and navigation
85# (`nav`), and for the same reason: this server is built out of gates and
86# derived states, every one of which produces a queue of things a machine
87# deliberately refused to decide -- scans `extracted`, time `submitted`,
88# invoices overdue, bank lines unmatched. It has been computing those
89# queues all along and throwing them away at the end of each pass;
90# `system_scan_processor` literally returns a field named
91# `needing_a_human` and nothing reads it. Only the package that owns a
92# domain can honestly say what "needs a human" means in it, so the
93# declaration ships with the app.
94ATTENTION_COLLECTION = "attention_sources"
95_ATTENTION_ID_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$")
96_ATTENTION_SEVERITIES = ("normal", "warning", "urgent")
97_ATTENTION_DEFAULT_GROUP = "Apps"
98_ATTENTION_DEFAULT_SEVERITY = "normal"
99# What the package OWNS: reinstalling restates every one of these.
100_ATTENTION_PACKAGE_FIELDS = ("object_id", "label", "path", "nav_id", "group", "severity")
101# What the OPERATOR owns. A deployment that does not care about a queue
102# silences it here rather than by editing somebody else's manifest, and an
103# install never writes this column -- same posture as nav's
104# `operator_hidden` and a paused schedule (doctrine #8).
105_ATTENTION_OPERATOR_FIELDS = ("operator_muted",)
106
107
108class InvalidPackageIdError(ValueError):
109 """Raised when a package id is not safe for routes or storage."""
110
111
112class PackageNotFoundError(LookupError):
113 """Raised when a package directory or manifest is missing."""
114
115
116class InvalidPackageManifestError(ValueError):
117 """Raised when a package manifest is invalid."""
118
119
120class PackageInstallError(RuntimeError):
121 """Raised when a package install would be unsafe or unsupported."""
122
123
124def validate_package_id(package_id: str) -> bool:
125 """Return True when a package id is route-safe."""
126 if not isinstance(package_id, str):
127 return False
128 return bool(_PACKAGE_ID_RE.fullmatch(package_id))
129
130
131def list_packages(
132 *,
133 root: Path | str = PACKAGES_DIR,
134) -> list[dict[str, Any]]:
135 """Return package summaries for all package manifests under root."""
136 packages_root = Path(root)
137 if not packages_root.exists() or not packages_root.is_dir():
138 return []
139
140 packages = []
141 for path in sorted(packages_root.iterdir(), key=lambda item: item.name):
142 if not path.is_dir() or not validate_package_id(path.name):
143 continue
144 manifest_path = path / MANIFEST_FILE
145 if manifest_path.is_file():
146 packages.append(_package_summary(_load_package(path.name, path)))
147 return packages
148
149
150def get_package(
151 package_id: str,
152 *,
153 root: Path | str = PACKAGES_DIR,
154) -> dict[str, Any]:
155 """Return one normalized package manifest."""
156 package_dir = _package_dir(package_id, root)
157 return _load_package(package_id, package_dir)
158
159
160DATA_DIR_ENV = "DBBASIC_DATA_DIR"
161
162
163def resolve_base_dir(base_dir: Path | str | None = None) -> Path:
164 """The data directory an install should write to.
165
166 An explicit `base_dir` always wins. Otherwise the ENVIRONMENT decides,
167 because `DBBASIC_DATA_DIR` is the one authoritative statement of where
168 this server's data lives -- it is what the server itself reads, what
169 every package object reads, and what the systemd unit sets.
170
171 This used to fall straight through to the literal "data", ignoring the
172 environment even when it was set to somewhere else entirely. The result
173 was the worst shape a bug can take: an install that reported
174 `"status": "written"` for every schema and seed, having written them
175 into a freshly created `./data` beside the checkout that nothing reads.
176 Nothing failed, nothing warned, and the package was simply absent from
177 the running server. Hit twice in one afternoon on a box whose layout was
178 well understood, which is the argument for fixing the default rather
179 than remembering harder.
180
181 Callers that genuinely want a relative "data" can still ask for it by
182 name; what they can no longer do is get it by accident.
183 """
184 if base_dir is not None:
185 return Path(base_dir)
186 return Path(os.environ.get(DATA_DIR_ENV) or DEFAULT_DATA_DIR)
187
188
189def dry_run_package(
190 package_id: str,
191 *,
192 root: Path | str = PACKAGES_DIR,
193 base_dir: Path | str | None = None,
194 object_roots: Iterable[Path] | None = None,
195) -> dict[str, Any]:
196 """Return a non-mutating package install plan."""
197 package_dir = _package_dir(package_id, root)
198 package = _load_package(package_id, package_dir)
199 base = resolve_base_dir(base_dir)
200
201 warnings: list[str] = []
202 objects = [
203 _object_change(entry, package_dir=package_dir, object_roots=object_roots, warnings=warnings)
204 for entry in package["objects"]
205 ]
206 schemas = [
207 _schema_change(entry, package_dir=package_dir, base_dir=base, warnings=warnings)
208 for entry in package["schemas"]
209 ]
210 permissions = [
211 _permission_change(
212 entry,
213 package_id=package_id,
214 package_dir=package_dir,
215 base_dir=base,
216 warnings=warnings,
217 )
218 for entry in package["permissions"]
219 ]
220 seed = [
221 _seed_change(entry, package_dir=package_dir, base_dir=base, warnings=warnings)
222 for entry in package["seed"]
223 ]
224 migrations = [
225 _migration_change(entry, package=package_id, package_dir=package_dir, base_dir=base, warnings=warnings)
226 for entry in package["migrations"]
227 ]
228 # A schedule may name an object this same install is about to write, so
229 # "does the target exist" is asked of the union of what is already
230 # resolvable and what this package provides -- not of the server as it
231 # stands right now.
232 existing_tasks = _load_scheduler_tasks(base)
233 installable_object_ids = {entry["id"] for entry in package["objects"]}
234 roots_for_lookup = list(object_roots) if object_roots is not None else get_object_roots()
235 for entry in package["schedules"] + package["attention"]:
236 if entry["object_id"] in installable_object_ids:
237 continue
238 if resolve_object_id(entry["object_id"], roots_for_lookup) is not None:
239 installable_object_ids.add(entry["object_id"])
240 schedules = [
241 _schedule_change(
242 entry,
243 existing=existing_tasks,
244 installable_object_ids=installable_object_ids,
245 warnings=warnings,
246 )
247 for entry in package["schedules"]
248 ]
249 # Nav entries are compared against what is already registered, which
250 # is how "two packages claim one door" becomes visible before either
251 # of them lands. A missing nav_entries collection reads as "nothing
252 # registered yet", so the plan is create-everything rather than an
253 # error -- app-nav is not a dependency of being installable.
254 registered_nav = _load_nav_entries(base, roots=object_roots) or {}
255 nav = [
256 _nav_change(entry, existing=registered_nav, package_id=package_id, warnings=warnings)
257 for entry in package["nav"]
258 ]
259 # Attention sources compare the same way nav entries do, and against
260 # the same missing-collection posture: app-nav absent reads as
261 # "nothing declared yet" rather than an error, because a package must
262 # not fail to install because the home screen is not installed.
263 registered_attention = _load_attention_sources(base, roots=object_roots) or {}
264 attention = [
265 _attention_change(
266 entry,
267 existing=registered_attention,
268 package_id=package_id,
269 installable_object_ids=installable_object_ids,
270 warnings=warnings,
271 )
272 for entry in package["attention"]
273 ]
274
275 return {
276 "package": _package_summary(package),
277 "mode": "dry_run",
278 "install_enabled": False,
279 "safe_to_install": not warnings,
280 # WHERE, resolved and absolute. Every entry below reports a
281 # `destination` relative to this, and a status of "written" means
282 # nothing at all without knowing which tree it was written into --
283 # an install can succeed completely into a directory the running
284 # server does not read.
285 "data_dir": str(base.resolve()),
286 "objects": objects,
287 "schemas": schemas,
288 "permissions": permissions,
289 "seed": seed,
290 "migrations": migrations,
291 "schedules": schedules,
292 "nav": nav,
293 "attention": attention,
294 "warnings": warnings,
295 }
296
297
298def install_package(
299 package_id: str,
300 *,
301 root: Path | str = PACKAGES_DIR,
302 base_dir: Path | str | None = None,
303 object_roots: Iterable[Path] | None = None,
304 allow_replace: bool = False,
305 force: bool = False,
306 before_write: Callable[[Mapping[str, Any]], Mapping[str, Any] | None] | None = None,
307) -> dict[str, Any]:
308 """Install a package using the conservative public write contract.
309
310 On an upgrade (an object/schema that already exists), writes are no
311 longer blind: each artifact is three-way compared against its recorded
312 baseline (see object_package_baselines) and its live content. Pristine
313 artifacts fast-forward, customized-but-unchanged artifacts are kept, and
314 genuine conflicts are parked as pending-reconcile records (never
315 overwritten) unless force=True. See docs/upgrade-and-customization.md
316 (Rule 1: Reconcile, Don't Replace).
317 """
318 # force implies allow_replace: forcing only makes sense when you may
319 # touch existing artifacts, so the replace-without-allow_replace blocker
320 # still applies unless force is also set.
321 allow_replace = allow_replace or force
322 roots = list(object_roots) if object_roots is not None else get_object_roots()
323 if not roots:
324 raise PackageInstallError("Package installs require at least one object root")
325
326 package_dir = _package_dir(package_id, root)
327 package = _load_package(package_id, package_dir)
328 base = resolve_base_dir(base_dir)
329 plan = dry_run_package(package_id, root=root, base_dir=base, object_roots=roots)
330
331 blockers = _install_blockers(plan, package=package, allow_replace=allow_replace)
332 if blockers:
333 raise PackageInstallError("; ".join(blockers))
334
335 object_writes = []
336 for entry, planned in zip(package["objects"], plan["objects"], strict=True):
337 source = _package_file(package_dir, entry["path"])
338 existing = resolve_object_id(entry["id"], roots)
339 # An object that already exists stays where it lives -- a customized
340 # object must not be yanked out of the override root by an upgrade.
341 #
342 # A NEW object goes to the LAST root, which is the base/system root:
343 # get_object_roots() returns [override, base] in LOOKUP order, so
344 # roots[0] is the override root, and installing there was silently
345 # wrong. object_namespace.get_base_object_roots says so directly --
346 # "the root packages install into and reconcile against ... the
347 # pristine, upgradeable copy" -- but this line used roots[0] and
348 # contradicted it. The consequence was not cosmetic: iter_object_sources
349 # labels anything under the override root kind="override", and
350 # object_handlers.build_index indexes ONLY kind=="system" (Decision 2,
351 # user-authored handlers deferred). So every package-installed object
352 # declaring HANDLES installed successfully, reported "written", and
353 # then never received a single event.
354 #
355 # With one root configured (no overrides) this is the same root it
356 # always was.
357 destination_root = (_root_for_path(existing, roots) if existing is not None
358 else roots[-1])
359 if destination_root is None:
360 raise PackageInstallError(f"Existing object is outside configured object roots: {entry['id']}")
361 destination = existing or _object_destination(entry, destination_root)
362 _ensure_inside(destination, destination_root, label="object")
363 try:
364 mapped_id = object_id_from_path(destination, destination_root)
365 except ValueError as exc:
366 raise PackageInstallError(
367 f"Package object path is not a valid object destination: {entry['path']}"
368 ) from exc
369 if mapped_id != entry["id"]:
370 raise PackageInstallError(
371 f"Package object path does not map to object id {entry['id']}: {entry['path']}"
372 )
373 object_writes.append((planned, destination, destination_root, source.read_bytes()))
374
375 schema_writes = []
376 for entry, planned in zip(package["schemas"], plan["schemas"], strict=True):
377 source = _package_file(package_dir, entry["path"])
378 try:
379 payload = json.loads(source.read_text(encoding="utf-8"))
380 except json.JSONDecodeError as exc:
381 raise PackageInstallError(f"Package schema contains invalid JSON: {entry['path']}") from exc
382 try:
383 normalized = object_schemas.normalize_schema(entry["collection"], payload, source="manual")
384 except ValueError as exc:
385 raise PackageInstallError(f"Package schema is invalid: {entry['path']}") from exc
386 schema_writes.append((entry, planned, normalized))
387
388 seed_writes = []
389 for entry, planned in zip(package["seed"], plan["seed"], strict=True):
390 source = _package_file(package_dir, entry["path"])
391 destination = base / "collections" / entry["collection"] / "records.tsv"
392 _ensure_inside(destination, base / "collections", label="seed")
393 seed_writes.append((entry, planned, destination, source.read_bytes()))
394
395 permission_writes = []
396 for entry, planned in zip(package["permissions"], plan["permissions"], strict=True):
397 rules = _load_permission_rules(package_dir, entry, package_id=package_id)
398 permission_writes.append((planned, rules))
399
400 restore_point = before_write(plan) if before_write is not None else None
401
402 existing_baseline = object_package_baselines.load_baseline(package_id, base_dir=base) or {}
403 base_objects = existing_baseline.get("objects") or {}
404 base_schemas = existing_baseline.get("schemas") or {}
405 base_schema_bodies = existing_baseline.get("schema_bodies") or {}
406 baseline_version = existing_baseline.get("version")
407
408 reconciles: list[str] = []
409 new_baseline_objects: dict[str, str] = {}
410 new_baseline_schemas: dict[str, str] = {}
411 new_baseline_schema_bodies: dict[str, Any] = {}
412
413 installed_objects = []
414 for planned, destination, destination_root, content in object_writes:
415 object_id = planned["id"]
416 extra: dict[str, Any] = {}
417
418 if planned["action"] == "create":
419 _write_file_atomic_bytes(destination, content)
420 status = "written"
421 new_baseline_objects[object_id] = object_package_baselines.sha256_text(content.decode("utf-8"))
422 else:
423 new_text = content.decode("utf-8")
424 new_sha = object_package_baselines.sha256_text(new_text)
425 live_text = object_source.get_object_source(object_id, roots)
426 live_sha = object_package_baselines.sha256_text(live_text)
427 old_sha = base_objects.get(object_id)
428
429 if live_sha == new_sha:
430 status = "unchanged"
431 new_baseline_objects[object_id] = new_sha
432 elif old_sha is not None and live_sha == old_sha:
433 _write_file_atomic_bytes(destination, content)
434 status = "updated"
435 new_baseline_objects[object_id] = new_sha
436 elif old_sha is not None and new_sha == old_sha:
437 status = "kept"
438 new_baseline_objects[object_id] = old_sha
439 elif force:
440 _write_file_atomic_bytes(destination, content)
441 status = "forced"
442 new_baseline_objects[object_id] = new_sha
443 else:
444 rec = object_reconciles.create_reconcile(
445 package=package_id,
446 target_version=package["version"],
447 baseline_version=baseline_version,
448 artifact={"kind": "object", "id": object_id},
449 mine=live_text,
450 theirs=new_text,
451 base_sha=old_sha,
452 base_dir=base,
453 )
454 reconciles.append(rec["id"])
455 status = "conflict"
456 extra["reconcile_id"] = rec["id"]
457 new_baseline_objects[object_id] = old_sha if old_sha is not None else live_sha
458
459 installed_objects.append(
460 {
461 **planned,
462 "status": status,
463 "destination": _relative_display_path(destination, destination_root),
464 **extra,
465 }
466 )
467
468 installed_schemas = []
469 for entry, planned, normalized in schema_writes:
470 collection = entry["collection"]
471 new_sha = object_package_baselines.canonical_schema_hash(normalized)
472 extra: dict[str, Any] = {}
473
474 live = None
475 if planned["action"] == "replace":
476 try:
477 live = object_schemas.get_schema(collection, base_dir=base, roots=roots)
478 except object_schemas.SchemaNotFoundError:
479 live = None
480
481 if live is None:
482 object_schemas.replace_schema(collection, normalized, base_dir=base)
483 status = "written"
484 new_baseline_schemas[collection] = new_sha
485 new_baseline_schema_bodies[collection] = normalized
486 else:
487 live_sha = object_package_baselines.canonical_schema_hash(live)
488 old_sha = base_schemas.get(collection)
489 base_body = base_schema_bodies.get(collection)
490
491 if live_sha == new_sha:
492 status = "unchanged"
493 new_baseline_schemas[collection] = new_sha
494 new_baseline_schema_bodies[collection] = normalized
495 elif old_sha is not None and live_sha == old_sha:
496 object_schemas.replace_schema(collection, normalized, base_dir=base)
497 status = "updated"
498 new_baseline_schemas[collection] = new_sha
499 new_baseline_schema_bodies[collection] = normalized
500 elif old_sha is not None and new_sha == old_sha:
501 status = "kept"
502 new_baseline_schemas[collection] = old_sha
503 new_baseline_schema_bodies[collection] = normalized
504 elif force:
505 object_schemas.replace_schema(collection, normalized, base_dir=base)
506 status = "forced"
507 new_baseline_schemas[collection] = new_sha
508 new_baseline_schema_bodies[collection] = normalized
509 elif base_body is not None:
510 # Customized AND shipped changed: try a field-union merge
511 # before parking a conflict (Rule 3: schemas are additive,
512 # so a same-named field changed on both sides is the only
513 # real conflict -- see docs/upgrade-and-customization.md).
514 merged, collisions = object_schemas.merge_schema_fields(base_body, live, normalized)
515 if not collisions:
516 object_schemas.replace_schema(collection, merged, base_dir=base)
517 status = "merged"
518 new_baseline_schemas[collection] = new_sha
519 new_baseline_schema_bodies[collection] = normalized
520 else:
521 rec = object_reconciles.create_reconcile(
522 package=package_id,
523 target_version=package["version"],
524 baseline_version=baseline_version,
525 artifact={"kind": "schema", "collection": collection},
526 mine=json.dumps(live, indent=2, sort_keys=True),
527 theirs=json.dumps(normalized, indent=2, sort_keys=True),
528 base_sha=old_sha,
529 base_dir=base,
530 collisions=collisions,
531 )
532 reconciles.append(rec["id"])
533 status = "conflict"
534 extra["reconcile_id"] = rec["id"]
535 extra["collisions"] = collisions
536 new_baseline_schemas[collection] = old_sha if old_sha is not None else live_sha
537 new_baseline_schema_bodies[collection] = base_body
538 else:
539 # No base schema body on record (baseline predates this
540 # feature, or was never stamped): fall back to parking a
541 # conflict, unchanged from prior behavior.
542 rec = object_reconciles.create_reconcile(
543 package=package_id,
544 target_version=package["version"],
545 baseline_version=baseline_version,
546 artifact={"kind": "schema", "collection": collection},
547 mine=json.dumps(live, indent=2, sort_keys=True),
548 theirs=json.dumps(normalized, indent=2, sort_keys=True),
549 base_sha=old_sha,
550 base_dir=base,
551 )
552 reconciles.append(rec["id"])
553 status = "conflict"
554 extra["reconcile_id"] = rec["id"]
555 new_baseline_schemas[collection] = old_sha if old_sha is not None else live_sha
556 if base_body is not None:
557 new_baseline_schema_bodies[collection] = base_body
558
559 installed_schemas.append(
560 {
561 **planned,
562 "status": status,
563 "destination": f"schemas/{collection}.json",
564 **extra,
565 }
566 )
567
568 installed_seed = []
569 for entry, planned, destination, content in seed_writes:
570 # Seed into a collection that already holds records: MERGE by id rather
571 # than skip. Add only the seed rows whose id isn't already present;
572 # existing rows (live data, possibly customized) are never touched. This
573 # is what lets several packages each seed a SHARED collection
574 # (views/site_routes) -- previously only the first-installed package's
575 # seed landed and every later one was skipped (needing manual inserts).
576 # A fresh collection (no records yet) still takes the whole seed file.
577 if planned.get("installed"):
578 added = _merge_seed_rows(
579 entry["collection"], content, base_dir=base, package_id=package_id
580 )
581 installed_seed.append(
582 {
583 **planned,
584 "status": "merged" if added else "skipped",
585 "reason": (
586 f"merged {added} new row(s); existing preserved"
587 if added else "collection already has these rows; preserved"
588 ),
589 "added": added,
590 "destination": f"collections/{entry['collection']}/records.tsv",
591 }
592 )
593 continue
594 _write_file_atomic_bytes(destination, content)
595 _attribute_seed_records(entry["collection"], package_id=package_id, base_dir=base)
596 installed_seed.append(
597 {
598 **planned,
599 "status": "written",
600 "destination": f"collections/{entry['collection']}/records.tsv",
601 }
602 )
603
604 installed_permissions = []
605 for planned, rules in permission_writes:
606 total, added = _merge_permission_rules(rules, base_dir=base)
607 installed_permissions.append(
608 {
609 **planned,
610 "status": "merged",
611 "rules": total,
612 "new_rules": added,
613 }
614 )
615
616 # Schedules land last: a task board that points at an object is only
617 # honest once that object is on disk.
618 installed_schedules = _apply_schedules(
619 package["schedules"], plan["schedules"], base_dir=base
620 )
621
622 # Nav lands after the seeds, because a package that ships the
623 # nav_entries schema itself (app-nav) has to have written it before
624 # its own doors can be registered.
625 installed_nav = _apply_nav(
626 package["nav"], plan["nav"], package_id=package_id, base_dir=base, roots=roots
627 )
628
629 # Attention sources land last of all, for the same reason schedules
630 # do: a counter that names an object is only honest once that object
631 # is on disk, and the row is what the daemon will start executing.
632 installed_attention = _apply_attention(
633 package["attention"], plan["attention"], package_id=package_id,
634 base_dir=base, roots=roots
635 )
636
637 object_package_baselines.record_baseline(
638 package_id,
639 version=package["version"],
640 objects=new_baseline_objects,
641 schemas=new_baseline_schemas,
642 schema_bodies=new_baseline_schema_bodies,
643 base_dir=base,
644 )
645
646 result = {
647 "package": plan["package"],
648 "mode": "install",
649 "install_enabled": True,
650 "allow_replace": allow_replace,
651 "safe_to_install": True,
652 # The resolved data directory everything below landed in. Reported
653 # because "written" is not a claim anybody can check without it --
654 # see resolve_base_dir for the afternoon that made this necessary.
655 "data_dir": plan["data_dir"],
656 "objects": installed_objects,
657 "schemas": installed_schemas,
658 "permissions": installed_permissions,
659 "seed": installed_seed,
660 "migrations": plan["migrations"],
661 "schedules": installed_schedules,
662 "nav": installed_nav,
663 "attention": installed_attention,
664 "reconciles": reconciles,
665 "warnings": [],
666 }
667 if restore_point is not None:
668 result["restore_point"] = dict(restore_point)
669 return result
670
671
672def package_status(
673 package_id: str,
674 *,
675 root: Path | str = PACKAGES_DIR,
676 base_dir: Path | str = DEFAULT_DATA_DIR,
677 object_roots: Iterable[Path] | None = None,
678) -> dict[str, Any]:
679 """Return install/customization status for a package's baselined artifacts."""
680 roots = list(object_roots) if object_roots is not None else get_object_roots()
681 base = Path(base_dir)
682
683 package = _load_package(package_id, _package_dir(package_id, root))
684 summary = _package_summary(package)
685 summary["pending_reconciles"] = object_reconciles.count_pending(base_dir=base, package=package_id)
686
687 baseline = object_package_baselines.load_baseline(package_id, base_dir=base)
688 if baseline is None:
689 summary["installed"] = False
690 summary["customized"] = False
691 summary["artifacts"] = []
692 return summary
693
694 artifacts: list[dict[str, Any]] = []
695 any_customized = False
696
697 for object_id, base_sha in (baseline.get("objects") or {}).items():
698 try:
699 live = object_source.get_object_source(object_id, roots)
700 except (FileNotFoundError, LookupError, OSError, ValueError, object_source.ObjectSourceError):
701 state = "removed"
702 else:
703 state = "pristine" if object_package_baselines.sha256_text(live) == base_sha else "customized"
704 if state == "customized":
705 any_customized = True
706 artifacts.append(
707 {
708 "kind": "object",
709 "id": object_id,
710 "state": state,
711 "overridden": object_namespace.has_override(object_id),
712 }
713 )
714
715 for collection, base_sha in (baseline.get("schemas") or {}).items():
716 try:
717 live = object_schemas.get_schema(collection, base_dir=base, roots=roots)
718 except (object_schemas.SchemaNotFoundError, LookupError, OSError, ValueError):
719 state = "removed"
720 else:
721 state = (
722 "pristine"
723 if object_package_baselines.canonical_schema_hash(live) == base_sha
724 else "customized"
725 )
726 if state == "customized":
727 any_customized = True
728 artifacts.append({"kind": "schema", "collection": collection, "state": state})
729
730 summary["installed"] = True
731 summary["installed_version"] = baseline.get("version")
732 summary["customized"] = any_customized
733 summary["artifacts"] = artifacts
734 return summary
735
736
737def _package_dir(package_id: str, root: Path | str) -> Path:
738 if not validate_package_id(package_id):
739 raise InvalidPackageIdError(f"Invalid package id: {package_id}")
740
741 package_dir = Path(root) / package_id
742 manifest_path = package_dir / MANIFEST_FILE
743 if not package_dir.is_dir() or not manifest_path.is_file():
744 raise PackageNotFoundError(f"Package not found: {package_id}")
745 return package_dir
746
747
748def _load_package(package_id: str, package_dir: Path) -> dict[str, Any]:
749 manifest_path = package_dir / MANIFEST_FILE
750 try:
751 payload = json.loads(manifest_path.read_text())
752 except json.JSONDecodeError as exc:
753 raise InvalidPackageManifestError(f"Package manifest contains invalid JSON: {package_id}") from exc
754
755 return _normalize_manifest(package_id, payload)
756
757
758def _normalize_manifest(package_id: str, payload: Any) -> dict[str, Any]:
759 if not isinstance(payload, Mapping):
760 raise InvalidPackageManifestError(f"Package manifest must contain an object: {package_id}")
761
762 manifest_id = _required_text(payload, "id", package_id=package_id)
763 if manifest_id != package_id:
764 raise InvalidPackageManifestError(
765 f"Package manifest id does not match directory: {package_id}"
766 )
767 if not validate_package_id(manifest_id):
768 raise InvalidPackageIdError(f"Invalid package id: {manifest_id}")
769
770 version = _required_text(payload, "version", package_id=package_id)
771 if not _VERSION_RE.fullmatch(version):
772 raise InvalidPackageManifestError(f"Invalid package version: {package_id}")
773
774 return {
775 "id": manifest_id,
776 "name": _required_text(payload, "name", package_id=package_id),
777 "version": version,
778 "description": _optional_text(payload.get("description")),
779 "compatibility": _mapping_field(payload.get("compatibility"), package_id=package_id),
780 "dependencies": _normalize_dependencies(payload.get("dependencies", []), package_id=package_id),
781 "objects": _normalize_objects(payload.get("objects", []), package_id=package_id),
782 "schemas": _normalize_collection_paths(
783 payload.get("schemas", []),
784 package_id=package_id,
785 collection_key="collection",
786 section="schemas",
787 ),
788 "permissions": _normalize_path_entries(
789 payload.get("permissions", []),
790 package_id=package_id,
791 section="permissions",
792 ),
793 "seed": _normalize_collection_paths(
794 payload.get("seed", []),
795 package_id=package_id,
796 collection_key="collection",
797 section="seed",
798 ),
799 "migrations": _normalize_migrations(payload.get("migrations", []), package_id=package_id),
800 "connectors": _normalize_connectors(payload.get("connectors", []), package_id=package_id),
801 "schedules": _normalize_schedules(payload.get("schedules", []), package_id=package_id),
802 "nav": _normalize_nav(payload.get("nav", []), package_id=package_id),
803 "attention": _normalize_attention(payload.get("attention", []), package_id=package_id),
804 # The documented opt-out for a package that ships a site_* object
805 # which is not a door: a JS widget (site_thread), a fragment
806 # (site_materialize_run_button), or a per-record document reached
807 # from the record itself (site_packing_slip). Saying so in the
808 # manifest is the point -- "this app has no front page" becomes a
809 # reviewable claim rather than an omission nobody notices.
810 "nav_optional": bool(payload.get("nav_optional", False)),
811 }
812
813
814def _package_summary(package: Mapping[str, Any]) -> dict[str, Any]:
815 return {
816 "id": package["id"],
817 "name": package["name"],
818 "version": package["version"],
819 "description": package.get("description"),
820 "status": "available",
821 "object_count": len(package["objects"]),
822 "schema_count": len(package["schemas"]),
823 "permission_count": len(package["permissions"]),
824 "seed_count": len(package["seed"]),
825 "migration_count": len(package["migrations"]),
826 "dependency_count": len(package["dependencies"]),
827 "connector_count": len(package.get("connectors", [])),
828 "schedule_count": len(package.get("schedules", [])),
829 "nav_count": len(package.get("nav", [])),
830 "attention_count": len(package.get("attention", [])),
831 }
832
833
834def _normalize_objects(payload: Any, *, package_id: str) -> list[dict[str, str]]:
835 entries = _list_field(payload, package_id=package_id, section="objects")
836 normalized = []
837 for entry in entries:
838 mapping = _entry_mapping(entry, package_id=package_id, section="objects")
839 object_id = _required_text(mapping, "id", package_id=package_id)
840 if not validate_object_id(object_id):
841 raise InvalidPackageManifestError(f"Invalid package object id: {object_id}")
842 normalized.append(
843 {
844 "id": object_id,
845 "path": _safe_relative_path(
846 _required_text(mapping, "path", package_id=package_id),
847 package_id=package_id,
848 section="objects",
849 ),
850 }
851 )
852 return normalized
853
854
855def _normalize_collection_paths(
856 payload: Any,
857 *,
858 package_id: str,
859 collection_key: str,
860 section: str,
861) -> list[dict[str, str]]:
862 entries = _list_field(payload, package_id=package_id, section=section)
863 normalized = []
864 for entry in entries:
865 mapping = _entry_mapping(entry, package_id=package_id, section=section)
866 collection = _required_text(mapping, collection_key, package_id=package_id)
867 if not object_collections.validate_collection_name(collection):
868 raise InvalidPackageManifestError(f"Invalid package collection: {collection}")
869 normalized.append(
870 {
871 collection_key: collection,
872 "path": _safe_relative_path(
873 _required_text(mapping, "path", package_id=package_id),
874 package_id=package_id,
875 section=section,
876 ),
877 }
878 )
879 return normalized
880
881
882def _normalize_path_entries(
883 payload: Any,
884 *,
885 package_id: str,
886 section: str,
887) -> list[dict[str, str]]:
888 entries = _list_field(payload, package_id=package_id, section=section)
889 normalized = []
890 for entry in entries:
891 mapping = _entry_mapping(entry, package_id=package_id, section=section)
892 normalized.append(
893 {
894 "path": _safe_relative_path(
895 _required_text(mapping, "path", package_id=package_id),
896 package_id=package_id,
897 section=section,
898 )
899 }
900 )
901 return normalized
902
903
904def _normalize_migrations(payload: Any, *, package_id: str) -> list[dict[str, str]]:
905 entries = _list_field(payload, package_id=package_id, section="migrations")
906 normalized = []
907 for entry in entries:
908 mapping = _entry_mapping(entry, package_id=package_id, section="migrations")
909 migration_id = _required_text(mapping, "id", package_id=package_id)
910 if not _MIGRATION_ID_RE.fullmatch(migration_id):
911 raise InvalidPackageManifestError(f"Invalid package migration id: {migration_id}")
912 normalized.append(
913 {
914 "id": migration_id,
915 "path": _safe_relative_path(
916 _required_text(mapping, "path", package_id=package_id),
917 package_id=package_id,
918 section="migrations",
919 ),
920 }
921 )
922 return normalized
923
924
925def _normalize_connectors(payload: Any, *, package_id: str) -> list[dict[str, str]]:
926 """A `connectors` entry declares that a collection is reconciled against an
927 external system by a connector module inside this package (see
928 object_connectors / plan/vocabulary/03-external-connectors-spec.md):
929 `{collection, module, entry?}`. `module` is validated as a package-relative
930 path (escape-checked again at load time); `entry` defaults to `reconcile`."""
931 entries = _list_field(payload, package_id=package_id, section="connectors")
932 normalized = []
933 for entry in entries:
934 mapping = _entry_mapping(entry, package_id=package_id, section="connectors")
935 collection = _required_text(mapping, "collection", package_id=package_id)
936 if not object_collections.validate_collection_name(collection):
937 raise InvalidPackageManifestError(f"Invalid package connector collection: {collection}")
938 module = _safe_relative_path(
939 _required_text(mapping, "module", package_id=package_id),
940 package_id=package_id,
941 section="connectors",
942 )
943 entry_name = _optional_text(mapping.get("entry")) or "reconcile"
944 if not entry_name.isidentifier():
945 raise InvalidPackageManifestError(f"Invalid connector entry: {entry_name}")
946 normalized.append({"collection": collection, "module": module, "entry": entry_name})
947 return normalized
948
949
950def _normalize_schedules(payload: Any, *, package_id: str) -> list[dict[str, Any]]:
951 """A `schedules` entry declares a recurring pass the app needs to work:
952 `{id, object_id, schedule, type?, method?, payload?, description?}`.
953
954 Time-driven work belongs to the daemon (docs/logic-decisions.md #2),
955 and until now the declaration of WHICH work lived only in a running
956 server's state. That made a schedule invisible to review, absent from
957 a fresh install, and lost on a rebuild. Declaring it here makes the
958 recurring pass part of the package, like the object it calls.
959
960 The id becomes a `task_<id>` state key, so it is restricted to the
961 characters a key can safely hold; every other field is validated here
962 rather than at 3am inside the daemon, where a bad cron string is a
963 pass that quietly does nothing.
964 """
965 entries = _list_field(payload, package_id=package_id, section="schedules")
966 normalized: list[dict[str, Any]] = []
967 seen: set[str] = set()
968 for entry in entries:
969 mapping = _entry_mapping(entry, package_id=package_id, section="schedules")
970 schedule_id = _required_text(mapping, "id", package_id=package_id)
971 if not _SCHEDULE_ID_RE.fullmatch(schedule_id):
972 raise InvalidPackageManifestError(f"Invalid package schedule id: {schedule_id}")
973 if schedule_id in seen:
974 raise InvalidPackageManifestError(
975 f"Duplicate package schedule id: {schedule_id}"
976 )
977 seen.add(schedule_id)
978
979 object_id = _required_text(mapping, "object_id", package_id=package_id)
980 if not validate_object_id(object_id):
981 raise InvalidPackageManifestError(
982 f"Invalid package schedule object_id: {object_id}"
983 )
984
985 schedule_type = (_optional_text(mapping.get("type")) or "cron").lower()
986 if schedule_type not in _SCHEDULE_TYPES:
987 raise InvalidPackageManifestError(
988 f"Package schedule type must be cron|onetime: {schedule_id}"
989 )
990
991 expression = _required_text(mapping, "schedule", package_id=package_id)
992 _validate_schedule_expression(expression, schedule_type, schedule_id=schedule_id)
993
994 method = (_optional_text(mapping.get("method")) or "POST").upper()
995 if method not in _SCHEDULE_METHODS:
996 raise InvalidPackageManifestError(
997 f"Invalid package schedule method: {schedule_id}"
998 )
999
1000 task_payload = mapping.get("payload", {})
1001 if task_payload in (None, ""):
1002 task_payload = {}
1003 if not isinstance(task_payload, Mapping):
1004 raise InvalidPackageManifestError(
1005 f"Package schedule payload must be an object: {schedule_id}"
1006 )
1007
1008 normalized.append({
1009 "id": schedule_id,
1010 "object_id": object_id,
1011 "method": method,
1012 "payload": dict(task_payload),
1013 "schedule": expression,
1014 "type": schedule_type,
1015 "description": _optional_text(mapping.get("description")),
1016 })
1017 return normalized
1018
1019
1020def _validate_schedule_expression(
1021 expression: str,
1022 schedule_type: str,
1023 *,
1024 schedule_id: str,
1025) -> None:
1026 """Reject a schedule that could never fire, at install time.
1027
1028 The daemon treats an unparseable expression as "no next run" and moves
1029 on in silence, so a typo here is a pass that never runs and never
1030 complains -- the failure this whole feature exists to end. Cron is
1031 checked with croniter where it is installed, and structurally where it
1032 is not, so validation degrades rather than disappearing.
1033 """
1034 if "\t" in expression or "\n" in expression:
1035 raise InvalidPackageManifestError(
1036 f"Package schedule expression may not contain tabs or newlines: {schedule_id}"
1037 )
1038 if schedule_type == "onetime":
1039 from datetime import datetime
1040
1041 try:
1042 datetime.fromisoformat(expression.replace("Z", "+00:00"))
1043 except ValueError as exc:
1044 raise InvalidPackageManifestError(
1045 f"Package onetime schedule must be an ISO timestamp: {schedule_id}"
1046 ) from exc
1047 return
1048
1049 try:
1050 from croniter import croniter
1051 except ImportError:
1052 croniter = None
1053 if croniter is not None:
1054 if not croniter.is_valid(expression):
1055 raise InvalidPackageManifestError(
1056 f"Invalid package cron expression: {schedule_id}"
1057 )
1058 return
1059 if len(expression.split()) not in (5, 6):
1060 raise InvalidPackageManifestError(
1061 f"Package cron expression must have 5 or 6 fields: {schedule_id}"
1062 )
1063
1064
1065def _load_scheduler_tasks(base_dir: Path) -> dict[str, dict[str, Any]]:
1066 """Every task currently on the daemon's board, keyed by schedule id.
1067
1068 Unreadable rows are skipped rather than raising: one hand-edited task
1069 must not make a package uninstallable.
1070 """
1071 try:
1072 state = object_state.get_object_state(SCHEDULER_OBJECT_ID, base_dir=base_dir)
1073 except Exception:
1074 return {}
1075 tasks: dict[str, dict[str, Any]] = {}
1076 for key, value in state.items():
1077 if not str(key).startswith(_TASK_KEY_PREFIX):
1078 continue
1079 try:
1080 task = json.loads(value)
1081 except (TypeError, ValueError):
1082 continue
1083 if isinstance(task, Mapping):
1084 tasks[str(key)[len(_TASK_KEY_PREFIX):]] = dict(task)
1085 return tasks
1086
1087
1088def _schedule_change(
1089 entry: Mapping[str, Any],
1090 *,
1091 existing: Mapping[str, dict[str, Any]],
1092 installable_object_ids: set[str],
1093 warnings: list[str],
1094) -> dict[str, Any]:
1095 live = existing.get(entry["id"])
1096 if live is None:
1097 action = "create"
1098 elif all(live.get(field) == entry[field]
1099 for field in ("object_id", "method", "payload", "schedule", "type")) \
1100 and (live.get("description") or None) == entry["description"]:
1101 # The description is compared too, so a plan that says "unchanged"
1102 # is one where nothing at all is about to be written -- a dry run
1103 # that under-reports is worse than no dry run.
1104 action = "unchanged"
1105 else:
1106 action = "update"
1107
1108 if entry["object_id"] not in installable_object_ids:
1109 # A schedule aimed at an object that will not exist is the silent
1110 # failure this feature exists to end, so it blocks the install
1111 # rather than landing as a task that can never run.
1112 warnings.append(
1113 f"Schedule {entry['id']} targets an object this install will not "
1114 f"provide: {entry['object_id']}"
1115 )
1116
1117 change = {
1118 "id": entry["id"],
1119 "object_id": entry["object_id"],
1120 "schedule": entry["schedule"],
1121 "type": entry["type"],
1122 "exists": live is not None,
1123 "action": action,
1124 }
1125 if live is not None:
1126 change["status"] = live.get("status", "active")
1127 change["run_count"] = live.get("run_count", 0)
1128 return change
1129
1130
1131def _apply_schedules(
1132 schedules: Iterable[Mapping[str, Any]],
1133 planned: Iterable[Mapping[str, Any]],
1134 *,
1135 base_dir: Path,
1136) -> list[dict[str, Any]]:
1137 """Write the package's schedules onto the daemon's task board.
1138
1139 Two things are deliberately preserved across an install:
1140
1141 **Run history** -- last_run/run_count/next_run belong to the daemon,
1142 not the package. Resetting them on an upgrade would make an operator's
1143 "when did this last work?" unanswerable.
1144
1145 **A pause** -- if someone paused a task, it stays paused. The package
1146 declares what SHOULD run; an operator decides what DOES right now, and
1147 a reinstall that silently restarts a deliberately-stopped nightly pass
1148 is how an upgrade becomes an incident.
1149
1150 A changed expression clears next_run so the daemon recomputes it from
1151 the new one; an unchanged expression keeps the pending firing exactly
1152 where it was, so reinstalling an app does not skip tonight's run.
1153 """
1154 entries = list(schedules)
1155 plans = list(planned)
1156 if not entries:
1157 return []
1158
1159 manager = object_state.ObjectStateManager(SCHEDULER_OBJECT_ID, base_dir=base_dir)
1160 applied = []
1161 for entry, plan in zip(entries, plans, strict=True):
1162 key = f"{_TASK_KEY_PREFIX}{entry['id']}"
1163 raw = manager.get(key)
1164 live: dict[str, Any] = {}
1165 if raw is not None:
1166 try:
1167 parsed = json.loads(raw)
1168 except (TypeError, ValueError):
1169 parsed = None
1170 if isinstance(parsed, Mapping):
1171 live = dict(parsed)
1172
1173 task = {
1174 "id": entry["id"],
1175 "object_id": entry["object_id"],
1176 "method": entry["method"],
1177 "payload": entry["payload"],
1178 "schedule": entry["schedule"],
1179 "type": entry["type"],
1180 "status": live.get("status", "active"),
1181 }
1182 if entry.get("description"):
1183 task["description"] = entry["description"]
1184 for field in _SCHEDULE_RUNTIME_FIELDS:
1185 if field in live:
1186 task[field] = live[field]
1187 if live and live.get("schedule") != entry["schedule"]:
1188 task["next_run"] = None
1189
1190 if live and json.dumps(live, sort_keys=True) == json.dumps(task, sort_keys=True):
1191 status = "unchanged"
1192 else:
1193 manager.set(key, json.dumps(task))
1194 status = "updated" if live else "written"
1195
1196 applied.append({**plan, "status": status, "task_status": task["status"],
1197 "destination": f"state/{SCHEDULER_OBJECT_ID}/{key}"})
1198 return applied
1199
1200
1201def _normalize_nav(payload: Any, *, package_id: str) -> list[dict[str, Any]]:
1202 """A `nav` entry declares a door the app puts on the site:
1203 `{id, label, path, blurb?, surface?, group?, order?}`.
1204
1205 Navigation used to be a hand-maintained list, and there were three of
1206 them -- the app switcher's JS array, the home page's tile list, and
1207 the search result URL map -- each edited by whoever remembered. They
1208 disagreed by four entries and between them knew about none of the
1209 eight newest apps, so the front door of the server advertised a
1210 server that no longer existed. A menu built by hand rots the moment
1211 somebody ships without editing it; a menu FOLDED over what every
1212 installed package declares cannot.
1213
1214 The id becomes the `nav_entries` record id, so it is restricted to
1215 the characters a record id can safely hold. `path` is validated for
1216 shape only -- whether anything answers it is a routing question this
1217 module deliberately does not pretend to answer (see `_nav_change`).
1218 """
1219 entries = _list_field(payload, package_id=package_id, section="nav")
1220 normalized: list[dict[str, Any]] = []
1221 seen: set[str] = set()
1222 for entry in entries:
1223 mapping = _entry_mapping(entry, package_id=package_id, section="nav")
1224 nav_id = _required_text(mapping, "id", package_id=package_id)
1225 if not _NAV_ID_RE.fullmatch(nav_id):
1226 raise InvalidPackageManifestError(f"Invalid package nav id: {nav_id}")
1227 if nav_id in seen:
1228 raise InvalidPackageManifestError(f"Duplicate package nav id: {nav_id}")
1229 seen.add(nav_id)
1230
1231 label = _required_text(mapping, "label", package_id=package_id)
1232 path = _required_text(mapping, "path", package_id=package_id)
1233 if not path.startswith("/"):
1234 raise InvalidPackageManifestError(
1235 f"Package nav path must start with '/': {nav_id}"
1236 )
1237 if any(character.isspace() for character in path):
1238 # A path with a space or a newline in it is a link that never
1239 # resolves and a TSV cell that can break the row it lives in.
1240 raise InvalidPackageManifestError(
1241 f"Package nav path may not contain whitespace: {nav_id}"
1242 )
1243
1244 surface = (_optional_text(mapping.get("surface")) or "member").lower()
1245 if surface not in _NAV_SURFACES:
1246 raise InvalidPackageManifestError(
1247 f"Package nav surface must be public|member|operator|hidden: {nav_id}"
1248 )
1249
1250 order = mapping.get("order", _NAV_DEFAULT_ORDER)
1251 if order in (None, ""):
1252 order = _NAV_DEFAULT_ORDER
1253 if isinstance(order, bool) or not isinstance(order, int):
1254 raise InvalidPackageManifestError(
1255 f"Package nav order must be an integer: {nav_id}"
1256 )
1257
1258 normalized.append({
1259 "id": nav_id,
1260 "label": label,
1261 "path": path,
1262 "blurb": _optional_text(mapping.get("blurb")) or "",
1263 "group": _optional_text(mapping.get("group")) or _NAV_DEFAULT_GROUP,
1264 "surface": surface,
1265 "order": order,
1266 })
1267 return normalized
1268
1269
1270def _load_nav_entries(
1271 base_dir: Path,
1272 *,
1273 roots: Iterable[Path] | None = None,
1274) -> dict[str, dict[str, str]] | None:
1275 """Every nav entry currently registered, keyed by record id.
1276
1277 ``None`` means the `nav_entries` collection is not installed at all
1278 (app-nav absent). That is not an error: a package must never fail to
1279 install because the navigation app is missing, any more than an app
1280 should fail because nobody has opened the menu yet.
1281 """
1282 try:
1283 rows = object_records.read_collection_records(
1284 NAV_COLLECTION, base_dir=base_dir, roots=roots)
1285 except (object_collections.CollectionNotFoundError,
1286 object_collections.InvalidCollectionNameError):
1287 return None
1288 except (OSError, ValueError):
1289 # A hand-edited row must not make a package uninstallable, the
1290 # same posture _load_scheduler_tasks takes.
1291 return {}
1292 return {str(row.get("id") or ""): dict(row) for row in rows if row.get("id")}
1293
1294
1295def _nav_change(
1296 entry: Mapping[str, Any],
1297 *,
1298 existing: Mapping[str, Mapping[str, str]],
1299 package_id: str,
1300 warnings: list[str],
1301) -> dict[str, Any]:
1302 live = existing.get(entry["id"])
1303 if live is None:
1304 action = "create"
1305 elif all(str(live.get(field) or "") == str(entry[field]) for field in _NAV_PACKAGE_FIELDS):
1306 # Every package-owned field is compared, blurb and order
1307 # included, so a plan that says "unchanged" is one where nothing
1308 # at all is about to be written -- a dry run that under-reports
1309 # is worse than no dry run.
1310 action = "unchanged"
1311 else:
1312 action = "update"
1313
1314 # Whether a path is actually SERVED is not knowable here: routing has
1315 # three sources (convention, site_routes records, views records) and
1316 # two of them are data this install may be about to seed. Guessing
1317 # would produce false blockers, so the only thing refused is the one
1318 # collision that is unambiguous and destructive: two packages fighting
1319 # over one door, where whichever installs last silently wins.
1320 owner = str((live or {}).get("package") or "")
1321 if owner and owner != package_id:
1322 warnings.append(
1323 f"Nav entry {entry['id']} is already registered by another package: {owner}"
1324 )
1325 for other_id, other in existing.items():
1326 if other_id == entry["id"]:
1327 continue
1328 other_owner = str(other.get("package") or "")
1329 if other_owner and other_owner != package_id and other.get("path") == entry["path"]:
1330 warnings.append(
1331 f"Nav entry {entry['id']} claims a path already registered by "
1332 f"{other_owner}: {entry['path']}"
1333 )
1334
1335 return {
1336 "id": entry["id"],
1337 "label": entry["label"],
1338 "path": entry["path"],
1339 "group": entry["group"],
1340 "surface": entry["surface"],
1341 "exists": live is not None,
1342 "action": action,
1343 }
1344
1345
1346def _apply_nav(
1347 nav: Iterable[Mapping[str, Any]],
1348 planned: Iterable[Mapping[str, Any]],
1349 *,
1350 package_id: str,
1351 base_dir: Path,
1352 roots: Iterable[Path] | None = None,
1353) -> list[dict[str, Any]]:
1354 """Register the package's doors in the `nav_entries` collection.
1355
1356 Two things are deliberately preserved across an install:
1357
1358 **The operator's `hidden`** -- `operator_hidden` is the one column an
1359 install never writes on an existing row. The package declares what
1360 the app IS; the operator decides what they want to SEE, and an
1361 upgrade that silently un-hides a page somebody deliberately took off
1362 the menu is the same class of incident as one that restarts a paused
1363 nightly pass.
1364
1365 **Somebody else's row** -- entries this package did not register are
1366 not touched and not deleted. Removing a `nav` entry from a manifest
1367 leaves the row alone, matching the rest of install: deregistering
1368 never destroys.
1369
1370 A missing `nav_entries` collection is reported and skipped rather
1371 than raised. The nav app is not a dependency of being installable.
1372 """
1373 entries = list(nav)
1374 plans = list(planned)
1375 if not entries:
1376 return []
1377
1378 existing = _load_nav_entries(base_dir, roots=roots)
1379 if existing is None:
1380 return [
1381 {**plan, "status": "skipped",
1382 "reason": f"{NAV_COLLECTION} collection is not installed"}
1383 for plan in plans
1384 ]
1385
1386 applied = []
1387 for entry, plan in zip(entries, plans, strict=True):
1388 live = existing.get(entry["id"])
1389 row = {
1390 "package": package_id,
1391 "label": entry["label"],
1392 "path": entry["path"],
1393 "blurb": entry["blurb"],
1394 "group": entry["group"],
1395 "surface": entry["surface"],
1396 "order": str(entry["order"]),
1397 }
1398 if live is None:
1399 object_records.create_collection_record(
1400 NAV_COLLECTION,
1401 {"id": entry["id"], "operator_hidden": "false", **row},
1402 base_dir=base_dir, roots=roots, actor=f"package:{package_id}",
1403 )
1404 status = "written"
1405 elif all(str(live.get(field) or "") == row[field] for field in row):
1406 status = "unchanged"
1407 else:
1408 # `row` carries no operator_hidden key at all, so the update
1409 # cannot reach that column even by accident.
1410 object_records.update_collection_record(
1411 NAV_COLLECTION, entry["id"], row,
1412 base_dir=base_dir, roots=roots, actor=f"package:{package_id}",
1413 )
1414 status = "updated"
1415
1416 applied.append({**plan, "status": status,
1417 "destination": f"collections/{NAV_COLLECTION}/{entry['id']}"})
1418 return applied
1419
1420
1421def _normalize_attention(payload: Any, *, package_id: str) -> list[dict[str, Any]]:
1422 """An `attention` entry declares one queue of things that need a human:
1423 `{id, object_id, label, path, nav_id?, group?, severity?}`.
1424
1425 Every gate and derived state on this server ends in a pile somebody
1426 has to look at -- receipts waiting to be confirmed into expenses, time
1427 entries waiting for an approver, invoices past their due date, bank
1428 lines nothing matched. The system computes those piles constantly and
1429 then discards them: `system_scan_processor` returns a field called
1430 `needing_a_human` that no surface has ever read. What was missing was
1431 not the arithmetic, it was a place to declare that the arithmetic
1432 MEANS something -- and the only honest place for that definition is
1433 the package that owns the domain, because nobody else knows that
1434 `extracted` is a waiting room and `confirmed` is not.
1435
1436 `object_id` names the provider, an object exposing `COUNT(request) ->
1437 {"count": int, "detail"?: str}`. `path` is validated for shape only,
1438 exactly as `nav` paths are: whether anything answers it is a routing
1439 question this module deliberately does not pretend to answer.
1440 `nav_id` is the door this count decorates, so the app list can read
1441 "Invoices - 5 overdue" without a second surface to maintain; it is
1442 optional because a queue can belong to a package that ships no page
1443 at all (app-intake is one).
1444 """
1445 entries = _list_field(payload, package_id=package_id, section="attention")
1446 normalized: list[dict[str, Any]] = []
1447 seen: set[str] = set()
1448 for entry in entries:
1449 mapping = _entry_mapping(entry, package_id=package_id, section="attention")
1450 attention_id = _required_text(mapping, "id", package_id=package_id)
1451 if not _ATTENTION_ID_RE.fullmatch(attention_id):
1452 raise InvalidPackageManifestError(
1453 f"Invalid package attention id: {attention_id}")
1454 if attention_id in seen:
1455 raise InvalidPackageManifestError(
1456 f"Duplicate package attention id: {attention_id}")
1457 seen.add(attention_id)
1458
1459 object_id = _required_text(mapping, "object_id", package_id=package_id)
1460 if not validate_object_id(object_id):
1461 raise InvalidPackageManifestError(
1462 f"Invalid package attention object_id: {object_id}")
1463
1464 label = _required_text(mapping, "label", package_id=package_id)
1465 path = _required_text(mapping, "path", package_id=package_id)
1466 if not path.startswith("/"):
1467 raise InvalidPackageManifestError(
1468 f"Package attention path must start with '/': {attention_id}"
1469 )
1470 if any(character.isspace() for character in path):
1471 # A count nobody can click is a number on a poster. A path
1472 # with a space or a newline in it is both a link that never
1473 # resolves and a TSV cell that can break the row it lives in.
1474 raise InvalidPackageManifestError(
1475 f"Package attention path may not contain whitespace: {attention_id}"
1476 )
1477
1478 nav_id = _optional_text(mapping.get("nav_id")) or ""
1479 if nav_id and not _NAV_ID_RE.fullmatch(nav_id):
1480 raise InvalidPackageManifestError(
1481 f"Invalid package attention nav_id: {attention_id}")
1482
1483 severity = (_optional_text(mapping.get("severity"))
1484 or _ATTENTION_DEFAULT_SEVERITY).lower()
1485 if severity not in _ATTENTION_SEVERITIES:
1486 raise InvalidPackageManifestError(
1487 f"Package attention severity must be normal|warning|urgent: {attention_id}"
1488 )
1489
1490 normalized.append({
1491 "id": attention_id,
1492 "object_id": object_id,
1493 "label": label,
1494 "path": path,
1495 "nav_id": nav_id,
1496 "group": _optional_text(mapping.get("group")) or _ATTENTION_DEFAULT_GROUP,
1497 "severity": severity,
1498 })
1499 return normalized
1500
1501
1502def _load_attention_sources(
1503 base_dir: Path,
1504 *,
1505 roots: Iterable[Path] | None = None,
1506) -> dict[str, dict[str, str]] | None:
1507 """Every attention source currently registered, keyed by record id.
1508
1509 ``None`` means the `attention_sources` collection is not installed at
1510 all (app-nav absent). Same posture as `_load_nav_entries`: a package
1511 must never fail to install because the home screen is missing, any
1512 more than an app should fail because nobody has looked at it yet.
1513 """
1514 try:
1515 rows = object_records.read_collection_records(
1516 ATTENTION_COLLECTION, base_dir=base_dir, roots=roots)
1517 except (object_collections.CollectionNotFoundError,
1518 object_collections.InvalidCollectionNameError):
1519 return None
1520 except (OSError, ValueError):
1521 # A hand-edited row must not make a package uninstallable, the
1522 # same posture _load_nav_entries takes.
1523 return {}
1524 return {str(row.get("id") or ""): dict(row) for row in rows if row.get("id")}
1525
1526
1527def _attention_change(
1528 entry: Mapping[str, Any],
1529 *,
1530 existing: Mapping[str, Mapping[str, str]],
1531 package_id: str,
1532 installable_object_ids: set[str],
1533 warnings: list[str],
1534) -> dict[str, Any]:
1535 live = existing.get(entry["id"])
1536 if live is None:
1537 action = "create"
1538 elif all(str(live.get(field) or "") == str(entry[field])
1539 for field in _ATTENTION_PACKAGE_FIELDS):
1540 # Every package-owned field is compared, nav_id and severity
1541 # included, so a plan that says "unchanged" is one where nothing
1542 # at all is about to be written.
1543 action = "unchanged"
1544 else:
1545 action = "update"
1546
1547 if entry["object_id"] not in installable_object_ids:
1548 # A counter aimed at an object that will not exist reads as zero
1549 # forever, and a queue that reads zero forever is indistinguishable
1550 # from a queue that is empty. That silent failure is the entire
1551 # reason this section exists, so it blocks the install.
1552 warnings.append(
1553 f"Attention source {entry['id']} targets an object this install "
1554 f"will not provide: {entry['object_id']}"
1555 )
1556
1557 # Two packages writing one attention_sources row is two definitions of
1558 # what needs a human, where whichever installed last silently wins.
1559 # Unlike nav there is no path-collision rule: two queues legitimately
1560 # point at one list (an orders page can be both "to pick" and "to
1561 # invoice"), and blocking that would be a false blocker.
1562 owner = str((live or {}).get("package") or "")
1563 if owner and owner != package_id:
1564 warnings.append(
1565 f"Attention source {entry['id']} is already registered by another "
1566 f"package: {owner}"
1567 )
1568
1569 return {
1570 "id": entry["id"],
1571 "object_id": entry["object_id"],
1572 "label": entry["label"],
1573 "path": entry["path"],
1574 "nav_id": entry["nav_id"],
1575 "group": entry["group"],
1576 "severity": entry["severity"],
1577 "exists": live is not None,
1578 "action": action,
1579 }
1580
1581
1582def _apply_attention(
1583 attention: Iterable[Mapping[str, Any]],
1584 planned: Iterable[Mapping[str, Any]],
1585 *,
1586 package_id: str,
1587 base_dir: Path,
1588 roots: Iterable[Path] | None = None,
1589) -> list[dict[str, Any]]:
1590 """Register the package's queues in the `attention_sources` collection.
1591
1592 Two things are deliberately preserved across an install:
1593
1594 **The operator's `operator_muted`** -- the one column an install never
1595 writes on an existing row. The package declares what a queue MEANS;
1596 the operator decides whether this deployment wants to be told about
1597 it, and an upgrade that silently un-mutes a counter somebody turned
1598 off is the same class of incident as one that restarts a paused
1599 nightly pass.
1600
1601 **Somebody else's row** -- sources this package did not register are
1602 not touched and not deleted. Removing an `attention` entry from a
1603 manifest leaves the row alone, matching the rest of install:
1604 deregistering never destroys. (The rollup pass is what stops feeding
1605 a stale row, because it only executes providers it can still resolve.)
1606
1607 A missing `attention_sources` collection is reported and skipped
1608 rather than raised. The home screen is not a dependency of being
1609 installable.
1610 """
1611 entries = list(attention)
1612 plans = list(planned)
1613 if not entries:
1614 return []
1615
1616 existing = _load_attention_sources(base_dir, roots=roots)
1617 if existing is None:
1618 return [
1619 {**plan, "status": "skipped",
1620 "reason": f"{ATTENTION_COLLECTION} collection is not installed"}
1621 for plan in plans
1622 ]
1623
1624 applied = []
1625 for entry, plan in zip(entries, plans, strict=True):
1626 live = existing.get(entry["id"])
1627 row = {
1628 "package": package_id,
1629 "object_id": entry["object_id"],
1630 "label": entry["label"],
1631 "path": entry["path"],
1632 "nav_id": entry["nav_id"],
1633 "group": entry["group"],
1634 "severity": entry["severity"],
1635 }
1636 if live is None:
1637 object_records.create_collection_record(
1638 ATTENTION_COLLECTION,
1639 {"id": entry["id"], "operator_muted": "false", **row},
1640 base_dir=base_dir, roots=roots, actor=f"package:{package_id}",
1641 )
1642 status = "written"
1643 elif all(str(live.get(field) or "") == row[field] for field in row):
1644 status = "unchanged"
1645 else:
1646 # `row` carries no operator_muted key at all, so the update
1647 # cannot reach that column even by accident.
1648 object_records.update_collection_record(
1649 ATTENTION_COLLECTION, entry["id"], row,
1650 base_dir=base_dir, roots=roots, actor=f"package:{package_id}",
1651 )
1652 status = "updated"
1653
1654 applied.append({**plan, "status": status,
1655 "destination": f"collections/{ATTENTION_COLLECTION}/{entry['id']}"})
1656 return applied
1657
1658
1659def iter_connectors(*, root: Path | str = PACKAGES_DIR) -> list[dict[str, str]]:
1660 """Every connector declaration under a package root, each with its module
1661 resolved to an absolute path proven to live INSIDE its package. Robust to a
1662 single bad package: a manifest that won't parse, or a module that is missing
1663 or escapes the package dir, is skipped (never raises) so one bad package
1664 can't blind the whole reconcile pass."""
1665 packages_root = Path(root)
1666 if not packages_root.is_dir():
1667 return []
1668 out: list[dict[str, str]] = []
1669 for path in sorted(packages_root.iterdir(), key=lambda item: item.name):
1670 if not path.is_dir() or not validate_package_id(path.name):
1671 continue
1672 if not (path / MANIFEST_FILE).is_file():
1673 continue
1674 try:
1675 package = _load_package(path.name, path)
1676 except (InvalidPackageManifestError, InvalidPackageIdError, OSError, ValueError):
1677 continue
1678 for decl in package.get("connectors", []):
1679 status = _package_file_status(path, decl["module"])
1680 if not status["exists"]: # missing module, or escapes the package
1681 continue
1682 out.append({
1683 "package_id": package["id"],
1684 "collection": decl["collection"],
1685 "module": str((path / decl["module"]).resolve()),
1686 "entry": decl["entry"],
1687 })
1688 return out
1689
1690
1691def _normalize_dependencies(payload: Any, *, package_id: str) -> list[dict[str, str | None]]:
1692 entries = _list_field(payload, package_id=package_id, section="dependencies")
1693 normalized = []
1694 for entry in entries:
1695 if isinstance(entry, str):
1696 dependency_id = entry
1697 version = None
1698 else:
1699 mapping = _entry_mapping(entry, package_id=package_id, section="dependencies")
1700 dependency_id = _required_text(mapping, "id", package_id=package_id)
1701 version = _optional_text(mapping.get("version"))
1702 if not validate_package_id(dependency_id):
1703 raise InvalidPackageManifestError(f"Invalid package dependency id: {dependency_id}")
1704 normalized.append({"id": dependency_id, "version": version})
1705 return normalized
1706
1707
1708def _object_change(
1709 entry: Mapping[str, str],
1710 *,
1711 package_dir: Path,
1712 object_roots: Iterable[Path] | None,
1713 warnings: list[str],
1714) -> dict[str, Any]:
1715 file_status = _package_file_status(package_dir, entry["path"])
1716 if not file_status["exists"]:
1717 warnings.append(f"Missing package object file: {entry['path']}")
1718 installed = resolve_object_id(entry["id"], object_roots) is not None
1719 return {
1720 "id": entry["id"],
1721 "path": entry["path"],
1722 "exists": file_status["exists"],
1723 "action": "replace" if installed else "create",
1724 "installed": installed,
1725 }
1726
1727
1728def _schema_change(
1729 entry: Mapping[str, str],
1730 *,
1731 package_dir: Path,
1732 base_dir: Path,
1733 warnings: list[str],
1734) -> dict[str, Any]:
1735 file_status = _package_file_status(package_dir, entry["path"])
1736 if not file_status["exists"]:
1737 warnings.append(f"Missing package schema file: {entry['path']}")
1738 installed = (base_dir / "schemas" / f"{entry['collection']}.json").is_file()
1739 return {
1740 "collection": entry["collection"],
1741 "path": entry["path"],
1742 "exists": file_status["exists"],
1743 "action": "replace" if installed else "create",
1744 "installed": installed,
1745 }
1746
1747
1748def _seed_change(
1749 entry: Mapping[str, str],
1750 *,
1751 package_dir: Path,
1752 base_dir: Path,
1753 warnings: list[str],
1754) -> dict[str, Any]:
1755 file_status = _package_file_status(package_dir, entry["path"])
1756 if not file_status["exists"]:
1757 warnings.append(f"Missing package seed file: {entry['path']}")
1758 installed = (base_dir / "collections" / entry["collection"] / "records.tsv").is_file()
1759 return {
1760 "collection": entry["collection"],
1761 "path": entry["path"],
1762 "exists": file_status["exists"],
1763 # Seed is install-once: on an upgrade where the collection already has
1764 # records, seeding is skipped so live data is preserved (not merged).
1765 "action": "skip" if installed else "create",
1766 "installed": installed,
1767 }
1768
1769
1770def _migration_change(
1771 entry: Mapping[str, str],
1772 *,
1773 package: str,
1774 package_dir: Path,
1775 base_dir: Path,
1776 warnings: list[str],
1777) -> dict[str, Any]:
1778 file_status = _package_file_status(package_dir, entry["path"])
1779 if not file_status["exists"]:
1780 warnings.append(f"Missing package migration file: {entry['path']}")
1781 marker = base_dir / PACKAGE_MIGRATIONS_DIR / package / f"{entry['id']}.json"
1782 applied = marker.is_file()
1783 return {
1784 "id": entry["id"],
1785 "path": entry["path"],
1786 "exists": file_status["exists"],
1787 "action": "skip" if applied else "apply",
1788 "applied": applied,
1789 }
1790
1791
1792def _permission_change(
1793 entry: Mapping[str, str],
1794 *,
1795 package_id: str,
1796 package_dir: Path,
1797 base_dir: Path,
1798 warnings: list[str],
1799) -> dict[str, Any]:
1800 change = {
1801 "path": entry["path"],
1802 "action": "merge",
1803 "exists": False,
1804 "rules": 0,
1805 "new_rules": 0,
1806 }
1807 file_status = _package_file_status(package_dir, entry["path"])
1808 if not file_status["exists"]:
1809 warnings.append(f"Missing package permissions file: {entry['path']}")
1810 return change
1811 change["exists"] = True
1812
1813 try:
1814 rules = _load_permission_rules(package_dir, entry, package_id=package_id)
1815 except PackageInstallError as exc:
1816 warnings.append(str(exc))
1817 return change
1818
1819 change["rules"] = len(rules)
1820 try:
1821 policy = object_permission_store.load_policy(base_dir)
1822 existing_keys = {_rule_merge_key(rule) for rule in policy.rules}
1823 except ValueError:
1824 existing_keys = set()
1825 change["new_rules"] = sum(1 for rule in rules if _rule_merge_key(rule) not in existing_keys)
1826 return change
1827
1828
1829def _load_permission_rules(
1830 package_dir: Path,
1831 entry: Mapping[str, str],
1832 *,
1833 package_id: str,
1834) -> list[object_permissions.PermissionRule]:
1835 source = _package_file(package_dir, entry["path"])
1836 try:
1837 payload = json.loads(source.read_text(encoding="utf-8"))
1838 except (OSError, json.JSONDecodeError) as exc:
1839 raise PackageInstallError(
1840 f"Package permissions file contains invalid JSON: {entry['path']}"
1841 ) from exc
1842
1843 rules_payload = payload.get("rules") if isinstance(payload, dict) else payload
1844 if not isinstance(rules_payload, list):
1845 raise PackageInstallError(
1846 f"Package permissions file must contain a rules list: {entry['path']}"
1847 )
1848
1849 rules = []
1850 for rule_payload in rules_payload:
1851 if not isinstance(rule_payload, dict):
1852 raise PackageInstallError(
1853 f"Package permission rules must be objects: {entry['path']}"
1854 )
1855 merged_payload = {**rule_payload, "package": package_id}
1856 try:
1857 rules.append(object_permissions.rule_from_dict(merged_payload))
1858 except ValueError as exc:
1859 raise PackageInstallError(
1860 f"Package permission rule is invalid in {entry['path']}: {exc}"
1861 ) from exc
1862 return rules
1863
1864
1865def _rule_merge_key(rule: object_permissions.PermissionRule) -> str:
1866 payload = object_permissions.rule_to_dict(rule)
1867 payload.pop("reason", None)
1868 payload.pop("package", None)
1869 return json.dumps(payload, sort_keys=True)
1870
1871
1872def _merge_permission_rules(
1873 rules: list[object_permissions.PermissionRule],
1874 *,
1875 base_dir: Path,
1876) -> tuple[int, int]:
1877 """Append new package rules to the policy; return (total, newly added)."""
1878 policy = object_permission_store.load_policy(base_dir)
1879 existing_keys = {_rule_merge_key(rule) for rule in policy.rules}
1880
1881 added = []
1882 for rule in rules:
1883 if _rule_merge_key(rule) in existing_keys:
1884 continue
1885 added.append(rule)
1886 existing_keys.add(_rule_merge_key(rule))
1887
1888 if added:
1889 merged = object_permissions.PermissionPolicy(
1890 access_mode=policy.access_mode,
1891 rules=tuple(policy.rules) + tuple(added),
1892 roles=policy.roles,
1893 user_roles=policy.user_roles,
1894 admin_roles=policy.admin_roles,
1895 )
1896 object_permission_store.save_policy(merged, base_dir=base_dir)
1897
1898 return len(rules), len(added)
1899
1900
1901def _path_change(
1902 entry: Mapping[str, str],
1903 *,
1904 package_dir: Path,
1905 section: str,
1906 action: str,
1907 warnings: list[str],
1908) -> dict[str, Any]:
1909 file_status = _package_file_status(package_dir, entry["path"])
1910 if not file_status["exists"]:
1911 warnings.append(f"Missing package {section} file: {entry['path']}")
1912 return {
1913 "path": entry["path"],
1914 "exists": file_status["exists"],
1915 "action": action,
1916 }
1917
1918
1919def _package_file_status(package_dir: Path, relative_path: str) -> dict[str, bool]:
1920 package_root = package_dir.resolve()
1921 candidate = package_dir / relative_path
1922 resolved = candidate.resolve(strict=False)
1923 inside = resolved == package_root or package_root in resolved.parents
1924 return {
1925 "exists": inside and candidate.is_file(),
1926 "inside_package": inside,
1927 }
1928
1929
1930def _install_blockers(
1931 plan: Mapping[str, Any],
1932 *,
1933 package: Mapping[str, Any],
1934 allow_replace: bool,
1935) -> list[str]:
1936 blockers = [str(warning) for warning in plan.get("warnings", [])]
1937
1938 if package["migrations"]:
1939 blockers.append("Package migration execution is not implemented yet")
1940
1941 if not allow_replace:
1942 for entry in plan["objects"]:
1943 if entry["action"] == "replace":
1944 blockers.append(f"Object already exists; set allow_replace=true: {entry['id']}")
1945 for entry in plan["schemas"]:
1946 if entry["action"] == "replace":
1947 blockers.append(
1948 f"Schema already exists; set allow_replace=true: {entry['collection']}"
1949 )
1950
1951 # Existing seed data is NOT a blocker: seed is install-once, so an upgrade
1952 # of an already-installed package skips seeding and preserves live records
1953 # (see the seed apply step). This is what makes "upgrade the app, keep the
1954 # data" work — the package ships code + schema + a seed template, while the
1955 # records live outside the package in the server's data dir.
1956
1957 return blockers
1958
1959
1960def _package_file(package_dir: Path, relative_path: str) -> Path:
1961 status = _package_file_status(package_dir, relative_path)
1962 if not status["inside_package"]:
1963 raise PackageInstallError(f"Package file escapes package directory: {relative_path}")
1964 if not status["exists"]:
1965 raise PackageInstallError(f"Package file does not exist: {relative_path}")
1966 return package_dir / relative_path
1967
1968
1969def _object_destination(entry: Mapping[str, str], object_root: Path) -> Path:
1970 relative = Path(entry["path"])
1971 if relative.parts and relative.parts[0] == "objects":
1972 relative = Path(*relative.parts[1:])
1973 if not relative.parts or relative.suffix != ".py":
1974 raise PackageInstallError(f"Package object path must point to a Python file: {entry['path']}")
1975 destination = object_root / relative
1976 _ensure_inside(destination, object_root, label="object")
1977 return destination
1978
1979
1980def _ensure_inside(path: Path, root: Path, *, label: str) -> None:
1981 resolved_path = path.resolve(strict=False)
1982 resolved_root = root.resolve(strict=False)
1983 try:
1984 resolved_path.relative_to(resolved_root)
1985 except ValueError as exc:
1986 raise PackageInstallError(f"Package {label} destination escapes its root: {path}") from exc
1987
1988
1989def _root_for_path(path: Path | None, roots: Iterable[Path]) -> Path | None:
1990 if path is None:
1991 return None
1992 for root in roots:
1993 try:
1994 _ensure_inside(path, root, label="object")
1995 except PackageInstallError:
1996 continue
1997 return root
1998 return None
1999
2000
2001def _merge_seed_rows(
2002 collection: str, content: bytes, *, base_dir: Path, package_id: str
2003) -> int:
2004 """Add the seed rows whose id isn't already in ``collection``; leave every
2005 existing row untouched. Returns how many were added. Seed rows carry
2006 explicit ids and created_at, which ``create_collection_record`` honors
2007 (require_id + preserve_read_only), so a re-run adds nothing. One bad row is
2008 skipped, never fails the install."""
2009 try:
2010 text = content.decode("utf-8")
2011 except UnicodeDecodeError:
2012 return 0
2013 rows = list(csv.DictReader(io.StringIO(text), delimiter="\t"))
2014 if not rows:
2015 return 0
2016 try:
2017 existing = {
2018 r.get("id")
2019 for r in object_records.read_collection_records(collection, base_dir=base_dir)
2020 }
2021 except (object_collections.CollectionNotFoundError,
2022 object_collections.InvalidCollectionNameError, OSError, ValueError):
2023 existing = set()
2024 added = 0
2025 for row in rows:
2026 record_id = (row.get("id") or "").strip()
2027 if not record_id or record_id in existing:
2028 continue
2029 try:
2030 object_records.create_collection_record(
2031 collection, {k: v for k, v in row.items() if v is not None},
2032 base_dir=base_dir, actor=f"package:{package_id}", preserve_read_only=True,
2033 )
2034 existing.add(record_id)
2035 added += 1
2036 except Exception: # noqa: BLE001 -- one malformed seed row never fails the install
2037 continue
2038 return added
2039
2040
2041def _write_file_atomic_bytes(path: Path, content: bytes) -> None:
2042 path.parent.mkdir(parents=True, exist_ok=True)
2043 tmp_name = ""
2044 try:
2045 with tempfile.NamedTemporaryFile(
2046 "wb",
2047 dir=path.parent,
2048 prefix=f".{path.name}.",
2049 suffix=".tmp",
2050 delete=False,
2051 ) as tmp:
2052 tmp_name = tmp.name
2053 tmp.write(content)
2054 tmp.flush()
2055 os.fsync(tmp.fileno())
2056 os.replace(tmp_name, path)
2057 finally:
2058 if tmp_name:
2059 try:
2060 Path(tmp_name).unlink()
2061 except FileNotFoundError:
2062 pass
2063
2064
2065def _attribute_seed_records(collection: str, *, package_id: str, base_dir: Path) -> None:
2066 """Emit an attributed "create" change for every row a seed file just wrote.
2067
2068 Seed writes land the whole records.tsv in one atomic byte-for-byte
2069 copy (see the seed_writes loop above), bypassing
2070 object_records.create_collection_record entirely -- so, unlike a
2071 normal record write, nothing emits a change automatically. This is
2072 the seed-install side of universal attribution: read the file back
2073 (install-once, so every row it now holds is a fresh create) and log
2074 one change per row, attributed to the installing package rather than
2075 left unattributed.
2076 """
2077 try:
2078 rows = object_records.read_collection_records(collection, base_dir=base_dir)
2079 except (object_collections.CollectionNotFoundError, object_collections.InvalidCollectionNameError):
2080 return
2081 for row in rows:
2082 object_record_changes.append_record_change(
2083 collection=collection,
2084 record_id=row["id"],
2085 action="create",
2086 before=None,
2087 after=row,
2088 actor=f"package-install:{package_id}",
2089 base_dir=base_dir,
2090 )
2091
2092
2093def _relative_display_path(path: Path, root: Path) -> str:
2094 try:
2095 return path.resolve(strict=False).relative_to(root.resolve(strict=False)).as_posix()
2096 except ValueError:
2097 return path.name
2098
2099
2100def _safe_relative_path(value: str, *, package_id: str, section: str) -> str:
2101 if "\x00" in value:
2102 raise InvalidPackageManifestError(f"Package {section} path contains a null byte: {package_id}")
2103 path = Path(value)
2104 if path.is_absolute() or not path.parts:
2105 raise InvalidPackageManifestError(f"Package {section} path must be relative: {package_id}")
2106 if any(part in {"", ".", ".."} for part in path.parts):
2107 raise InvalidPackageManifestError(f"Package {section} path is not safe: {package_id}")
2108 return path.as_posix()
2109
2110
2111def _required_text(payload: Mapping[str, Any], key: str, *, package_id: str) -> str:
2112 value = payload.get(key)
2113 text = _optional_text(value)
2114 if text is None:
2115 raise InvalidPackageManifestError(f"Package manifest requires '{key}': {package_id}")
2116 return text
2117
2118
2119def _optional_text(value: Any) -> str | None:
2120 if value is None:
2121 return None
2122 if not isinstance(value, str):
2123 return None
2124 text = value.strip()
2125 return text or None
2126
2127
2128def _mapping_field(value: Any, *, package_id: str) -> dict[str, Any]:
2129 if value is None:
2130 return {}
2131 if not isinstance(value, Mapping):
2132 raise InvalidPackageManifestError(f"Package compatibility must be an object: {package_id}")
2133 return dict(value)
2134
2135
2136def _list_field(value: Any, *, package_id: str, section: str) -> list[Any]:
2137 if value is None:
2138 return []
2139 if not isinstance(value, list):
2140 raise InvalidPackageManifestError(f"Package {section} must be a list: {package_id}")
2141 return value
2142
2143
2144def _entry_mapping(value: Any, *, package_id: str, section: str) -> Mapping[str, Any]:
2145 if not isinstance(value, Mapping):
2146 raise InvalidPackageManifestError(f"Package {section} entries must be objects: {package_id}")
2147 return value