Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_schemas.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_schemas.py · 24.0 KB · 620 lines
1"""Schema metadata for DBBASIC collections.
2
3Schemas describe the fields, validation hints, and relations Scroll can use to
4render forms, tables, and diagrams. They are metadata, not the default storage
5engine; objects can still read/write TSV files, APIs, SQL, or other backends.
6"""
7
8from __future__ import annotations
9
10import json
11import os
12import re
13import tempfile
14from pathlib import Path
15from typing import Any, Iterable, Mapping
16
17import object_collections
18from object_versions import DEFAULT_DATA_DIR
19
20SCHEMAS_DIR = "schemas"
21_FIELD_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$")
22
23# Storage engine opt-in (docs/append-only-storage-design.md). A schema's
24# top-level "storage" key selects how object_records.py physically writes
25# a collection's records.tsv: "classic" (default, omitted key) rewrites the
26# whole file per write, unchanged from the original behavior; "append"
27# opts a collection into append-only writes with last-wins-by-id reads.
28# Only present in the normalized schema when the source payload set it, so
29# a schema that never mentions storage stays byte-identical to before this
30# feature existed (see the metadata-key handling in _normalize_schema).
31STORAGE_CLASSIC = "classic"
32STORAGE_APPEND = "append"
33VALID_STORAGE_MODES = frozenset({STORAGE_CLASSIC, STORAGE_APPEND})
34
35# Module-level cache for parsed+normalized manual schemas, keyed by the
36# resolved schema file path. Value is ((mtime_ns, size), normalized_schema).
37# Every caller of get_schema/_load_manual_schema was audited (object_records,
38# object_field_permissions, object_packages, object_server, object_mcp
39# argument-building, and the test suite) and none of them mutate the
40# returned dict -- they only read from it (json.dumps for hashing, `.get`
41# for field lookups, dict/set comprehensions that copy). So cache hits
42# return the cached dict directly rather than paying for a defensive copy;
43# if a future caller starts mutating it, that would need revisiting.
44# Missing files are never cached (see _load_manual_schema): a schema being
45# created later must be picked up on the very next call.
46_SCHEMA_CACHE: dict[str, tuple[tuple[int, int], dict[str, Any]]] = {}
47
48
49class InvalidSchemaNameError(ValueError):
50 """Raised when a schema name is not safe for routes or storage."""
51
52
53class SchemaNotFoundError(LookupError):
54 """Raised when a schema cannot be found or derived."""
55
56
57def validate_schema_name(schema: str) -> bool:
58 """Return True when a schema/collection name is route-safe."""
59 return object_collections.validate_collection_name(schema)
60
61
62def list_schemas(
63 *,
64 base_dir: Path | str = DEFAULT_DATA_DIR,
65 roots: Iterable[Path] | None = None,
66) -> list[dict[str, Any]]:
67 """Return summaries for manual schemas and derived collection schemas."""
68 base = Path(base_dir)
69 collection_names = {item["name"] for item in object_collections.list_collections(base_dir=base, roots=roots)}
70 manual_names = set(_iter_manual_schema_names(base))
71 schemas = [
72 _schema_summary(name, _load_manual_schema(name, base_dir=base), source="manual")
73 if name in manual_names
74 else _schema_summary(name, _derived_schema(name), source="derived")
75 for name in sorted(collection_names | manual_names)
76 ]
77 return schemas
78
79
80def list_append_storage_collections(
81 *,
82 base_dir: Path | str = DEFAULT_DATA_DIR,
83) -> list[str]:
84 """Return the names of collections whose manual schema currently
85 declares ``"storage": "append"``, sorted.
86
87 Only a manual schema file can set `storage` (a derived schema -- a
88 collection with no schema file -- always defaults to classic; see
89 _collection_storage_mode in object_records.py), so this only needs to
90 glob the schemas directory and load each manual schema through
91 _load_manual_schema's existing mtime/size-keyed _SCHEMA_CACHE, rather
92 than walking every collection's records directory. Repeated calls
93 (e.g. object_records.list_append_collection_stats on every /admin/
94 storage poll, or the daemon's compaction pass on every tick) are cheap
95 after the first, same as every other schema read in this codebase.
96 """
97 base = Path(base_dir)
98 names = []
99 for name in _iter_manual_schema_names(base):
100 schema = _load_manual_schema(name, base_dir=base)
101 if schema is not None and schema.get("storage") == STORAGE_APPEND:
102 names.append(name)
103 return sorted(names)
104
105
106def get_schema(
107 schema: str,
108 *,
109 base_dir: Path | str = DEFAULT_DATA_DIR,
110 roots: Iterable[Path] | None = None,
111) -> dict[str, Any]:
112 """Return one manual schema, or an empty derived schema for a collection."""
113 if not validate_schema_name(schema):
114 raise InvalidSchemaNameError(f"Invalid schema name: {schema}")
115
116 base = Path(base_dir)
117 manual_schema = _load_manual_schema(schema, base_dir=base)
118 if manual_schema is not None:
119 return manual_schema
120
121 collection_names = {
122 item["name"]
123 for item in object_collections.list_collections(base_dir=base, roots=roots)
124 }
125 if schema in collection_names:
126 return _derived_schema(schema)
127
128 raise SchemaNotFoundError(f"Schema not found: {schema}")
129
130
131def replace_schema(
132 schema: str,
133 payload: Mapping[str, Any],
134 *,
135 base_dir: Path | str = DEFAULT_DATA_DIR,
136) -> dict[str, Any]:
137 """Validate and atomically replace a manual schema file."""
138 if not validate_schema_name(schema):
139 raise InvalidSchemaNameError(f"Invalid schema name: {schema}")
140
141 base = Path(base_dir)
142 normalized = normalize_schema(schema, payload, source="manual")
143 root = _schema_root(base)
144 root.mkdir(parents=True, exist_ok=True)
145 path = _schema_path(schema, base)
146
147 tmp_name = ""
148 try:
149 with tempfile.NamedTemporaryFile(
150 "w",
151 encoding="utf-8",
152 dir=root,
153 prefix=f".{schema}.",
154 suffix=".tmp",
155 delete=False,
156 ) as tmp:
157 tmp_name = tmp.name
158 json.dump(normalized, tmp, indent=2, sort_keys=True)
159 tmp.write("\n")
160 tmp.flush()
161 os.fsync(tmp.fileno())
162 os.replace(tmp_name, path)
163 finally:
164 if tmp_name:
165 try:
166 Path(tmp_name).unlink()
167 except FileNotFoundError:
168 pass
169
170 # Explicit invalidate rather than relying solely on the next read's stat
171 # mismatch: mtime_ns has nanosecond resolution on APFS/ext4, but some
172 # filesystems (and clocks) are coarser, so a write that lands in the
173 # same tick as the cached signature could otherwise look unchanged.
174 _SCHEMA_CACHE.pop(str(path.resolve(strict=False)), None)
175
176 return normalized
177
178
179def merge_schema_fields(
180 base: Mapping[str, Any],
181 mine: Mapping[str, Any],
182 theirs: Mapping[str, Any],
183) -> tuple[dict[str, Any], list[str]]:
184 """Three-way merge a schema's field list; return (merged_schema, collisions).
185
186 Schemas are additive (docs/upgrade-and-customization.md, Rule 3: Data
187 Fields That Survive Schema Upgrades): an operator-added field and a
188 package-added field should both survive an upgrade rather than forcing a
189 conflict just because *something* changed on both sides. `base` is the
190 schema as last recorded in the package baseline, `mine` is the live
191 (possibly customized) schema, and `theirs` is the newly shipped schema.
192
193 The result is `dict(theirs)` with `"fields"` replaced by the merged list.
194 Field order is theirs' order first, then any mine-only fields appended in
195 mine's order. `collisions` names fields that changed incompatibly on both
196 sides (same field, no shared ancestor value, and mine != theirs); those
197 fields keep the operator's version in `merged_schema`, pending a human
198 decision -- callers should treat any non-empty `collisions` as "do not
199 apply, park a conflict instead."
200
201 Pure function: no I/O, no validation of the inputs or output.
202 """
203 base_by = {field["name"]: field for field in (base or {}).get("fields", [])}
204 mine_by = {field["name"]: field for field in (mine or {}).get("fields", [])}
205 theirs_by = {field["name"]: field for field in (theirs or {}).get("fields", [])}
206
207 theirs_names = [field["name"] for field in (theirs or {}).get("fields", [])]
208 mine_names = [field["name"] for field in (mine or {}).get("fields", [])]
209 mine_only_names = [name for name in mine_names if name not in theirs_by]
210 ordered_names = theirs_names + mine_only_names
211
212 collisions: list[str] = []
213 merged_fields: list[dict[str, Any]] = []
214
215 for name in ordered_names:
216 b = base_by.get(name)
217 m = mine_by.get(name)
218 t = theirs_by.get(name)
219
220 if m is not None and t is not None:
221 if m == t:
222 merged_fields.append(m)
223 elif b is not None and m == b:
224 # Operator didn't touch it; package changed it.
225 merged_fields.append(t)
226 elif b is not None and t == b:
227 # Package didn't touch it; operator changed it.
228 merged_fields.append(m)
229 else:
230 # Both changed it (or there's no shared base to compare to):
231 # a genuine collision. Keep the operator's version pending
232 # a resolution, but flag it.
233 collisions.append(name)
234 merged_fields.append(m)
235 elif m is not None:
236 # Operator has it, package doesn't (never shipped it, or
237 # removed it). Additive-safe: never silently drop a field the
238 # operator kept.
239 merged_fields.append(m)
240 else:
241 # Package has it, operator doesn't.
242 if b is None:
243 # Newly added by the package.
244 merged_fields.append(t)
245 elif t == b:
246 # Operator removed it, package left it unchanged: respect
247 # the removal.
248 continue
249 else:
250 # Operator removed it, but the package changed it:
251 # collision -- surface it rather than silently resurrect
252 # a field the operator deliberately dropped.
253 collisions.append(name)
254 merged_fields.append(t)
255
256 merged = dict(theirs)
257 merged["fields"] = merged_fields
258 return merged, collisions
259
260
261def normalize_schema(
262 schema: str,
263 payload: Mapping[str, Any],
264 *,
265 source: str = "manual",
266) -> dict[str, Any]:
267 """Validate schema metadata and return canonical JSON-compatible data."""
268 if not validate_schema_name(schema):
269 raise InvalidSchemaNameError(f"Invalid schema name: {schema}")
270 return _normalize_schema(schema, payload, source=source)
271
272
273def _iter_manual_schema_names(base_dir: Path) -> list[str]:
274 root = _schema_root(base_dir)
275 if not root.exists() or not root.is_dir():
276 return []
277
278 names = []
279 for path in sorted(root.glob("*.json"), key=lambda item: item.name):
280 name = path.stem
281 if validate_schema_name(name):
282 names.append(name)
283 return names
284
285
286def _load_manual_schema(schema: str, *, base_dir: Path) -> dict[str, Any] | None:
287 path = _schema_path(schema, base_dir)
288 if not path.exists():
289 return None
290 if not path.is_file():
291 raise ValueError(f"Schema path is not a file: {schema}")
292
293 cache_key = str(path.resolve(strict=False))
294 try:
295 stat_result = path.stat()
296 except OSError:
297 stat_result = None
298
299 if stat_result is not None:
300 signature = (stat_result.st_mtime_ns, stat_result.st_size)
301 cached = _SCHEMA_CACHE.get(cache_key)
302 if cached is not None and cached[0] == signature:
303 return cached[1]
304
305 try:
306 payload = json.loads(path.read_text())
307 except json.JSONDecodeError as exc:
308 raise ValueError(f"Schema file contains invalid JSON: {schema}") from exc
309
310 normalized = _normalize_schema(schema, payload, source="manual")
311 if stat_result is not None:
312 _SCHEMA_CACHE[cache_key] = (signature, normalized)
313 return normalized
314
315
316def _derived_schema(schema: str) -> dict[str, Any]:
317 return {
318 "name": schema,
319 "title": _title_from_name(schema),
320 "source": "derived",
321 "version": 1,
322 "fields": [],
323 "field_count": 0,
324 }
325
326
327def _schema_summary(
328 schema: str,
329 payload: dict[str, Any] | None,
330 *,
331 source: str,
332) -> dict[str, Any]:
333 active = payload or _derived_schema(schema)
334 return {
335 "name": active["name"],
336 "title": active["title"],
337 "source": source,
338 "version": active["version"],
339 "field_count": active["field_count"],
340 }
341
342
343def _normalize_schema(
344 schema: str,
345 payload: Any,
346 *,
347 source: str,
348) -> dict[str, Any]:
349 if not isinstance(payload, Mapping):
350 raise ValueError(f"Schema file must contain an object: {schema}")
351
352 name = _optional_string(payload.get("name")) or _optional_string(payload.get("collection")) or schema
353 if name != schema:
354 raise ValueError(f"Schema file name does not match schema collection: {schema}")
355 if not validate_schema_name(name):
356 raise InvalidSchemaNameError(f"Invalid schema name: {name}")
357
358 fields_payload = payload.get("fields", [])
359 if not isinstance(fields_payload, list):
360 raise ValueError(f"Schema fields must be a list: {schema}")
361
362 fields = [_normalize_field(field, schema=schema) for field in fields_payload]
363 normalized = {
364 "name": name,
365 "title": _optional_string(payload.get("title")) or _title_from_name(name),
366 "source": source,
367 "version": _optional_int(payload.get("version")) or 1,
368 "fields": fields,
369 "field_count": len(fields),
370 }
371 if "storage" in payload:
372 storage = payload.get("storage")
373 if storage not in VALID_STORAGE_MODES:
374 raise ValueError(
375 f"Schema storage must be one of {sorted(VALID_STORAGE_MODES)}: {schema}"
376 )
377 normalized["storage"] = storage
378 metadata_keys = (
379 "description",
380 "permissions",
381 "ui",
382 "layout",
383 "views",
384 "forms",
385 "search",
386 "table",
387 "diagram",
388 "validation",
389 # 10-flow-spec.md's root-level `flow` key (which field on this
390 # collection owns a `transitions` map, plus who/what annotations on
391 # top of it) -- no shipped schema declares this yet, but it must
392 # survive normalization for 60-list-modes-spec.md's board mode to
393 # honor its own documented default-group-field chain the moment one
394 # does. Passed through unchanged, same as every other metadata key
395 # here: this whitelists the key, it does not interpret it.
396 "flow",
397 # 64-feed-spec.md's root-level `blocks` key (a collection opts into a
398 # block -- e.g. `blocks.feed` names the owner/visibility/time fields
399 # that make it a feed source). Must survive normalization or install
400 # would strip it from the stored schema and the feed would find zero
401 # sources. Whitelisted, not interpreted -- same posture as `flow`.
402 "blocks",
403 # Root-level `capabilities` key: generic per-collection behaviors a
404 # collection opts into, wired by the platform rather than hand-built
405 # per app (e.g. `capabilities.comments: true` -> the detail page mounts
406 # window.dbbasicThread for that record; see app-thread + view_render).
407 # Must survive normalization or the flag is stripped on install and the
408 # capability never turns on. Whitelisted, not interpreted -- same
409 # posture as `flow`/`blocks`.
410 "capabilities",
411 # Root-level `hooks` key: per-collection pre-write hook declaration
412 # (`hooks.before_write` names an object the generic HTTP write path
413 # executes synchronously -- reject/transform -- after permission
414 # checks, before persist; see object_server._apply_before_write_hook
415 # and plan/pre-write-hook-spec.md). Server-side only: deliberately NOT
416 # exposed by /api/schema. Whitelisted, not interpreted.
417 "hooks",
418 )
419 for key in metadata_keys:
420 if key in payload:
421 normalized[key] = _json_compatible(payload[key], field_name=f"{schema}.{key}")
422 return normalized
423
424
425def _normalize_field(payload: Any, *, schema: str) -> dict[str, Any]:
426 if not isinstance(payload, Mapping):
427 raise ValueError(f"Schema field must be an object: {schema}")
428
429 name = _optional_string(payload.get("name"))
430 if name is None or not _FIELD_NAME_RE.fullmatch(name):
431 raise ValueError(f"Schema field has invalid name: {schema}")
432 if name == "_op":
433 # Reserved for the append-only storage engine's internal op column
434 # (docs/append-only-storage-design.md) -- never a real schema
435 # field, so a schema field literally named "_op" can never collide
436 # with it once a collection opts into "storage": "append".
437 raise ValueError(f"Schema field name is reserved: {schema}._op")
438
439 field: dict[str, Any] = {
440 "name": name,
441 "type": _optional_string(payload.get("type")) or "text",
442 "required": bool(payload.get("required", False)),
443 }
444
445 metadata_keys = (
446 "label",
447 "description",
448 "relation",
449 "validation",
450 "default",
451 "enum",
452 "computed",
453 "read_only",
454 "readonly",
455 "readOnly",
456 "ui",
457 "layout",
458 "permissions",
459 "placeholder",
460 "help",
461 "store",
462 # Conditional field visibility (Stage-6 extraction): a field may
463 # declare `visible_when` = {field, equals|in} so the /form renderer
464 # shows it only when another field on the record holds a given value
465 # (e.g. product asset fields visible only when product_type ==
466 # "asset"). Must survive normalization or install would strip it and
467 # every field would always show. Whitelisted, not interpreted here --
468 # same posture as the root-level `blocks`/`flow`/`views` keys.
469 "visible_when",
470 # Formula & rollup declarations on `type: computed` fields
471 # (plan/formula-rollup-spec.md): `formula` is a tiny expression over
472 # sibling fields, `rollup` aggregates a child collection. Interpreted
473 # by object_records/_apply_computed_fields at write time; must survive
474 # normalization or derived values silently never compute.
475 "formula",
476 "rollup",
477 )
478 for key in metadata_keys:
479 if key in payload:
480 field[key] = _json_compatible(payload[key], field_name=f"{schema}.{name}.{key}")
481 if "transitions" in payload:
482 field["transitions"] = _normalize_transitions(
483 payload["transitions"], field_name=f"{schema}.{name}.transitions"
484 )
485
486 return field
487
488
489def _normalize_transitions(value: Any, *, field_name: str) -> dict[str, list[Any]]:
490 """Normalize a field's ``transitions`` map, validating the new guarded shape.
491
492 Each current-value key maps to a list where an entry is either a plain
493 string (allowed for any caller who may update the record) or an object
494 ``{"to": "<value>", "when": {<field>: <$var or literal>, ...}}`` whose
495 move is only allowed once a subject is available and every ``when``
496 clause matches (see object_records._validate_field_transitions). This
497 only validates shape; the closed set of $-variables is resolved later,
498 against the record and subject, by object_permissions.
499 """
500 if not isinstance(value, Mapping):
501 raise ValueError(f"Schema field '{field_name}' must be an object")
502
503 normalized: dict[str, list[Any]] = {}
504 for from_value, entries in value.items():
505 if not isinstance(from_value, str):
506 raise ValueError(f"Schema field '{field_name}' keys must be strings")
507 if not isinstance(entries, list):
508 raise ValueError(f"Schema field '{field_name}[{from_value!r}]' must be a list")
509
510 normalized_entries: list[Any] = []
511 for entry in entries:
512 if isinstance(entry, str):
513 normalized_entries.append(entry)
514 continue
515 if isinstance(entry, Mapping):
516 to_value = entry.get("to")
517 if not isinstance(to_value, str) or not to_value:
518 raise ValueError(
519 f"Schema field '{field_name}[{from_value!r}]' entry is missing a "
520 "string 'to' value"
521 )
522 normalized_entry: dict[str, Any] = {"to": to_value}
523 if "when" in entry:
524 when = entry["when"]
525 if not isinstance(when, Mapping) or not when:
526 raise ValueError(
527 f"Schema field '{field_name}[{from_value!r}]' 'when' must be a "
528 "non-empty object"
529 )
530 normalized_when: dict[str, str] = {}
531 for when_key, when_value in when.items():
532 if not isinstance(when_key, str) or not when_key:
533 raise ValueError(
534 f"Schema field '{field_name}[{from_value!r}]' 'when' keys "
535 "must be non-empty strings"
536 )
537 if not isinstance(when_value, str) or not when_value:
538 raise ValueError(
539 f"Schema field '{field_name}[{from_value!r}]' 'when' values "
540 "must be non-empty strings"
541 )
542 normalized_when[when_key] = when_value
543 normalized_entry["when"] = normalized_when
544 extra_keys = sorted(set(entry.keys()) - {"to", "when"})
545 if extra_keys:
546 raise ValueError(
547 f"Schema field '{field_name}[{from_value!r}]' has unexpected keys: "
548 f"{extra_keys}"
549 )
550 normalized_entries.append(normalized_entry)
551 continue
552 raise ValueError(
553 f"Schema field '{field_name}[{from_value!r}]' entries must be strings or objects"
554 )
555 normalized[from_value] = normalized_entries
556
557 return normalized
558
559
560def _schema_root(base_dir: Path) -> Path:
561 return base_dir / SCHEMAS_DIR
562
563
564def _schema_path(schema: str, base_dir: Path) -> Path:
565 if not validate_schema_name(schema):
566 raise InvalidSchemaNameError(f"Invalid schema name: {schema}")
567
568 root = _schema_root(base_dir)
569 path = root / f"{schema}.json"
570 resolved_root = root.resolve(strict=False)
571 resolved_path = path.resolve(strict=False)
572
573 try:
574 resolved_path.relative_to(resolved_root)
575 except ValueError as exc:
576 raise InvalidSchemaNameError(f"Schema path escapes schema directory: {schema}") from exc
577
578 return path
579
580
581def _title_from_name(name: str) -> str:
582 return name.replace("_", " ").title()
583
584
585def _optional_string(value: Any) -> str | None:
586 if value is None:
587 return None
588 if not isinstance(value, str):
589 raise ValueError("Schema string fields must be strings")
590 text = value.strip()
591 return text or None
592
593
594def _optional_int(value: Any) -> int | None:
595 if value is None:
596 return None
597 if isinstance(value, bool):
598 raise ValueError("Schema version must be an integer")
599 try:
600 number = int(value)
601 except (TypeError, ValueError) as exc:
602 raise ValueError("Schema version must be an integer") from exc
603 if number < 1:
604 raise ValueError("Schema version must be at least 1")
605 return number
606
607
608def _json_compatible(value: Any, *, field_name: str) -> Any:
609 if value is None or isinstance(value, (str, int, float, bool)):
610 return value
611 if isinstance(value, list):
612 return [_json_compatible(item, field_name=field_name) for item in value]
613 if isinstance(value, Mapping):
614 normalized = {}
615 for key, item in value.items():
616 if not isinstance(key, str):
617 raise ValueError(f"Schema field '{field_name}' contains a non-string key")
618 normalized[key] = _json_compatible(item, field_name=field_name)
619 return normalized
620 raise ValueError(f"Schema field '{field_name}' is not JSON-compatible")