Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_records.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_records.py · 175.9 KB · 4080 lines
1"""TSV-backed collection records.
2
3Collection records are the simple data surface Scroll can point generated
4tables and forms at. Records live in
5``data/collections/{collection}/records.tsv``.
6"""
7
8from __future__ import annotations
9
10import csv
11import hashlib
12import io
13import json
14import math
15import os
16import re
17import threading
18from collections import OrderedDict
19from contextlib import contextmanager
20from datetime import date, datetime, timezone
21from pathlib import Path
22from typing import Any, Iterable, Mapping
23
24import object_collections
25import object_computed
26import object_correlation
27import object_ids
28import object_permissions
29import object_record_changes
30import object_schemas
31from object_versions import DEFAULT_DATA_DIR
32
33COLLECTIONS_DIR = "collections"
34RECORDS_FILE = "records.tsv"
35DEFAULT_RECORD_LIMIT = 100
36MAX_RECORD_LIMIT = 1000
37EXTRA_FIELD = "extra"
38
39# Maximum bytes for a single TSV field value. Python's csv module defaults
40# csv.field_size_limit() to 128 KiB; a single record field can legitimately
41# exceed that (a long article body, a big template JSON, the embedded-items
42# array of plan/vocabulary/66). Left unraised, a cell over 128 KiB is a
43# silent data-loss bug: a classic-mode read raises an uncaught csv.Error,
44# and an APPEND-mode read returns an EMPTY collection (the oversize row trips
45# the torn-tail-tolerant `except csv.Error: break` in _parse_append_body,
46# discarding every row from that point on). We raise the PARSE ceiling here
47# and enforce the SAME ceiling on WRITE (_check_field_sizes, called from
48# create/update) so no row is ever persisted that cannot be read back. 16 MiB
49# matches the platform's max request body -- a field cannot exceed the
50# request that wrote it. Per-surface limits (e.g. the 256 KiB line-items cap)
51# are stricter and enforced above this hard floor.
52MAX_TSV_FIELD_BYTES = 16 * 1024 * 1024
53csv.field_size_limit(MAX_TSV_FIELD_BYTES)
54
55_RECORD_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$")
56
57# Append-only storage (docs/append-only-storage-design.md). `OP_FIELD` is
58# an INTERNAL, file-layer-only column: when present, it is always the
59# first column of the physical header, and it never appears in a schema,
60# an API response, or a record returned by any public function in this
61# module -- it is stripped during parse-fold and only ever written by the
62# append/rewrite helpers below. `OP_UPSERT` ("") marks a row that sets/
63# replaces a record; `OP_DELETE` ("del") marks a tombstone.
64OP_FIELD = "_op"
65OP_UPSERT = ""
66OP_DELETE = "del"
67
68# Threshold (physical row count) above which a collection becomes eligible
69# for auto-compaction: see _maybe_flag_auto_compact.
70APPEND_COMPACT_MIN_ROWS_ENV = "DBBASIC_APPEND_COMPACT_MIN_ROWS"
71_DEFAULT_APPEND_COMPACT_MIN_ROWS = 10_000
72
73# id -> byte-offset sidecar (docs/append-only-storage-design.md item 4).
74# Lives next to records.tsv as a dotfile so it is invisible to every glob
75# this codebase uses to enumerate a collection's real content
76# (iter_record_collections' `*/records.tsv` glob, object_collections' same
77# pattern) and to backup (object_backup._should_skip drops any path with a
78# dotfile part) -- both were checked, not just assumed. Format (text,
79# greppable, torn-tail-tolerant like the data file itself):
80# line 1 (header): "oidx1\t<data_ino>" -- the records.tsv inode this
81# sidecar describes. No byte count lives in the header (a header
82# rewrite-in-place would itself need the same torn-tail care as a data
83# row, for no real benefit): "how much of the file is indexed" is
84# instead DERIVED from the body, as the row_end_offset of the last
85# complete data line -- see _load_oidx_body.
86# subsequent lines: "<row_start_offset>\t<row_end_offset>\t<op>\t<id>",
87# one per indexed PHYSICAL row of records.tsv (op "" or "del", exactly
88# like OP_UPSERT/OP_DELETE). No CSV quoting is used or needed here:
89# unlike a record's own field values, `id` is restricted to
90# _RECORD_ID_RE (no tabs/newlines possible) and the offsets/op are
91# plain digits/literals, so a bare tab-split is always unambiguous.
92# The sidecar is 100% disposable and NEVER required for correctness: any
93# reader unable to make sense of it (missing, corrupt, inode mismatch,
94# torn tail beyond what a catch-up scan can resolve) rebuilds it with one
95# sequential scan of records.tsv rather than raising -- see _load_oidx.
96OIDX_FILE = ".records.oidx"
97OIDX_HEADER_TAG = "oidx1"
98
99OIDX_CACHE_MAX_ENTRIES_ENV = "DBBASIC_OIDX_CACHE_MAX_ENTRIES"
100_DEFAULT_OIDX_CACHE_MAX_ENTRIES = 2
101
102# Env knobs for the records cache below. Read at call time (not cached in a
103# module global) via _records_cache_max_entries()/_records_cache_max_rows()
104# so tests can monkeypatch os.environ and see the change take effect on the
105# next cache operation without reimporting the module.
106RECORDS_CACHE_MAX_ENTRIES_ENV = "DBBASIC_RECORDS_CACHE_MAX_ENTRIES"
107RECORDS_CACHE_MAX_ROWS_ENV = "DBBASIC_RECORDS_CACHE_MAX_ROWS"
108_DEFAULT_RECORDS_CACHE_MAX_ENTRIES = 64
109_DEFAULT_RECORDS_CACHE_MAX_ROWS = 500_000
110
111# Module-level cache for parsed records.tsv files, keyed by the resolved
112# file path. An OrderedDict bounded to DBBASIC_RECORDS_CACHE_MAX_ENTRIES
113# entries (default 64): every cache hit calls move_to_end() so the dict
114# stays ordered oldest-to-newest by use, and storing a new entry evicts the
115# oldest (popitem(last=False)) once the dict exceeds capacity -- see
116# _store_records_cache. Without a bound, a long-running server that touches
117# many collections over its lifetime (e.g. a no-downtime game server) would
118# pin every collection it has ever parsed in RAM forever; the LRU keeps
119# only the most-recently-used entries resident.
120#
121# A second knob, DBBASIC_RECORDS_CACHE_MAX_ROWS (default 500_000), bounds
122# the cache by size rather than recency: a collection whose parse produces
123# more rows than this is still returned to its caller normally, but is
124# never stored in (or left in) the cache. Without this, reading one huge
125# collection once would occupy a full LRU slot and could evict many small,
126# hot collections just to make room for something that will likely never
127# be reused whole. _cache_entry and _refresh_records_cache both apply this
128# threshold before writing to _RECORDS_CACHE; _cache_entry additionally
129# drops any stale entry already cached for a path that has since grown
130# past the threshold, so an oversized collection never lingers there under
131# an old signature.
132#
133# Value is (stat_signature, fields, records, id_index, covered_bytes):
134# - stat_signature = (st_mtime_ns, st_size, st_ino). A CLASSIC-mode write
135# goes through an atomic tempfile-then-replace, so a new inode backs the
136# path after each write; including st_ino means a same-tick mtime
137# collision (coarse clocks/filesystems) still isn't mistaken for
138# "unchanged" as long as the replace produced a different inode, which
139# is the common case for both our own writes and any other well-behaved
140# writer using the same replace pattern.
141# APPEND-mode writes (docs/append-only-storage-design.md) are the
142# deliberate exception: a fast append opens the SAME inode in "a" mode
143# and writes one more line, so st_ino is unchanged and only st_mtime_ns
144# / st_size move. That's still sufficient to invalidate a stale cache
145# entry (the signature tuple as a whole changes), it just means "same
146# inode" is no longer proof of "unchanged content" the way it was
147# before this feature -- st_size growing is what a reader actually
148# relies on to notice a completed append. A torn tail (a final physical
149# line with no trailing newline, from a write interrupted mid-append)
150# is never cached as data: the append-mode parser drops it, and the
151# next append's self-heal (_repair_torn_tail) removes it from disk
152# before adding a new line, so it can never resurface as a stale-but-
153# plausible cached row either.
154# - fields/records mirror exactly what a fresh parse of that exact file
155# content would produce (see _refresh_records_cache: written rows are
156# projected to the full field set before caching, matching what
157# csv.DictWriter/csv.reader would round-trip). For an append-mode file
158# this means the FOLDED (last-wins-by-id, tombstones removed, `_op`
159# stripped) view -- see _parse_append_body/_fold_append_rows -- never
160# the raw physical rows; everything below this cache (get, list,
161# window-copy, extra-field surfacing) operates on folded records with
162# zero changes, exactly as it does for a classic-mode file.
163# - id_index maps record id -> index into `records`, first-occurrence-wins
164# (matching the original linear scan's behavior on hand-edited files
165# with duplicate ids). For append mode this is built from the already-
166# deduplicated folded records, so in practice every live id maps to
167# exactly one position.
168# - covered_bytes is the byte offset (from 0) that this entry's fold
169# already accounts for -- always equal to the file's size at the
170# stat this entry was built from (signature[1]), for every entry
171# produced by a full parse or a full rewrite (_cache_entry's
172# full-parse branch, _refresh_records_cache, _refresh_records_cache_
173# after_append). It exists so _cache_entry's INCREMENTAL branch
174# (docs/append-only-storage-design.md Sidecars, bullet 2: "cache
175# entries remember the byte offset they consumed; when a stat
176# signature changes by growth alone, parse only the tail delta") can
177# tell how much of a GROWN append-mode file is already folded into
178# `records`, so only the new bytes need parsing -- see
179# _try_incremental_cache_entry. A CLASSIC-mode entry carries this
180# field too (harmless: it's just the file's size at cache time,
181# unused by anything) since a classic rewrite always produces a new
182# inode, which alone rules out the incremental branch regardless of
183# covered_bytes. Only an incrementally-built entry can end up with
184# covered_bytes < signature[1] -- when the tail delta itself ends in
185# a torn (uncommitted) row, that fragment's bytes are correctly left
186# out of both `records` and covered_bytes, exactly as a full parse
187# would also drop them (_drop_torn_tail); a later append still
188# compares against this same covered_bytes to find its own delta.
189#
190# ALIASING SAFETY: the `records` list and its dicts stored here are shared,
191# mutable module state. Nothing in this module may mutate them in place.
192# Every function that hands a record or records list back across the
193# public API must copy first (see _read_collection_records,
194# get_collection_record, list_collection_records). Internal callers that
195# need to build on cached data for a write (create/update/delete) read
196# `_cache_entry`'s tuple directly -- fine as long as they only ever READ
197# from `records_ref`/`id_index_ref` (e.g. an O(1) duplicate/lookup check)
198# or capture them in a closure that builds a fresh, independent copy on
199# demand (see each function's `build_folded_records`, and _persist_write's
200# `folded_records_fn`, which calls it only on a full-rewrite path -- never
201# on the fast-append path, which is what keeps a steady-state append O(1)
202# instead of paying an eager O(n) copy on every single write).
203#
204# CONCURRENCY (readers hold no lock; writers hold _records_file_lock only
205# around their own read-modify-write): a reader's path is stat -> dict.get
206# -> compare signature -> return (falling through to parse+store on a
207# miss). A writer's path builds a complete new (signature, fields, records,
208# id_index) tuple off to the side and installs it with a single dict
209# assignment (see _store_records_cache). That assignment is one atomic
210# store under the GIL, so a concurrent reader's dict.get() always observes
211# either the entry from before the write or the entry from after it in
212# full -- never a torn mix of old fields with new records, or similar.
213#
214# The only reachable skew is an old signature serving newer content: a
215# reader stats the file (capturing the OLD signature), a writer's replace()
216# lands, and only then does the reader open+parse the file via its own
217# fresh fd. Because replace() is atomic and that fd is opened after the
218# stat, the fd sees either the pre- or post-write content in full, never a
219# mix -- so if it lands on the post-write content, what the reader parses
220# is complete and correct, it's just paired with a signature it captured a
221# moment too early. That mismatch means this parse won't be stored under a
222# key matching what's already cached, so it simply fails every future
223# signature comparison and self-heals with one extra reparse next time
224# (typically immediately, since the writer's own _refresh_records_cache
225# call installs the current signature right after the write completes).
226# The reverse -- a new signature paired with old content -- is structurally
227# impossible: content is always read at or after the stat that produced
228# its signature, never before.
229#
230# Cross-process coherence relies on every writer in this codebase using the
231# same temp-file + atomic-rename pattern, so a fresh inode backs every
232# write (st_ino is part of the signature): this module's own
233# _write_collection_records (below), object_packages.py's package-install
234# writer (~line 1022, os.replace), and object_backup.py's restore writer
235# (~line 607, Path.replace) were all checked and write via a temp path
236# followed by an atomic replace, never an in-place open("w").
237_RECORDS_CACHE: "OrderedDict[str, tuple[tuple[int, int, int], list[str], list[dict[str, str]], dict[str, int], int]]" = OrderedDict()
238
239# Observability-only counter: incremented once per successful tail-delta
240# incremental fold in _cache_entry (see _try_incremental_cache_entry).
241# Never read by any runtime code path -- it exists purely so tests can
242# assert the incremental path actually ran (rather than a full re-parse)
243# without needing to monkeypatch internals just to observe that.
244_INCREMENTAL_CACHE_FOLDS = 0
245
246# In-memory mirror of the on-disk id->offset sidecar (see OIDX_FILE above),
247# keyed by resolved records.tsv path. Value is (data_ino, covered_bytes,
248# id->row_start_offset): `covered_bytes` is how much of the file (from
249# byte 0) this dict accounts for -- see _load_oidx, the sole reader AND
250# writer of this dict. A DELIBERATELY SEPARATE, much smaller LRU than
251# _RECORDS_CACHE (default 2 entries, not 64): the two caches serve
252# opposite size regimes -- _RECORDS_CACHE holds fully-parsed records and
253# is capped by ROW COUNT (never stores anything over
254# DBBASIC_RECORDS_CACHE_MAX_ROWS), while this sidecar is only ever
255# consulted for collections ABOVE that same threshold (see
256# _fast_record_lookup: it's tried only on a _RECORDS_CACHE miss), so an
257# entry here is exactly the case _RECORDS_CACHE refuses to hold. A plain
258# {id: int} dict is far lighter per row than a full record dict, but at
259# millions of rows it is still tens of MB (roughly 80-120MB at 1M ids per
260# the design doc) -- sharing _RECORDS_CACHE's 64-entry budget would let a
261# few huge collections' sidecars alone occupy gigabytes. Bounded instead
262# by DBBASIC_OIDX_CACHE_MAX_ENTRIES (default 2): a server touching more
263# than a couple of huge append collections concurrently pays a reload
264# (still just a sidecar-body read, not a records.tsv fold) rather than
265# holding every one of them resident forever.
266_OIDX_CACHE: "OrderedDict[str, tuple[int, int, dict[str, int]]]" = OrderedDict()
267
268# Collections (by resolved records.tsv path) flagged during a parse/fold as
269# having enough superseded+deleted rows to be worth compacting -- see
270# _maybe_flag_auto_compact. Consulted (and cleared) by _persist_write on
271# that collection's NEXT write, which performs a compacting rewrite instead
272# of a plain append. Never acted on from a read path: setting membership
273# here is the only thing a read ever does, and it is not a disk write.
274_PENDING_COMPACTION: set[str] = set()
275_INTEGER_TYPES = {"int", "integer"}
276_FLOAT_TYPES = {"float", "number", "currency"}
277_BOOLEAN_TYPES = {"bool", "boolean"}
278_TRUE_VALUES = {"true", "1", "yes", "on"}
279_FALSE_VALUES = {"false", "0", "no", "off"}
280
281
282class InvalidRecordIdError(ValueError):
283 """Raised when a record id is not safe for routes or lookup."""
284
285
286class RecordNotFoundError(LookupError):
287 """Raised when a record cannot be found in a collection."""
288
289
290class DuplicateRecordIdError(ValueError):
291 """Raised when a create would reuse an existing record id."""
292
293
294class InvalidRecordPayloadError(ValueError):
295 """Raised when a record write payload is not usable."""
296
297
298class TransitionNotAllowedError(InvalidRecordPayloadError):
299 """Raised when a transition's value is valid but its guard denies the subject.
300
301 Subclasses InvalidRecordPayloadError so callers that don't distinguish
302 the two keep working (still a 400-shaped payload error); callers that
303 plumb a subject into update_collection_record (see
304 _validate_field_transitions) can catch this specifically to report a
305 permission-style 403 instead.
306 """
307
308
309class VersionConflictError(Exception):
310 """Raised when an update's ``expected_rev`` precondition does not match
311 the record's current content (63 optimistic concurrency,
312 plan/vocabulary/63-concurrency-spec.md). Deliberately NOT a subclass of
313 InvalidRecordPayloadError/ValueError so the generic write-error clauses
314 that map to 400 never swallow it -- the server maps it to a distinct
315 409, mirroring how TransitionNotAllowedError maps to 403.
316 """
317
318
319# 63: the wire name for the content fingerprint. ETag-shaped -- opaque to
320# callers, compared for equality only, never parsed, never a schema field.
321REV_FIELD = "_rev"
322
323
324def compute_record_rev(record: Mapping[str, Any]) -> str:
325 """Return an opaque content fingerprint of a record's field values.
326
327 The version signal for optimistic concurrency (63): a deterministic
328 hash of the record's current fields, computed for free from the
329 surfaced/projected row already in hand on every read and, inside the
330 write lock, from the freshly-read row at update time -- no stored
331 column, no migration, works retroactively on every record. Excludes
332 REV_FIELD itself so echoing a read's ``_rev`` back into a full-record
333 round-trip never changes the fingerprint. Sorted keys + a stable JSON
334 encoding so serialization noise (key order, whitespace) never makes two
335 logically-identical records disagree. ``default=str`` keeps mixed-type
336 surfaced values (extra-blob JSON: ints, lists) hashable deterministically.
337 """
338 payload = json.dumps(
339 {k: v for k, v in record.items() if k != REV_FIELD},
340 sort_keys=True,
341 ensure_ascii=False,
342 separators=(",", ":"),
343 default=str,
344 )
345 return hashlib.sha256(payload.encode("utf-8")).hexdigest()
346
347
348def validate_record_id(record_id: str) -> bool:
349 """Return True when a record id is route-safe."""
350 if not isinstance(record_id, str):
351 return False
352 return bool(_RECORD_ID_RE.fullmatch(record_id))
353
354
355def normalize_record_payload(payload: dict[str, Any], *, require_id: bool = False) -> dict[str, str]:
356 """Return a TSV-safe record payload with scalar values converted to strings."""
357 return _normalize_record_payload(payload, require_id=require_id)
358
359
360def list_collection_records(
361 collection: str,
362 *,
363 base_dir: Path | str = DEFAULT_DATA_DIR,
364 roots: Iterable[Path] | None = None,
365 limit: int = DEFAULT_RECORD_LIMIT,
366 offset: int = 0,
367 where: Mapping[str, Any] | None = None,
368) -> dict[str, Any]:
369 """Return a paginated record list for one collection.
370
371 Copies only the window (records[offset:offset + limit]) rather than
372 the whole collection first: callers of a paginated list only ever
373 consume a small page, and copying every record before slicing it away
374 (the straightforward implementation) dominates cost on large
375 collections for a window that is thrown away 99% copied. `total` is
376 computed from the cache's own record count, not from the copied
377 window.
378
379 `where` (plan/vocabulary/58-query-filter-spec.md) is an optional
380 normalized field filter, evaluated with `filter_records` -- see there
381 for the shape. It is purely a field filter with no permission
382 awareness of its own: callers that also need row-level permission
383 filtering (object_server's HTTP/MCP surfaces) apply that FIRST and
384 pass only the already-permitted records through the same evaluator
385 (`filter_records`), so a field filter can only narrow what a caller
386 already decided is readable, never widen it -- this function's own
387 `where` follows the identical narrow-only contract, just without a
388 row filter of its own to narrow. When `where` is given, the windowing
389 optimization above no longer applies (a filtered window can't be
390 known without first testing every row), so the whole collection is
391 read and filtered before slicing -- same O(rows) cost as any other
392 full-collection read (see 58's Storage section).
393 """
394 _ensure_collection_known(collection, base_dir=base_dir, roots=roots)
395 _validate_page(limit=limit, offset=offset)
396
397 records_ref = _cached_records_ref(collection, base_dir=base_dir)
398 extra_names = _extra_field_names(collection, base_dir=base_dir, roots=roots)
399
400 if where:
401 matching = [
402 _surface_extra(dict(record), extra_names=extra_names)
403 for record in records_ref
404 if object_permissions.record_matches_filter(
405 record, where, object_permissions.PermissionSubject.anonymous()
406 )
407 ]
408 total = len(matching)
409 window = matching[offset:offset + limit]
410 return _build_records_payload(collection, window, total=total, limit=limit, offset=offset)
411
412 total = len(records_ref)
413 # Copy + surface only the window's records: entries in `records_ref`
414 # are the module cache's own dicts (see _RECORDS_CACHE ALIASING
415 # SAFETY above) and _surface_extra mutates its argument in place, so
416 # each must be copied before use -- but only the rows inside the
417 # window, never the rest of the collection.
418 window = [
419 _surface_extra(dict(record), extra_names=extra_names)
420 for record in records_ref[offset:offset + limit]
421 ]
422 return _build_records_payload(collection, window, total=total, limit=limit, offset=offset)
423
424
425def filter_records(
426 records: Iterable[Mapping[str, Any]],
427 where: Mapping[str, Any] | None,
428) -> list[dict[str, Any]]:
429 """Return the subset of `records` matching a normalized field filter.
430
431 Pure and in-memory: no read, no schema/permission awareness of its
432 own. Reuses `object_permissions.record_matches_filter` -- the same
433 flat-condition evaluator that backs guarded transitions and
434 permission row filters -- so there is exactly one filter evaluator in
435 the codebase (plan/vocabulary/58-query-filter-spec.md).
436
437 Callers that also apply a permission row filter MUST do so first and
438 pass only its output here: this function can only remove rows from
439 what it is given, never add any back, so "row filter's output is
440 this function's input" is what structurally guarantees a field
441 filter can narrow the readable set but never widen it (58's
442 Permissions Posture).
443 """
444 if not where:
445 return [dict(record) for record in records]
446 anonymous = object_permissions.PermissionSubject.anonymous()
447 return [
448 dict(record)
449 for record in records
450 if object_permissions.record_matches_filter(record, where, anonymous)
451 ]
452
453
454_ORDERED_FILTER_OPERATORS = frozenset({"gte", "lte", "gt", "lt"})
455_ORDERED_FILTERABLE_TYPES = _INTEGER_TYPES | _FLOAT_TYPES | {"date", "datetime", "timestamp"}
456
457
458def normalize_filter_value(field: dict[str, Any] | None, op: str, value: str) -> str:
459 """Validate one query-filter condition's value against a schema field
460 and return its canonical stored-string form.
461
462 `field` is the schema field definition (or None for a schemaless/
463 derived collection, which has no type to validate against and is
464 treated as permissive -- consistent with every other schema-optional
465 check in this module). Raises InvalidRecordPayloadError -- the same
466 exception schema validation already raises -- naming the field on a
467 bad type, or on an ordered operator (gte/lte/gt/lt) against a field
468 type that doesn't support ordered comparison (58 restricts those to
469 date/datetime/integer/number fields). Reuses `_validate_field_type`,
470 the same per-type parser create/update already validates against, so
471 there is exactly one value-type validator, not a second one for
472 filters.
473
474 A boolean field's value is normalized to "true"/"false" -- the same
475 canonical form `_canonicalize_schema_values` stores on write -- so an
476 `eq`/`ne` filter written as `?is_public=1` still string-matches a
477 stored "true" rather than silently never matching.
478 """
479 if field is None:
480 return value
481
482 name = field.get("name", "")
483 field_type = str(field.get("type") or "text").lower()
484 if op in _ORDERED_FILTER_OPERATORS and field_type not in _ORDERED_FILTERABLE_TYPES:
485 raise InvalidRecordPayloadError(
486 f"Record field '{name}' does not support operator '{op}' "
487 f"(only date, datetime, integer, and number fields support "
488 f"ordered comparison)"
489 )
490 _validate_field_type(field, value)
491 if field_type in _BOOLEAN_TYPES:
492 return "true" if _parse_boolean(value, field_name=name) else "false"
493 return value
494
495
496def read_collection_records(
497 collection: str,
498 *,
499 base_dir: Path | str = DEFAULT_DATA_DIR,
500 roots: Iterable[Path] | None = None,
501) -> list[dict[str, str]]:
502 """Return all records for one known collection."""
503 _ensure_collection_known(collection, base_dir=base_dir, roots=roots)
504 records = _read_collection_records(collection, base_dir=base_dir)
505 extra_names = _extra_field_names(collection, base_dir=base_dir, roots=roots)
506 return [_surface_extra(record, extra_names=extra_names) for record in records]
507
508
509def collection_records_payload(
510 collection: str,
511 records: list[dict[str, str]],
512 *,
513 limit: int = DEFAULT_RECORD_LIMIT,
514 offset: int = 0,
515) -> dict[str, Any]:
516 """Return the standard paginated record-list payload shape.
517
518 Takes a full records list and slices the window out of it here --
519 unlike list_collection_records, which slices before copying. This
520 public entry point's contract is "you already have the full list";
521 callers (e.g. object_server's search/permission-filtered list paths)
522 rely on that.
523 """
524 _validate_page(limit=limit, offset=offset)
525 total = len(records)
526 window = records[offset:offset + limit]
527 return _build_records_payload(collection, window, total=total, limit=limit, offset=offset)
528
529
530def _build_records_payload(
531 collection: str,
532 window: list[dict[str, str]],
533 *,
534 total: int,
535 limit: int,
536 offset: int,
537) -> dict[str, Any]:
538 return {
539 "collection": collection,
540 "records": window,
541 "count": len(window),
542 "total": total,
543 "limit": limit,
544 "offset": offset,
545 "has_more": offset + len(window) < total,
546 }
547
548
549def get_collection_record(
550 collection: str,
551 record_id: str,
552 *,
553 base_dir: Path | str = DEFAULT_DATA_DIR,
554 roots: Iterable[Path] | None = None,
555) -> dict[str, Any]:
556 """Return one record by its ``id`` column."""
557 _ensure_collection_known(collection, base_dir=base_dir, roots=roots)
558 if not validate_record_id(record_id):
559 raise InvalidRecordIdError(f"Invalid record id: {record_id}")
560
561 extra_names = _extra_field_names(collection, base_dir=base_dir, roots=roots)
562 path = collection_records_file(collection, base_dir=base_dir)
563 # _fast_record_lookup tries the warm cache, then (append mode, over
564 # the cache's row threshold) the id->offset sidecar, before falling
565 # back to a full fold -- see its docstring. `existing_row`, when
566 # returned, is always already a fresh, independent copy.
567 _, exists, existing_row, _, _ = _fast_record_lookup(
568 collection, path, record_id, need_row=True
569 )
570 if not exists:
571 raise RecordNotFoundError(f"Record not found: {collection}/{record_id}")
572
573 return _surface_extra(existing_row, extra_names=extra_names)
574
575
576def create_collection_record(
577 collection: str,
578 record: dict[str, Any],
579 *,
580 base_dir: Path | str = DEFAULT_DATA_DIR,
581 roots: Iterable[Path] | None = None,
582 actor: str | None = None,
583 preserve_read_only: bool = False,
584 allow_computed_submission: bool = False,
585 record_changes: bool = True,
586) -> dict[str, str]:
587 """Append one record to a collection TSV and return the stored row.
588
589 `record_changes=False` skips the change-log entry, and exists for
590 exactly one caller shape: a full recompute writing DERIVED rows. A
591 rollup target's history is not "who did what" -- it is the source
592 collection plus the definition, both of which are still there, so the
593 log adds nothing anyone could audit while costing one entry per row
594 per pass, forever. One such log reached 944MB describing a collection
595 of 1818 rows, and the activity feed that reads every log stopped
596 finishing. Do NOT reach for this to make an ordinary write quieter:
597 a write nobody can trace is the thing the log exists to prevent.
598
599 Every successful write is durably attributed here (universal
600 attribution -- every mutation, on every path, emits a record change).
601 ``actor`` identifies who/what caused the write; callers that omit it
602 are logged as ``"unattributed"`` rather than silently skipped, so gaps
603 stay visible instead of disappearing. See object_record_changes for
604 the append itself and its recursion note.
605
606 ``preserve_read_only``: an ordinary write (client-submitted payload)
607 may never set a ``read_only`` field (e.g. a schema's ``created_at``) --
608 the server owns it, per ``_apply_auto_created_at``. A bulk loader
609 replaying another system's history (object_import.py) legitimately
610 needs to carry that system's own ``created_at`` through instead of
611 stamping "now", which is exactly what a hand-typed form must never be
612 able to do. Setting this narrows, not removes, the read-only
613 protection: a genuinely ``computed`` field (a server-derived formula,
614 not just a field the client shouldn't touch) still rejects a
615 client-submitted value regardless of this flag -- see
616 ``_validate_record_against_schema``. Trusted, explicit, opt-in callers
617 only; never wired to an HTTP request payload.
618
619 ``allow_computed_submission`` is the narrower sibling ``preserve_read_only``
620 deliberately does not grant: a field marked ``computed`` (e.g. a rollup
621 target's metric columns, ``plan/vocabulary/14-rollup-spec.md``) is meant
622 to hold a server-derived formula's result, and no HTTP-facing caller
623 should ever be able to write one directly. The one legitimate writer is
624 the block that owns the derivation itself (the rollup daemon pass
625 writing its own computed metric columns into its own target row) --
626 trusted, explicit, opt-in, same posture as ``preserve_read_only``, never
627 wired to a request payload.
628 """
629 _ensure_collection_known(collection, base_dir=base_dir, roots=roots)
630 extra_names = _extra_field_names(collection, base_dir=base_dir, roots=roots)
631 clean = _normalize_record_payload(record, require_id=True)
632 record_id = clean["id"]
633 if not validate_record_id(record_id):
634 raise InvalidRecordIdError(f"Invalid record id: {record_id}")
635 submitted_fields = frozenset(clean)
636 clean = _apply_schema_defaults(collection, clean, base_dir=base_dir, roots=roots)
637 _check_field_storable(clean)
638 clean = _apply_auto_created_at(collection, clean, submitted_fields, base_dir=base_dir, roots=roots)
639 clean = _apply_computed_fields(collection, clean, base_dir=base_dir, roots=roots)
640 _validate_record_against_schema(
641 collection,
642 clean,
643 submitted_fields=submitted_fields,
644 base_dir=base_dir,
645 roots=roots,
646 allow_read_only_submission=preserve_read_only,
647 allow_computed_submission=allow_computed_submission,
648 )
649 clean = _canonicalize_schema_values(collection, clean, base_dir=base_dir, roots=roots)
650 clean = _route_extra(clean, existing_blob={}, extra_names=extra_names)
651
652 path = collection_records_file(collection, base_dir=base_dir)
653 with _records_file_lock(path):
654 # _fast_record_lookup resolves the duplicate check via the
655 # cheapest source available: warm cache, then (append mode, over
656 # threshold) the id->offset sidecar -- neither of which needs a
657 # full fold or a copy of every existing record. `records_ref`/
658 # `id_index_ref` are the module cache's own uncopied objects when
659 # sourced from the cache (or a forced full fold below) -- fine to
660 # read (the O(1) duplicate check already happened above) or
661 # capture in a closure that builds a fresh copy lazily, only if a
662 # full rewrite turns out to be needed (see _persist_write's
663 # folded_records_fn). Skipping an eager copy of every existing
664 # record is what keeps a fast-append create O(1) rather than
665 # O(n): an `any(row.get("id") == ... for row in records)` linear
666 # scan plus a full deep-copy of every existing record were,
667 # together, the dominant cost of a "fast" append in an earlier
668 # version of this function -- and, before the sidecar, the O(n)
669 # fold itself was the dominant cost above the cache's row
670 # threshold.
671 #
672 # In append mode, "exists" reflects the last-wins-by-id folded
673 # view either way (a deleted id is absent, so a create can reuse
674 # it -- resurrection), same as classic mode's index, which never
675 # indexes a stale/removed row.
676 fields, exists, _, records_ref, id_index_ref = _fast_record_lookup(
677 collection, path, record_id, need_row=False
678 )
679 if exists:
680 raise DuplicateRecordIdError(f"Record already exists: {collection}/{record_id}")
681
682 merged_fields = _merge_fields(fields, clean)
683
684 def build_folded_records() -> list[dict[str, str]]:
685 if records_ref is not None:
686 records = [dict(existing) for existing in records_ref]
687 else:
688 # Sidecar-sourced lookup: no full record list was ever
689 # built. Only reached on a rare full-rewrite path (new
690 # field / pending auto-compaction / mode transition), so
691 # paying one full fold here -- unavoidable, a full
692 # rewrite needs the complete record set -- doesn't
693 # regress the fast steady-state append this whole path
694 # exists to keep O(1).
695 _, full_records, _ = _cache_entry(collection, path)
696 records = [dict(existing) for existing in full_records]
697 records.append(clean)
698 return records
699
700 _persist_write(
701 collection,
702 path,
703 base_dir=base_dir,
704 roots=roots,
705 prior_fields=fields,
706 merged_fields=merged_fields,
707 folded_records_fn=build_folded_records,
708 delta_row=clean,
709 delta_op=OP_UPSERT,
710 delta_id=record_id,
711 )
712 result = _surface_extra(_project_record(clean, merged_fields), extra_names=extra_names)
713 if record_changes:
714 object_record_changes.append_record_change(
715 collection=collection,
716 record_id=record_id,
717 action="create",
718 before=None,
719 after=result,
720 actor=actor or "unattributed",
721 correlation_id=object_correlation.current_correlation_id(),
722 base_dir=base_dir,
723 )
724 _recompute_rollups_for_source(collection, [result], base_dir=base_dir, roots=roots)
725 return result
726
727
728def update_collection_record(
729 collection: str,
730 record_id: str,
731 changes: dict[str, Any],
732 *,
733 base_dir: Path | str = DEFAULT_DATA_DIR,
734 roots: Iterable[Path] | None = None,
735 actor: str | None = None,
736 transition_subject: object_permissions.PermissionSubject | None = None,
737 preserve_read_only: bool = False,
738 allow_computed_submission: bool = False,
739 expected_rev: str | None = None,
740 record_changes: bool = True,
741) -> dict[str, str]:
742 """Update one existing record by id and return the stored row.
743
744 See create_collection_record for the attribution contract: every
745 successful update is durably attributed, defaulting to
746 ``"unattributed"`` when the caller doesn't identify itself.
747
748 ``transition_subject`` is optional and only affects schema-declared
749 transition guards (see _validate_field_transitions). The HTTP update
750 path resolves the request subject once permissions have already run
751 and passes it through here; direct library callers (daemon, CLI,
752 tests) that don't pass one are trusted callers -- guarded moves are
753 still checked for validity, just not for who is making them.
754
755 ``preserve_read_only``/``allow_computed_submission`` mirror
756 create_collection_record's own flags, same "trusted, explicit, opt-in,
757 never wired to a request payload" posture -- an update needs them too,
758 e.g. the rollup daemon pass (plan/vocabulary/14-rollup-spec.md) stamping
759 a definition's read-only ``last_computed_at`` and a target row's
760 computed metric columns on every recompute.
761
762 ``expected_rev`` is 63's optimistic-concurrency precondition
763 (plan/vocabulary/63-concurrency-spec.md): when supplied, the freshly-read
764 row's compute_record_rev is compared against it INSIDE the write lock,
765 before any validation or write; a mismatch raises VersionConflictError
766 and nothing is written. ``None`` (the default) means last-write-wins,
767 exactly as before -- the precondition is opt-in per write, never
768 mandatory. Because the check runs after the lock is held and reads the
769 same post-lock row every writer serializes behind, two racing updates
770 to the same record resolve as "first wins, second sees the changed row
771 and 409s" rather than a silent clobber -- see the spec's worked example.
772 """
773 _ensure_collection_known(collection, base_dir=base_dir, roots=roots)
774 if not validate_record_id(record_id):
775 raise InvalidRecordIdError(f"Invalid record id: {record_id}")
776 extra_names = _extra_field_names(collection, base_dir=base_dir, roots=roots)
777
778 clean = _normalize_record_payload(changes, require_id=False)
779 if "id" in clean and clean["id"] != record_id:
780 raise InvalidRecordPayloadError("Record id cannot be changed")
781 submitted_fields = frozenset(clean)
782
783 path = collection_records_file(collection, base_dir=base_dir)
784 with _records_file_lock(path):
785 # See the matching comment in create_collection_record: this
786 # resolves via warm cache, then (append mode, over threshold) the
787 # id->offset sidecar, before a full fold -- see
788 # _fast_record_lookup. `existing_row` is always already a fresh,
789 # independent copy. When sourced from the cache (or a forced full
790 # fold below), id_index_ref is first-occurrence-wins
791 # (_build_id_index), same as the linear scan this replaces, so
792 # this changes nothing about WHICH row a duplicate-id file
793 # resolves to.
794 fields, exists, existing_row, records_ref, id_index_ref = _fast_record_lookup(
795 collection, path, record_id, need_row=True
796 )
797 if not exists:
798 raise RecordNotFoundError(f"Record not found: {collection}/{record_id}")
799
800 existing_blob = _parse_extra_blob(existing_row.get(EXTRA_FIELD))
801 existing = _surface_extra(existing_row, extra_names=extra_names)
802
803 # 63: the precondition, checked against the row just read under the
804 # lock (the freshest possible state -- not the caller's earlier
805 # pre-lock read, which may be stale by now). `existing` is the same
806 # surfaced representation get_collection_record returns and the read
807 # side fingerprints, so the two `_rev` values are computed over
808 # identical inputs. Runs before any validation/write: a mismatch is
809 # a clean no-op.
810 if expected_rev is not None and compute_record_rev(existing) != expected_rev:
811 raise VersionConflictError(
812 f"Record {collection}/{record_id} has changed since it was read "
813 f"(expected _rev {expected_rev!r})."
814 )
815
816 updated = dict(existing)
817 updated.update(clean)
818 updated["id"] = record_id
819 updated = _apply_computed_fields(collection, updated, base_dir=base_dir, roots=roots)
820 _validate_record_against_schema(
821 collection,
822 updated,
823 submitted_fields=submitted_fields,
824 base_dir=base_dir,
825 roots=roots,
826 allow_read_only_submission=preserve_read_only,
827 allow_computed_submission=allow_computed_submission,
828 )
829 _check_field_storable(updated)
830 _validate_field_transitions(
831 collection,
832 existing,
833 updated,
834 base_dir=base_dir,
835 roots=roots,
836 subject=transition_subject,
837 )
838 updated = _canonicalize_schema_values(
839 collection, updated, base_dir=base_dir, roots=roots
840 )
841 updated = _route_extra(updated, existing_blob=existing_blob, extra_names=extra_names)
842 merged_fields = _merge_fields(fields, updated)
843
844 def build_folded_records() -> list[dict[str, str]]:
845 if records_ref is not None:
846 records = [dict(existing_row) for existing_row in records_ref]
847 records[id_index_ref[record_id]] = updated
848 else:
849 # Sidecar-sourced lookup -- see the matching comment in
850 # create_collection_record's build_folded_records.
851 _, full_records, full_id_index = _cache_entry(collection, path)
852 records = [dict(existing_row) for existing_row in full_records]
853 records[full_id_index[record_id]] = updated
854 return records
855
856 _persist_write(
857 collection,
858 path,
859 base_dir=base_dir,
860 roots=roots,
861 prior_fields=fields,
862 merged_fields=merged_fields,
863 folded_records_fn=build_folded_records,
864 delta_row=updated,
865 delta_op=OP_UPSERT,
866 delta_id=record_id,
867 )
868 result = _surface_extra(_project_record(updated, merged_fields), extra_names=extra_names)
869 if record_changes:
870 object_record_changes.append_record_change(
871 collection=collection,
872 record_id=record_id,
873 action="update",
874 before=existing,
875 after=result,
876 actor=actor or "unattributed",
877 correlation_id=object_correlation.current_correlation_id(),
878 base_dir=base_dir,
879 )
880 _recompute_rollups_for_source(
881 collection, [existing, result], base_dir=base_dir, roots=roots
882 )
883 return result
884
885
886def delete_collection_record(
887 collection: str,
888 record_id: str,
889 *,
890 base_dir: Path | str = DEFAULT_DATA_DIR,
891 roots: Iterable[Path] | None = None,
892 actor: str | None = None,
893 record_changes: bool = True,
894) -> dict[str, str]:
895 """Delete one existing record by id and return the removed row.
896
897 See create_collection_record for the attribution contract: every
898 successful delete is durably attributed, defaulting to
899 ``"unattributed"`` when the caller doesn't identify itself.
900 """
901 _ensure_collection_known(collection, base_dir=base_dir, roots=roots)
902 if not validate_record_id(record_id):
903 raise InvalidRecordIdError(f"Invalid record id: {record_id}")
904
905 path = collection_records_file(collection, base_dir=base_dir)
906 with _records_file_lock(path):
907 # See the matching comment in create_collection_record.
908 # `existing_row` is already a fresh, independent copy -- safe to
909 # hand back across the API as-is.
910 fields, exists, existing_row, records_ref, id_index_ref = _fast_record_lookup(
911 collection, path, record_id, need_row=True
912 )
913 if not exists:
914 raise RecordNotFoundError(f"Record not found: {collection}/{record_id}")
915
916 removed = existing_row
917
918 def build_folded_records() -> list[dict[str, str]]:
919 if records_ref is not None:
920 records = [dict(existing_row) for existing_row in records_ref]
921 records.pop(id_index_ref[record_id])
922 else:
923 # Sidecar-sourced lookup -- see the matching comment in
924 # create_collection_record's build_folded_records.
925 _, full_records, full_id_index = _cache_entry(collection, path)
926 records = [dict(existing_row) for existing_row in full_records]
927 records.pop(full_id_index[record_id])
928 return records
929
930 _persist_write(
931 collection,
932 path,
933 base_dir=base_dir,
934 roots=roots,
935 prior_fields=fields,
936 merged_fields=fields,
937 folded_records_fn=build_folded_records,
938 delta_row=None,
939 delta_op=OP_DELETE,
940 delta_id=record_id,
941 )
942 result = _project_record(removed, fields)
943 if record_changes:
944 object_record_changes.append_record_change(
945 collection=collection,
946 record_id=record_id,
947 action="delete",
948 before=result,
949 after=None,
950 actor=actor or "unattributed",
951 correlation_id=object_correlation.current_correlation_id(),
952 base_dir=base_dir,
953 )
954 _recompute_rollups_for_source(collection, [result], base_dir=base_dir, roots=roots)
955 return result
956
957
958def compact_collection(
959 collection: str,
960 *,
961 base_dir: Path | str = DEFAULT_DATA_DIR,
962 roots: Iterable[Path] | None = None,
963) -> dict[str, Any]:
964 """Fold an append-mode collection's records file to its live rows only.
965
966 Rewrites atomically via the same _write_collection_records path a
967 plain write uses (temp file + replace), under the collection's file
968 lock, keeping the `_op` column (every row becomes an upsert). Callable
969 directly (this function), via POST /admin/collections/{collection}/
970 compact (object_server.py), or automatically: Auto-compaction
971 (_maybe_flag_auto_compact) causes a collection's next ordinary write
972 to perform this same rewrite instead of a plain append, and the
973 daemon's process_compactions (object_daemon.py) polls append_
974 collection_stats and compacts any collection over its bloat/row
975 thresholds on a timer (see docs/append-only-storage-design.md
976 Compaction: "run on a schedule or when the superseded-row ratio
977 passes a threshold — never inline in a request"). This function can
978 also be called directly at any time, including on a classic-mode or
979 never-written collection, where it is a correctly-reported no-op.
980
981 Returns {"rows_before", "rows_after", "bytes_before", "bytes_after"}
982 (row counts are physical rows on disk before/after; for a collection
983 not currently in append physical format, before == after).
984 """
985 _ensure_collection_known(collection, base_dir=base_dir, roots=roots)
986 path = collection_records_file(collection, base_dir=base_dir)
987
988 with _records_file_lock(path):
989 try:
990 bytes_before = path.stat().st_size
991 except OSError:
992 bytes_before = 0
993
994 physical_header = _physical_header(path)
995 if not physical_header or physical_header[0] != OP_FIELD:
996 rows_before = len(_read_collection_records(collection, base_dir=base_dir))
997 return {
998 "rows_before": rows_before,
999 "rows_after": rows_before,
1000 "bytes_before": bytes_before,
1001 "bytes_after": bytes_before,
1002 }
1003
1004 with path.open(newline="") as handle:
1005 text = handle.read()
1006 folded_records, physical_row_count = _parse_append_body(collection, text, physical_header)
1007 logical_fields = physical_header[1:]
1008
1009 _write_collection_records(
1010 collection, path, physical_header, folded_records, cache_fields=logical_fields
1011 )
1012 _PENDING_COMPACTION.discard(str(path.resolve(strict=False)))
1013 _discard_oidx(path)
1014
1015 bytes_after = path.stat().st_size
1016 return {
1017 "rows_before": physical_row_count,
1018 "rows_after": len(folded_records),
1019 "bytes_before": bytes_before,
1020 "bytes_after": bytes_after,
1021 }
1022
1023
1024def prune_collection_records(
1025 collection: str,
1026 *,
1027 keep_newer_than: str,
1028 timestamp_field: str = "created_at",
1029 keep_last: int | None = None,
1030 base_dir: Path | str = DEFAULT_DATA_DIR,
1031 roots: Iterable[Path] | None = None,
1032) -> dict[str, Any]:
1033 """Retention / rotation: drop rows whose ``timestamp_field`` is strictly
1034 older than ``keep_newer_than`` (an ISO-8601 string), rewriting the file to
1035 only the kept rows in ONE atomic pass (temp + replace, under the file lock)
1036 -- the same rewrite path compaction uses.
1037
1038 This is the tool for high-volume append-mode event logs (page_views) where
1039 id-fold compaction cannot help: every row is a distinct event, nothing is
1040 superseded, so the only way to bound the file is to age rows out. A row with
1041 a missing or unparseable timestamp is KEPT (never delete data we can't
1042 date). Only meaningful for append-format collections; a classic-mode or
1043 never-written collection is a correctly-reported no-op.
1044
1045 ``keep_last`` additionally caps the row COUNT, keeping the newest N after
1046 the time filter. Time alone does not bound a log: a retention window says
1047 how far back to look, not how much can arrive inside it, so a traffic
1048 spike (or a bot storm) can fill a disk without aging a single row out.
1049 Small machines need the second bound, and a bound nobody set is a bound
1050 that fails on the worst day rather than an ordinary one.
1051
1052 Returns ``{"rows_before", "rows_after", "removed", "pruned"}``.
1053 """
1054 _ensure_collection_known(collection, base_dir=base_dir, roots=roots)
1055 path = collection_records_file(collection, base_dir=base_dir)
1056
1057 with _records_file_lock(path):
1058 physical_header = _physical_header(path)
1059 if not physical_header or physical_header[0] != OP_FIELD:
1060 live = len(_read_collection_records(collection, base_dir=base_dir))
1061 return {"rows_before": live, "rows_after": live, "removed": 0, "pruned": False}
1062
1063 with path.open(newline="") as handle:
1064 text = handle.read()
1065 folded_records, _physical_row_count = _parse_append_body(collection, text, physical_header)
1066 logical_fields = physical_header[1:]
1067
1068 cutoff = str(keep_newer_than)
1069
1070 def _keep(record: dict[str, str]) -> bool:
1071 stamp = str(record.get(timestamp_field) or "")
1072 return True if not stamp else stamp >= cutoff
1073
1074 kept = [record for record in folded_records if _keep(record)]
1075 if keep_last is not None and keep_last >= 0 and len(kept) > keep_last:
1076 # Newest N. The append log is written in arrival order, so the tail
1077 # IS the newest -- no sort, which matters because this runs against
1078 # the largest file the server owns.
1079 kept = kept[len(kept) - keep_last:]
1080 removed = len(folded_records) - len(kept)
1081 if removed == 0:
1082 return {
1083 "rows_before": len(folded_records),
1084 "rows_after": len(folded_records),
1085 "removed": 0,
1086 "pruned": False,
1087 }
1088
1089 _write_collection_records(
1090 collection, path, physical_header, kept, cache_fields=logical_fields
1091 )
1092 _PENDING_COMPACTION.discard(str(path.resolve(strict=False)))
1093 _discard_oidx(path)
1094 return {
1095 "rows_before": len(folded_records),
1096 "rows_after": len(kept),
1097 "removed": removed,
1098 "pruned": True,
1099 }
1100
1101
1102def append_collection_stats(
1103 collection: str,
1104 *,
1105 base_dir: Path | str = DEFAULT_DATA_DIR,
1106 roots: Iterable[Path] | None = None,
1107 allow_fold: bool = True,
1108) -> dict[str, Any] | None:
1109 """Return compaction-observability stats for one collection, or None
1110 when it is not currently in append storage mode (docs/storage-modes.md
1111 "Compaction").
1112
1113 Returns:
1114 {"collection", "storage": "append", "physical_rows", "live_rows",
1115 "dead_rows", "bloat_ratio" (dead_rows / max(live_rows, 1), rounded
1116 3dp), "file_bytes", "sidecar_present", "compaction_flagged"}
1117
1118 Cheapest-source-first, same doctrine as _fast_record_lookup's point-op
1119 resolution (docs/append-only-storage-design.md Sidecars), tried in
1120 order and combined -- either can answer `live_rows`, but only the
1121 sidecar can answer `physical_rows` short of a fold:
1122
1123 1. A warm _RECORDS_CACHE hit (_peek_records_cache) answers
1124 `live_rows` for free -- it's already the folded record count, no
1125 I/O at all.
1126 2. The id->offset sidecar (.records.oidx), read directly via
1127 _load_oidx_body -- a "pure line-count read" as opposed to a
1128 records.tsv parse: one line per physical row was written when
1129 that row was appended (OIDX_FILE's format), so the sidecar's own
1130 body length IS `physical_rows` without touching records.tsv at
1131 all, and the same read's last-wins fold gives `live_rows` too
1132 (used only when tier 1 didn't already answer it). Deliberately
1133 narrower than the hot-path loader _load_oidx: this never triggers
1134 a catch-up scan or a from-scratch rebuild when the sidecar is
1135 stale or missing (`covered_bytes` short of the file's current
1136 size, wrong header inode, or absent) -- an observability call
1137 should never be the reason records.tsv gets scanned; it just
1138 doesn't answer from this tier, and falls through to tier 3 (or
1139 the estimated shortcut below) instead.
1140 3. Neither tier answered both numbers: a full fold via
1141 _parse_append_body -- the same O(file) cost compact_collection
1142 itself pays to read the file (docs/storage-modes.md: "cold reads
1143 of an append file cost ~1.6x a classic parse"). This is the ONLY
1144 expensive path in this function, and only runs when `allow_fold`
1145 is True (the default for direct/CLI callers, and for the daemon's
1146 polling pass, where an occasional full fold is an acceptable
1147 cost against an interval timer). Callers that must never pay it
1148 -- GET /admin/storage (object_server.py), which must stay O(1)
1149 regardless of any one collection's cache/sidecar state -- pass
1150 allow_fold=False and get an {"estimated": True, "physical_rows":
1151 None, "live_rows": None, "dead_rows": None, "bloat_ratio": None}
1152 entry instead (still with a real, cheap `file_bytes` from the
1153 initial stat).
1154
1155 A collection whose records.tsv doesn't exist yet (schema declared
1156 "append" but never written) short-circuits to an honest all-zero
1157 result before any of the above -- that check (`path.exists()`, done as
1158 part of the initial stat) is itself O(1), so it's never worth
1159 representing as "estimated".
1160 """
1161 _ensure_collection_known(collection, base_dir=base_dir, roots=roots)
1162 mode = _collection_storage_mode(collection, base_dir=base_dir, roots=roots)
1163 if mode != object_schemas.STORAGE_APPEND:
1164 return None
1165
1166 path = collection_records_file(collection, base_dir=base_dir)
1167 try:
1168 st = path.stat()
1169 file_bytes = st.st_size
1170 data_ino: int | None = st.st_ino
1171 except OSError:
1172 file_bytes = 0
1173 data_ino = None
1174
1175 cache_key = str(path.resolve(strict=False))
1176 sidecar_path = _oidx_path(path)
1177 sidecar_present = sidecar_path.exists()
1178 compaction_flagged = cache_key in _PENDING_COMPACTION
1179
1180 if data_ino is None:
1181 # No records.tsv at all -- a schema can declare "append" storage
1182 # before its collection is ever written. Nothing to fold, sidecar
1183 # or otherwise; report honest zeros rather than routing this
1184 # through the fold/estimate machinery below.
1185 return {
1186 "collection": collection,
1187 "storage": object_schemas.STORAGE_APPEND,
1188 "physical_rows": 0,
1189 "live_rows": 0,
1190 "dead_rows": 0,
1191 "bloat_ratio": 0.0,
1192 "file_bytes": 0,
1193 "sidecar_present": sidecar_present,
1194 "compaction_flagged": compaction_flagged,
1195 }
1196
1197 live_rows: int | None = None
1198 physical_rows: int | None = None
1199
1200 peeked = _peek_records_cache(path)
1201 if peeked is not None:
1202 live_rows = len(peeked[1])
1203
1204 if sidecar_present:
1205 header_ino = _read_oidx_header(sidecar_path)
1206 if header_ino == data_ino:
1207 loaded = _load_oidx_body(sidecar_path)
1208 if loaded is not None:
1209 body_dict, body_covered, body_physical = loaded
1210 covered = (
1211 body_covered if body_covered is not None else _records_header_length(path)
1212 )
1213 if covered == file_bytes:
1214 physical_rows = body_physical
1215 if live_rows is None:
1216 live_rows = len(body_dict)
1217
1218 if physical_rows is None or live_rows is None:
1219 if not allow_fold:
1220 return {
1221 "collection": collection,
1222 "storage": object_schemas.STORAGE_APPEND,
1223 "physical_rows": None,
1224 "live_rows": None,
1225 "dead_rows": None,
1226 "bloat_ratio": None,
1227 "file_bytes": file_bytes,
1228 "sidecar_present": sidecar_present,
1229 "compaction_flagged": compaction_flagged,
1230 "estimated": True,
1231 }
1232
1233 physical_header = _physical_header(path)
1234 if not physical_header or physical_header[0] != OP_FIELD:
1235 # Exists on disk but never written in append physical format
1236 # (e.g. an empty file) -- nothing to fold.
1237 physical_rows = 0
1238 live_rows = 0
1239 else:
1240 with path.open(newline="") as handle:
1241 text = handle.read()
1242 folded_records, physical_row_count = _parse_append_body(collection, text, physical_header)
1243 physical_rows = physical_row_count
1244 live_rows = len(folded_records)
1245
1246 dead_rows = max(physical_rows - live_rows, 0)
1247 bloat_ratio = round(dead_rows / max(live_rows, 1), 3)
1248 return {
1249 "collection": collection,
1250 "storage": object_schemas.STORAGE_APPEND,
1251 "physical_rows": physical_rows,
1252 "live_rows": live_rows,
1253 "dead_rows": dead_rows,
1254 "bloat_ratio": bloat_ratio,
1255 "file_bytes": file_bytes,
1256 "sidecar_present": sidecar_present,
1257 "compaction_flagged": compaction_flagged,
1258 }
1259
1260
1261def list_append_collection_stats(
1262 *,
1263 base_dir: Path | str = DEFAULT_DATA_DIR,
1264 roots: Iterable[Path] | None = None,
1265 allow_fold: bool = True,
1266) -> list[dict[str, Any]]:
1267 """Return append_collection_stats() for every collection whose schema
1268 currently declares "storage": "append", sorted by collection name.
1269
1270 Discovery (object_schemas.list_append_storage_collections) is cheap
1271 regardless of how many collections exist on disk: it globs the schemas
1272 directory and loads each manual schema through the same mtime/size-
1273 keyed cache every other schema read in this codebase already goes
1274 through, never touching records.tsv. A collection whose schema
1275 disappears between that listing and its own stats call (a benign race
1276 under concurrent schema edits) is skipped rather than raising -- same
1277 tolerance _ensure_collection_known's callers generally apply to
1278 concurrent schema/collection changes.
1279 """
1280 names = object_schemas.list_append_storage_collections(base_dir=base_dir)
1281 stats: list[dict[str, Any]] = []
1282 for name in names:
1283 try:
1284 entry = append_collection_stats(
1285 name, base_dir=base_dir, roots=roots, allow_fold=allow_fold
1286 )
1287 except (
1288 object_collections.InvalidCollectionNameError,
1289 object_collections.CollectionNotFoundError,
1290 ):
1291 continue
1292 if entry is not None:
1293 stats.append(entry)
1294 return stats
1295
1296
1297def collection_records_file(collection: str, base_dir: Path | str = DEFAULT_DATA_DIR) -> Path:
1298 """Return the validated TSV file path for a collection."""
1299 if not object_collections.validate_collection_name(collection):
1300 raise object_collections.InvalidCollectionNameError(
1301 f"Invalid collection name: {collection}"
1302 )
1303
1304 root = Path(base_dir) / COLLECTIONS_DIR
1305 path = root / collection / RECORDS_FILE
1306 resolved_root = root.resolve(strict=False)
1307 resolved_path = path.resolve(strict=False)
1308
1309 try:
1310 resolved_path.relative_to(resolved_root)
1311 except ValueError as exc:
1312 raise object_collections.InvalidCollectionNameError(
1313 f"Collection path escapes collection directory: {collection}"
1314 ) from exc
1315
1316 return path
1317
1318
1319def collection_has_records(collection: str, base_dir: Path | str = DEFAULT_DATA_DIR) -> bool:
1320 """Return True when a collection has a records TSV file."""
1321 try:
1322 path = collection_records_file(collection, base_dir=base_dir)
1323 except object_collections.InvalidCollectionNameError:
1324 return False
1325 return path.is_file()
1326
1327
1328def iter_record_collections(base_dir: Path | str = DEFAULT_DATA_DIR) -> list[str]:
1329 """Return collection names with a ``records.tsv`` file."""
1330 root = Path(base_dir) / COLLECTIONS_DIR
1331 if not root.exists() or not root.is_dir():
1332 return []
1333
1334 names = []
1335 for path in sorted(root.glob(f"*/{RECORDS_FILE}"), key=lambda item: item.as_posix()):
1336 name = path.parent.name
1337 if object_collections.validate_collection_name(name):
1338 names.append(name)
1339 return names
1340
1341
1342def _ensure_collection_known(
1343 collection: str,
1344 *,
1345 base_dir: Path | str,
1346 roots: Iterable[Path] | None,
1347) -> None:
1348 if not object_collections.validate_collection_name(collection):
1349 raise object_collections.InvalidCollectionNameError(
1350 f"Invalid collection name: {collection}"
1351 )
1352
1353 if collection_has_records(collection, base_dir=base_dir):
1354 return
1355
1356 try:
1357 object_schemas.get_schema(collection, base_dir=base_dir, roots=roots)
1358 return
1359 except object_schemas.SchemaNotFoundError:
1360 pass
1361
1362 raise object_collections.CollectionNotFoundError(f"Collection not found: {collection}")
1363
1364
1365def _cached_records_ref(
1366 collection: str,
1367 *,
1368 base_dir: Path | str,
1369) -> list[dict[str, str]]:
1370 """Return the module cache's own records list for a collection.
1371
1372 Callers must not mutate the returned list or its dicts in place, and
1373 must copy before handing anything derived from it back across the
1374 public API -- see the _RECORDS_CACHE ALIASING SAFETY note above.
1375 """
1376 path = collection_records_file(collection, base_dir=base_dir)
1377 _, records, _ = _cache_entry(collection, path)
1378 return records
1379
1380
1381def _read_collection_records(
1382 collection: str,
1383 *,
1384 base_dir: Path | str,
1385) -> list[dict[str, str]]:
1386 records = _cached_records_ref(collection, base_dir=base_dir)
1387 # Copy: `records` is the module cache's own list of its own dicts.
1388 return [dict(record) for record in records]
1389
1390
1391def _stat_signature(path: Path) -> tuple[int, int, int] | None:
1392 try:
1393 st = path.stat()
1394 except OSError:
1395 return None
1396 return (st.st_mtime_ns, st.st_size, st.st_ino)
1397
1398
1399def _build_id_index(records: list[dict[str, str]]) -> dict[str, int]:
1400 index: dict[str, int] = {}
1401 for position, record in enumerate(records):
1402 record_id = record.get("id")
1403 if record_id and record_id not in index:
1404 index[record_id] = position
1405 return index
1406
1407
1408def _env_int(name: str, default: int) -> int:
1409 """Read an int env var, falling back to `default` when unset or unparseable."""
1410 raw = os.environ.get(name)
1411 if raw is None:
1412 return default
1413 try:
1414 return int(raw)
1415 except ValueError:
1416 return default
1417
1418
1419def _records_cache_max_entries() -> int:
1420 return _env_int(RECORDS_CACHE_MAX_ENTRIES_ENV, _DEFAULT_RECORDS_CACHE_MAX_ENTRIES)
1421
1422
1423def _records_cache_max_rows() -> int:
1424 return _env_int(RECORDS_CACHE_MAX_ROWS_ENV, _DEFAULT_RECORDS_CACHE_MAX_ROWS)
1425
1426
1427def _oidx_cache_max_entries() -> int:
1428 return _env_int(OIDX_CACHE_MAX_ENTRIES_ENV, _DEFAULT_OIDX_CACHE_MAX_ENTRIES)
1429
1430
1431def _store_records_cache(
1432 cache_key: str,
1433 signature: tuple[int, int, int],
1434 fields: list[str],
1435 records: list[dict[str, str]],
1436 id_index: dict[str, int],
1437 covered_bytes: int,
1438) -> None:
1439 """Insert/refresh one entry as most-recently-used, then evict over capacity.
1440
1441 Eviction is pure LRU by entry count: once the dict exceeds
1442 DBBASIC_RECORDS_CACHE_MAX_ENTRIES, the oldest (least-recently-used)
1443 entries are dropped first via OrderedDict.popitem(last=False).
1444
1445 `covered_bytes` is normally just `signature[1]` (every caller except
1446 _try_incremental_cache_entry's torn-tail case passes exactly that) --
1447 see the _RECORDS_CACHE block comment above for what it means and the
1448 one case where it differs.
1449 """
1450 _RECORDS_CACHE[cache_key] = (signature, fields, records, id_index, covered_bytes)
1451 _RECORDS_CACHE.move_to_end(cache_key)
1452 max_entries = _records_cache_max_entries()
1453 while len(_RECORDS_CACHE) > max_entries:
1454 _RECORDS_CACHE.popitem(last=False)
1455
1456
1457def _peek_records_cache(
1458 path: Path,
1459) -> tuple[list[str], list[dict[str, str]], dict[str, int]] | None:
1460 """Return a warm _RECORDS_CACHE hit for `path`, or None on any miss.
1461
1462 Never parses: this is the "is there already a fully-folded view of
1463 this file sitting in memory" check used to decide, in
1464 _fast_record_lookup, whether a point op can skip both a full fold AND
1465 the sidecar (cache hit -- the common case for any collection at or
1466 under DBBASIC_RECORDS_CACHE_MAX_ROWS once warm) or must try the
1467 cheaper-than-a-fold sidecar path next (a miss -- cold cache, an
1468 evicted LRU entry, or a collection that has never fit under the row
1469 threshold and so is never stored here at all; see _cache_entry, this
1470 function's parsing counterpart, which _fast_record_lookup and every
1471 non-fast-path caller fall back to when neither a cache hit nor the
1472 sidecar can answer).
1473 """
1474 cache_key = str(path.resolve(strict=False))
1475 signature = _stat_signature(path)
1476 if signature is None:
1477 return None
1478 cached = _RECORDS_CACHE.get(cache_key)
1479 if cached is not None and cached[0] == signature:
1480 _RECORDS_CACHE.move_to_end(cache_key)
1481 return cached[1], cached[2], cached[3]
1482 return None
1483
1484
1485def _cache_entry(
1486 collection: str,
1487 path: Path,
1488) -> tuple[list[str], list[dict[str, str]], dict[str, int]]:
1489 """Return (fields, records, id_index) for path's current content.
1490
1491 Serves from the module cache when the file's stat signature matches
1492 (moving the entry to most-recently-used); otherwise parses fresh and,
1493 when the file exists and its row count is within
1494 DBBASIC_RECORDS_CACHE_MAX_ROWS, stores the result for next time (a
1495 collection over that threshold is returned normally but left out of
1496 the cache -- see the _RECORDS_CACHE block comment above). The returned
1497 fields/records/id_index are the CACHED objects -- see the
1498 _RECORDS_CACHE docstring above. Callers within this module must copy
1499 before handing anything back across the public API.
1500
1501 On a peek miss, tries one more thing before paying a full parse: if a
1502 STALE entry is sitting in _RECORDS_CACHE for this exact path and the
1503 file has purely grown in place (same inode, size beyond what that
1504 entry covered) on an append-mode file, _try_incremental_cache_entry
1505 folds only the new tail bytes onto the stale entry instead -- O(new
1506 rows), not O(file). See that function for the full precondition list
1507 and fallback rules; anything it can't handle falls through to the
1508 ordinary full parse below, unchanged.
1509
1510 Absent that, this is the O(n) fold path: a full parse of the whole
1511 file whenever there's no warm cache hit. _fast_record_lookup (get/
1512 create/update/delete's shared point-op resolver) tries
1513 _peek_records_cache and then the id->offset sidecar FIRST specifically
1514 to avoid paying this on every write to a collection too large to fit
1515 in _RECORDS_CACHE; this function remains the correctness fallback for
1516 everything else (list, read_collection_records, a cold/small
1517 collection's first touch, and any point op the sidecar can't answer
1518 authoritatively).
1519 """
1520 if not path.exists():
1521 return ["id"], [], {}
1522 if not path.is_file():
1523 raise ValueError(f"Collection records path is not a file: {collection}")
1524
1525 peeked = _peek_records_cache(path)
1526 if peeked is not None:
1527 return peeked
1528
1529 cache_key = str(path.resolve(strict=False))
1530 signature = _stat_signature(path)
1531
1532 incremental = _try_incremental_cache_entry(path, cache_key, signature)
1533 if incremental is not None:
1534 return incremental
1535
1536 fields, records = _parse_records_file(collection, path)
1537 id_index = _build_id_index(records)
1538 if signature is not None:
1539 if len(records) <= _records_cache_max_rows():
1540 _store_records_cache(cache_key, signature, fields, records, id_index, signature[1])
1541 else:
1542 # Too large to cache: don't pin it, and drop any stale entry
1543 # left over from before this collection grew past the
1544 # threshold, so it doesn't linger under an old signature.
1545 _RECORDS_CACHE.pop(cache_key, None)
1546 return fields, records, id_index
1547
1548
1549def _try_incremental_cache_entry(
1550 path: Path,
1551 cache_key: str,
1552 signature: tuple[int, int, int] | None,
1553) -> tuple[list[str], list[dict[str, str]], dict[str, int]] | None:
1554 """Attempt the tail-delta incremental fold for an append-mode file that
1555 grew in place since it was last cached (docs/append-only-storage-
1556 design.md Sidecars, bullet 2: "when a stat signature changes by
1557 growth alone, parse only the tail delta"). Returns the fresh (fields,
1558 records, id_index) on success, or None when the fast path doesn't
1559 apply -- _cache_entry then falls back to its ordinary full parse,
1560 unchanged.
1561
1562 This is what makes a WARM cache stay O(delta) instead of O(file) when
1563 the growth happened outside this process's own write path (another
1564 process's writer, the CLI, a sibling pool worker, or this process's
1565 cache having gone cold and been rebuilt from a stat mismatch) -- the
1566 in-process fast-append path (_refresh_records_cache_after_append)
1567 already keeps this process's OWN writes O(1); this covers the gap
1568 where the growth was observed on a read instead.
1569
1570 Preconditions, ALL required (anything off -> None, no partial work is
1571 trusted):
1572 - a stat signature could be taken (file still exists, readable)
1573 - a PRIOR entry is warm in _RECORDS_CACHE for this exact path (a
1574 cold/evicted/never-cached collection has nothing to build on --
1575 the ordinary full parse is the only option)
1576 - same inode as the prior entry's signature -- a classic-mode
1577 rewrite, and an append-mode compaction/transition/new-field
1578 rewrite, all produce a fresh inode via temp+replace
1579 (_write_collection_records), so this alone rules out every case
1580 except an in-place append-mode growth
1581 - the new size is strictly greater than the prior entry's
1582 covered_bytes -- pure growth, nothing rewritten in place
1583 - the file's CURRENT physical header is append-format (`_op` first
1584 column) and its logical fields match the prior entry's fields
1585 exactly (a header/field change under the same inode should be
1586 impossible given the point above, but this stays defensive
1587 rather than trusting that unconditionally)
1588
1589 On success, parses only the new bytes (_append_tail_delta_records,
1590 the shared torn-tail-tolerant tail reader also used by the id->offset
1591 sidecar's catch-up scan -- see _scan_append_tail) and folds them onto
1592 the PRIOR entry's records/id_index -- never mutating the prior
1593 entry's own list/dict in place (a fresh list/dict is always built;
1594 other code in this same event-loop turn may still hold references to
1595 the prior entry's objects, same rule as every other cache-installing
1596 function in this module).
1597
1598 A tombstone (`_op == "del"`) ANYWHERE in the delta bails out (returns
1599 None, falling back to the full fold) rather than special-casing
1600 position shifts: a delete removes a slot and shifts every later
1601 position down by one (see _fold_append_rows), which this cheap
1602 replace-or-append merge does not model. Deletes are the rare case for
1603 the append-shaped workloads this exists for (game-server state,
1604 impression logs, price history), so this is an accepted, documented
1605 trade, not a gap -- correctness always wins over speed here. A
1606 blank-id row in the delta (only reachable via a hand-edited file --
1607 see _fold_append_rows) bails out for the same reason: its fold
1608 position depends on the FULL row history, which an incremental merge
1609 doesn't have.
1610
1611 The row-count cache threshold (DBBASIC_RECORDS_CACHE_MAX_ROWS) is
1612 still honored: a successful merge that grows the collection past it
1613 is still returned to the caller (correct, and far cheaper than a
1614 full parse either way), but is evicted rather than (re)cached -- the
1615 same rule _cache_entry's full-parse branch applies just above.
1616 """
1617 if signature is None:
1618 return None
1619
1620 prior = _RECORDS_CACHE.get(cache_key)
1621 if prior is None:
1622 return None
1623 old_signature, old_fields, old_records, old_id_index, old_covered = prior
1624
1625 if old_signature[2] != signature[2]:
1626 return None # different inode: not an in-place append
1627 if signature[1] <= old_covered:
1628 return None # no growth beyond what's already folded
1629
1630 physical_header = _physical_header(path)
1631 if not physical_header or physical_header[0] != OP_FIELD:
1632 return None # not (or no longer) append-physical
1633 if physical_header[1:] != old_fields:
1634 return None # defensive: header shouldn't change under the same inode
1635
1636 delta_rows, new_covered = _append_tail_delta_records(path, physical_header, old_covered)
1637 if not delta_rows:
1638 return None # nothing complete to fold yet (e.g. a torn tail only)
1639
1640 if any(
1641 row.get(OP_FIELD, OP_UPSERT) == OP_DELETE or not row.get("id")
1642 for row in delta_rows
1643 ):
1644 return None # tombstone or blank id: fall back to a full fold
1645
1646 new_records = list(old_records)
1647 new_id_index = dict(old_id_index)
1648 for row in delta_rows:
1649 record_id = row["id"]
1650 clean = {key: value for key, value in row.items() if key != OP_FIELD}
1651 idx = new_id_index.get(record_id)
1652 if idx is not None:
1653 new_records[idx] = clean
1654 else:
1655 new_id_index[record_id] = len(new_records)
1656 new_records.append(clean)
1657
1658 global _INCREMENTAL_CACHE_FOLDS
1659 _INCREMENTAL_CACHE_FOLDS += 1
1660
1661 if len(new_records) <= _records_cache_max_rows():
1662 _store_records_cache(cache_key, signature, old_fields, new_records, new_id_index, new_covered)
1663 else:
1664 _RECORDS_CACHE.pop(cache_key, None)
1665 return old_fields, new_records, new_id_index
1666
1667
1668def _parse_records_file(collection: str, path: Path) -> tuple[list[str], list[dict[str, str]]]:
1669 """Parse one records.tsv file into (fields, records).
1670
1671 Uses csv.reader rather than csv.DictReader: csv.reader is the same
1672 C-accelerated tokenizer DictReader wraps, but skips DictReader's
1673 per-row dict-construction overhead (zip + restkey/restval handling) in
1674 favor of a direct index walk here. Using csv.reader (not a naive
1675 line.split("\t")) is required for correctness, not just speed: a value
1676 written by csv.DictWriter's default QUOTE_MINIMAL may legitimately
1677 contain a quoted tab or embedded newline, and only real CSV/TSV parsing
1678 reconstructs that value correctly.
1679
1680 Dispatches on the physical header (docs/append-only-storage-design.md):
1681 a file whose first column is `_op` is in append-only physical format
1682 and is parsed+folded by _parse_append_body; every other file is parsed
1683 by the original classic-mode routine, unchanged. This dispatch is by
1684 the FILE's own header, not the collection's current schema `storage`
1685 setting -- a collection that just switched storage modes keeps reading
1686 correctly until its next write physically transitions the file (see
1687 _persist_write), and a collection that switched back to classic still
1688 reads correctly (via the append path) until its next write compacts
1689 the `_op` column away.
1690 """
1691 with path.open(newline="") as handle:
1692 reader = csv.reader(handle, delimiter="\t")
1693 try:
1694 header = next(reader)
1695 except StopIteration:
1696 header = None
1697 _validate_header(collection, header)
1698 physical_fields = list(header)
1699
1700 if physical_fields and physical_fields[0] == OP_FIELD:
1701 with path.open(newline="") as handle:
1702 text = handle.read()
1703 folded_records, physical_row_count = _parse_append_body(collection, text, physical_fields)
1704 _maybe_flag_auto_compact(
1705 path, physical_row_count=physical_row_count, live_row_count=len(folded_records)
1706 )
1707 return physical_fields[1:], folded_records
1708
1709 return physical_fields, _parse_classic_records(collection, path, physical_fields)
1710
1711
1712def _parse_classic_records(
1713 collection: str, path: Path, fields: list[str]
1714) -> list[dict[str, str]]:
1715 """Parse the data rows of a classic-mode (no `_op` column) file.
1716
1717 Replicates csv.DictReader's exact semantics for short/long rows and
1718 blank-line skipping -- see the block comment above _RECORDS_CACHE and
1719 the tests in tests/test_object_records.py for the specific cases this
1720 was checked against on the pre-change code. Unchanged from before
1721 append-only storage existed: a strict parse, no torn-tail tolerance.
1722 """
1723 field_count = len(fields)
1724 with path.open(newline="") as handle:
1725 reader = csv.reader(handle, delimiter="\t")
1726 next(reader, None) # header, already parsed/validated by the caller
1727
1728 records: list[dict[str, str]] = []
1729 row_number = 1 # the header is row 1; DictReader's enumerate started data rows at 2
1730 for row in reader:
1731 # csv.DictReader's __next__ silently skips rows the underlying
1732 # csv.reader returns as [] (blank physical lines) rather than
1733 # emitting a record for them; replicate that here so row_number
1734 # (used only in the error message below) still lines up with
1735 # DictReader's numbering of *emitted* rows, not physical lines.
1736 if not row:
1737 continue
1738 row_number += 1
1739 if len(row) > field_count:
1740 # Mirrors DictReader putting overflow values under the
1741 # `None` restkey, which the pre-change code detected via
1742 # `None in row`.
1743 raise ValueError(
1744 f"Collection records file has extra fields on row {row_number}: {collection}"
1745 )
1746 # Mirrors DictReader's restval=None for a short row, which the
1747 # pre-change code then normalized to "" (`value if value is not
1748 # None else ""`) -- do that normalization directly here.
1749 record = {
1750 fields[i]: (row[i] if i < len(row) else "")
1751 for i in range(field_count)
1752 }
1753 records.append(record)
1754
1755 return records
1756
1757
1758def _committed_prefix_len(text: str) -> int:
1759 """Char length of the prefix of `text` ending at the last CSV ROW
1760 TERMINATOR -- a "\\n" seen OUTSIDE a quoted field.
1761
1762 QUOTE-AWARE, single O(len) forward pass tracking csv quote state: a "\\n"
1763 INSIDE a quoted field (a multi-line cell) is field content, never a row
1764 boundary. "" inside a quoted field is an escaped quote (stays quoted),
1765 matching csv QUOTE_MINIMAL/QUOTE_ALL. Everything after the last real
1766 terminator is an incomplete (torn) trailing row -- an interrupted write --
1767 and is excluded. This can only ever trim a torn TAIL, never bisect a
1768 committed row.
1769 """
1770 in_quote = False
1771 last_row_end = 0
1772 i = 0
1773 n = len(text)
1774 while i < n:
1775 c = text[i]
1776 if in_quote:
1777 if c == '"':
1778 if i + 1 < n and text[i + 1] == '"':
1779 i += 2 # escaped "" -- stays inside the quoted field
1780 continue
1781 in_quote = False
1782 i += 1
1783 else:
1784 if c == '"':
1785 in_quote = True
1786 elif c == "\n":
1787 last_row_end = i + 1
1788 i += 1
1789 return last_row_end
1790
1791
1792# Public alias. `_committed_prefix_len` is the one correct answer to "where
1793# does the last complete CSV row end", and it is needed in more than one
1794# module: object_backup_index parses records.tsv out of a backup archive on
1795# the restore path and must reach the same verdict.
1796#
1797# It is exported rather than copied because the copy is what went wrong.
1798# object_backup_index kept its own `_drop_torn_tail` with a docstring saying
1799# it "mirrors object_records._drop_torn_tail" -- and when this module's copy
1800# was upgraded to be quote-aware, the mirror was not, silently, for months.
1801# Two implementations of one invariant drift; one implementation cannot.
1802committed_prefix_len = _committed_prefix_len
1803
1804
1805def _drop_torn_tail(text: str) -> str:
1806 """Return `text` truncated to the end of its last COMPLETE csv row.
1807
1808 QUOTE-AWARE. This was previously a quote-BLIND check ("ends with \\n", else
1809 rfind the last \\n) which mistook a "\\n" INSIDE a quoted multi-line field
1810 for a row boundary -- silently resurrecting a torn multi-line row and, in
1811 append mode, cascading to swallow later writes (the torn-write durability
1812 characterization mapped this to ~97% of a multi-line row's write window).
1813 A row is committed only when a row-terminating "\\n" (outside any quoted
1814 field) follows it; anything after the last such terminator is an
1815 interrupted write and is dropped.
1816
1817 Fast path: a text containing no '"' at all has no quoted fields, so every
1818 "\\n" is a row terminator and the cheap check is exact -- this keeps
1819 collections whose cells never need quoting (no tab/quote/newline in any
1820 value) on the original O(1)/rfind path. Only text with quoting pays the
1821 O(len) quote-aware scan, and that path is already parsing the same text.
1822 """
1823 if text == "":
1824 return text
1825 if '"' not in text:
1826 if text.endswith("\n"):
1827 return text
1828 cut = text.rfind("\n")
1829 return text[: cut + 1] if cut >= 0 else ""
1830 return text[:_committed_prefix_len(text)]
1831
1832
1833def _parse_append_body(
1834 collection: str, text: str, physical_fields: list[str]
1835) -> tuple[list[dict[str, str]], int]:
1836 """Parse+fold an append-mode file's body. Returns (folded_records, physical_row_count).
1837
1838 Torn-tail tolerant: a trailing unterminated line is dropped first
1839 (_drop_torn_tail); a row that still fails to parse (e.g. a value with
1840 an embedded newline whose closing quote never arrived, so its
1841 apparent line boundary landed mid-field) stops consumption at that
1842 point rather than raising -- everything after an unparseable point is,
1843 by construction, either that same torn write or unreachable. This
1844 differs from classic mode, which keeps raising on malformed content
1845 (see _parse_classic_records): append mode's contract is specifically
1846 that in-flight writes are tolerated, not silently-corrupt files.
1847 """
1848 field_count = len(physical_fields)
1849 reader = csv.reader(io.StringIO(_drop_torn_tail(text)), delimiter="\t")
1850 next(reader, None) # header, already parsed/validated by the caller
1851
1852 physical_rows: list[dict[str, str]] = []
1853 while True:
1854 try:
1855 row = next(reader)
1856 except StopIteration:
1857 break
1858 except csv.Error:
1859 break
1860 if not row:
1861 continue
1862 if len(row) > field_count:
1863 raise ValueError(
1864 f"Collection records file has extra fields: {collection}"
1865 )
1866 record = {
1867 physical_fields[i]: (row[i] if i < len(row) else "")
1868 for i in range(field_count)
1869 }
1870 physical_rows.append(record)
1871
1872 return _fold_append_rows(physical_rows), len(physical_rows)
1873
1874
1875def _fold_append_rows(physical_rows: list[dict[str, str]]) -> list[dict[str, str]]:
1876 """Fold physical append-mode rows into live records, last-wins per id.
1877
1878 An id's position in the returned list is where it was inserted since
1879 it was last live: an upsert for an id already present updates it in
1880 place (same position, matching classic mode's update-in-place); an
1881 upsert for a not-currently-live id inserts at the current end
1882 (matching classic mode's create-appends); a delete (`_op == "del"`)
1883 removes the id's slot entirely (matching classic mode's delete
1884 popping the row) -- so a later upsert for that same id is a fresh
1885 insertion at the end, not a return to its old position. Built with an
1886 OrderedDict so this falls out of normal dict semantics rather than
1887 needing to be modeled by hand; this is what gives append mode the
1888 same list order as classic mode for the same operation sequence (see
1889 the equivalence tests in tests/test_object_records.py). `_op` is
1890 stripped from every folded record.
1891
1892 Rows with a blank/missing id (only reachable via a hand-edited file --
1893 every write path in this module requires a valid id) cannot be
1894 last-wins folded against each other for lack of a key; each is kept as
1895 its own entry and appended after the id-keyed records, which does not
1896 preserve their original interleaving with id-keyed rows. This is a
1897 documented limitation of a pre-existing edge case, not a behavior this
1898 feature needs to support: normalize_record_payload never lets a create
1899 omit its id, and update always operates on an already-valid id.
1900 """
1901 folded: "OrderedDict[str, dict[str, str]]" = OrderedDict()
1902 blank_id_rows: list[dict[str, str]] = []
1903 blank_counter = 0
1904 for row in physical_rows:
1905 op = row.get(OP_FIELD, OP_UPSERT)
1906 record_id = row.get("id", "")
1907 clean = {key: value for key, value in row.items() if key != OP_FIELD}
1908 if not record_id:
1909 if op == OP_DELETE:
1910 continue
1911 blank_id_rows.append((f"__blank__{blank_counter}", clean))
1912 blank_counter += 1
1913 continue
1914 if op == OP_DELETE:
1915 folded.pop(record_id, None)
1916 continue
1917 folded[record_id] = clean
1918
1919 return list(folded.values()) + [record for _, record in blank_id_rows]
1920
1921
1922def _scan_append_tail(
1923 path: Path, physical_fields: list[str], start_byte: int
1924) -> tuple[list[tuple[int, int, dict[str, str]]], int]:
1925 """Read and parse complete physical rows of an append-mode file from
1926 `start_byte` to EOF, returning (rows, covered_bytes) where each row is
1927 (row_start_offset, row_end_offset, physical_record) -- `physical_record`
1928 keyed by every column in `physical_fields`, `_op` included, exactly
1929 like a row from _parse_append_body before folding. `covered_bytes` is
1930 the byte offset up to which parsing is authoritative (>= start_byte,
1931 equal to it when nothing new/complete was found). `start_byte` must
1932 land exactly on a row boundary -- every offset any caller ever hands
1933 back here (a prior scan/cache entry's covered_bytes, or the
1934 records-header length) always does, since it is always either 0 or a
1935 previous row's end.
1936
1937 Torn-tail tolerant exactly like _parse_append_body: a trailing
1938 unterminated physical line is dropped first (_drop_torn_tail), and a
1939 row that still fails to parse (or reports more fields than the header
1940 -- real corruption, not a torn write) stops consumption at that point
1941 rather than raising. NEVER raises: this is the shared low-level scan
1942 behind two callers that both need "read complete new rows appended
1943 since an earlier point, without re-reading or re-folding the whole
1944 file" -- the id->offset sidecar's incremental catch-up
1945 (_scan_append_rows_for_offsets, which projects each row down to just
1946 its (op, id) for the offset index) and the records-cache's tail-delta
1947 incremental fold (_append_tail_delta_records, which keeps every field
1948 since it must produce real records). Neither caller may raise either,
1949 so this doesn't.
1950
1951 Byte offsets are computed by decoding `path`'s bytes from `start_byte`
1952 once, then, after each row csv.reader consumes, converting the
1953 now-consumed prefix of that decoded text back to its exact byte
1954 length (`str.encode` on the newly consumed slice only, not the whole
1955 prefix each time, so total work stays O(scanned bytes) instead of
1956 O(scanned bytes squared)). `start_byte` itself is always a byte
1957 offset immediately after a "\\n", which is a single ASCII byte and
1958 therefore never a UTF-8 continuation byte -- so slicing raw bytes from
1959 there and decoding is always safe.
1960 """
1961 try:
1962 with path.open("rb") as handle:
1963 handle.seek(start_byte)
1964 raw = handle.read()
1965 except OSError:
1966 return [], start_byte
1967 if not raw:
1968 return [], start_byte
1969
1970 try:
1971 text = raw.decode("utf-8")
1972 except UnicodeDecodeError:
1973 return [], start_byte
1974 text = _drop_torn_tail(text)
1975 if not text:
1976 return [], start_byte
1977
1978 field_count = len(physical_fields)
1979 stream = io.StringIO(text)
1980 reader = csv.reader(stream, delimiter="\t")
1981 rows: list[tuple[int, int, dict[str, str]]] = []
1982 prev_char = 0
1983 byte_cursor = start_byte
1984 while True:
1985 try:
1986 row = next(reader)
1987 except StopIteration:
1988 break
1989 except csv.Error:
1990 break
1991 cur_char = stream.tell()
1992 row_bytes = len(text[prev_char:cur_char].encode("utf-8"))
1993 prev_char = cur_char
1994 row_start = byte_cursor
1995 row_end = byte_cursor + row_bytes
1996 byte_cursor = row_end
1997 if not row:
1998 continue
1999 if len(row) > field_count:
2000 break
2001 record = {
2002 physical_fields[i]: (row[i] if i < len(row) else "")
2003 for i in range(field_count)
2004 }
2005 rows.append((row_start, row_end, record))
2006
2007 return rows, byte_cursor
2008
2009
2010def _append_tail_delta_records(
2011 path: Path, physical_fields: list[str], start_byte: int
2012) -> tuple[list[dict[str, str]], int]:
2013 """Parse complete physical rows appended to an append-mode file since
2014 `start_byte`, returning (physical_rows, covered_bytes) -- a thin
2015 projection of _scan_append_tail down to just the row dicts (`_op`
2016 included, unfolded), for _try_incremental_cache_entry's tail-delta
2017 fold. See _scan_append_tail for the shared torn-tail-tolerant scan
2018 this builds on: never raises, stops at the first row it can't make
2019 sense of.
2020 """
2021 scanned, covered_bytes = _scan_append_tail(path, physical_fields, start_byte)
2022 return [record for _, _, record in scanned], covered_bytes
2023
2024
2025def _append_compact_min_rows() -> int:
2026 return _env_int(APPEND_COMPACT_MIN_ROWS_ENV, _DEFAULT_APPEND_COMPACT_MIN_ROWS)
2027
2028
2029def append_compact_min_rows() -> int:
2030 """Public read of DBBASIC_APPEND_COMPACT_MIN_ROWS (default 10_000).
2031
2032 Exists so callers outside this module (object_daemon.py's
2033 process_compactions) can apply the exact same physical-row-count floor
2034 _maybe_flag_auto_compact uses for its own auto-compaction trigger,
2035 without duplicating the env var's default value.
2036 """
2037 return _append_compact_min_rows()
2038
2039
2040def _maybe_flag_auto_compact(path: Path, *, physical_row_count: int, live_row_count: int) -> None:
2041 """Flag a collection for compaction on its NEXT WRITE.
2042
2043 Never compacts here -- this runs from the read/parse path, and reads
2044 must stay read-only (docs/append-only-storage-design.md Compaction).
2045 Triggers when the physical file is at least DBBASIC_APPEND_COMPACT_
2046 MIN_ROWS rows (so a small collection's ordinary churn never matters)
2047 AND superseded-or-deleted rows outnumber live rows.
2048 """
2049 if physical_row_count <= _append_compact_min_rows():
2050 return
2051 stale_row_count = physical_row_count - live_row_count
2052 if stale_row_count > live_row_count:
2053 _PENDING_COMPACTION.add(str(path.resolve(strict=False)))
2054
2055
2056def _physical_header(path: Path) -> list[str] | None:
2057 """Return the raw on-disk header (leading `_op` included, if present).
2058
2059 None when the file doesn't exist or is empty. Cheap: reads only the
2060 first physical line, not the whole file.
2061 """
2062 if not path.exists():
2063 return None
2064 with path.open(newline="") as handle:
2065 reader = csv.reader(handle, delimiter="\t")
2066 try:
2067 return next(reader)
2068 except StopIteration:
2069 return None
2070
2071
2072def _write_collection_records(
2073 collection: str,
2074 path: Path,
2075 fields: list[str],
2076 records: list[dict[str, str]],
2077 *,
2078 cache_fields: list[str] | None = None,
2079) -> None:
2080 """Rewrite the whole records file atomically (temp file + replace).
2081
2082 `fields` is the PHYSICAL header to write -- for an append-mode full
2083 rewrite (transition-in, compaction, or the new-field-fallback rewrite;
2084 see _persist_write) this includes a leading `_op` column, which every
2085 record in `records` implicitly projects to "" (upsert) since none of
2086 them carry an `_op` key (docs/append-only-storage-design.md).
2087 `cache_fields`, when given, is the LOGICAL field list (without `_op`)
2088 to store in the module cache instead of `fields` -- callers writing an
2089 append-format file must pass this, or `_op` would leak into the cache
2090 and, from there, into API responses (get/list/create/update all
2091 return cache-derived data). Classic-mode callers omit it and the
2092 written `fields` doubles as the cache's fields, exactly as before this
2093 parameter existed.
2094 """
2095 if "id" not in fields:
2096 fields = ["id", *fields]
2097
2098 path.parent.mkdir(parents=True, exist_ok=True)
2099 temp_path = path.with_name(f".{path.name}.{os.getpid()}.{threading.get_ident()}.tmp")
2100 try:
2101 with temp_path.open("w", newline="") as handle:
2102 writer = csv.DictWriter(
2103 handle,
2104 fieldnames=fields,
2105 delimiter="\t",
2106 lineterminator="\n",
2107 extrasaction="ignore",
2108 quoting=(csv.QUOTE_ALL if _rows_need_full_quoting(records) else csv.QUOTE_MINIMAL),
2109 )
2110 writer.writeheader()
2111 for record in records:
2112 writer.writerow(_project_record(record, fields))
2113 temp_path.replace(path)
2114 finally:
2115 if temp_path.exists():
2116 temp_path.unlink()
2117
2118 _refresh_records_cache(collection, path, cache_fields if cache_fields is not None else fields, records)
2119
2120
2121def _refresh_records_cache(
2122 collection: str,
2123 path: Path,
2124 fields: list[str],
2125 records: list[dict[str, str]],
2126) -> None:
2127 """Populate the module cache with the content just written to `path`.
2128
2129 Called immediately after the atomic replace above, under the caller's
2130 _records_file_lock, so this process's view of the file is authoritative
2131 right now. Storing the new content (rather than merely invalidating)
2132 means the next read for this collection -- often the same request,
2133 returning its own write -- skips re-parsing the file it just wrote.
2134
2135 If the post-write stat can't be trusted (the file vanished, or some
2136 other OSError), skip caching entirely rather than caching something
2137 stale or guessed: the next reader will just parse fresh, same as
2138 before this change existed.
2139
2140 Honors the same DBBASIC_RECORDS_CACHE_MAX_ROWS threshold as
2141 _cache_entry: a write that grows a collection past the threshold is
2142 not cached (and any stale smaller-collection entry under this key is
2143 dropped), so a single giant write can't occupy an LRU slot.
2144 """
2145 signature = _stat_signature(path)
2146 if signature is None:
2147 return
2148
2149 cache_key = str(path.resolve(strict=False))
2150 if len(records) > _records_cache_max_rows():
2151 _RECORDS_CACHE.pop(cache_key, None)
2152 return
2153
2154 # Project every record to the full field set before caching, so a
2155 # cache hit yields exactly what a fresh parse of this file would
2156 # yield (same as what was just written: `_project_record` fills any
2157 # field missing from a given record with "", matching how
2158 # csv.DictWriter/extrasaction wrote that row).
2159 cached_records = [_project_record(record, fields) for record in records]
2160 id_index = _build_id_index(cached_records)
2161 _store_records_cache(cache_key, signature, list(fields), cached_records, id_index, signature[1])
2162
2163
2164def _refresh_records_cache_after_append(
2165 collection: str,
2166 path: Path,
2167 fields: list[str],
2168 delta_op: str,
2169 delta_id: str,
2170 projected_row: dict[str, str],
2171) -> bool:
2172 """Incrementally update the cache after a fast append. Returns True if
2173 it did (the caller then skips the full _refresh_records_cache rebuild).
2174
2175 The entire point of the fast-append path is O(1) disk work -- calling
2176 the general _refresh_records_cache after every append would silently
2177 reintroduce an O(n) cost per write, just moved from disk I/O to cache
2178 bookkeeping (a full re-projection of every record plus a full
2179 id_index rebuild). This instead derives the new cache entry from the
2180 PRIOR entry plus this one delta:
2181
2182 - create (an id not already in the index): `old_records + [row]`
2183 and `dict(old_id_index)` plus one new key -- still technically
2184 O(n) (a fresh list/dict must be built; nothing cached is ever
2185 mutated in place, see the ALIASING SAFETY note on
2186 _RECORDS_CACHE), but as bulk C-level copies (list concat, dict
2187 copy) rather than a Python-level per-record rebuild.
2188 - update (an id already in the index -- always at the SAME
2189 position, since _fold_append_rows updates in place): `list(
2190 old_records)` with one element replaced; the id_index is REUSED
2191 UNCHANGED (no copy at all), since no id's position moved.
2192 - delete: a genuine O(n) Python-level index rebuild (removing a
2193 position means every later position shifts down by one) -- the
2194 one case this doesn't make cheap. Deletes are the rare case for
2195 the append-shaped workloads this feature targets (game-server
2196 state, impression logs, price history), so this is an accepted,
2197 documented trade rather than a gap.
2198
2199 Returns False when there is no usable prior entry to build on -- cold
2200 cache, evicted, over the row-count cache threshold, or (defensively)
2201 a fields mismatch. The caller (_persist_write's fast-append branch)
2202 does NOT force a full rebuild in that case (an earlier version of
2203 this code did, via folded_records_fn() + _refresh_records_cache --
2204 exactly the O(n)-per-write cost the id->offset sidecar, docs/append-
2205 only-storage-design.md item 4, exists to eliminate for a collection
2206 too large to be a "usable prior entry" in the first place). A False
2207 return simply leaves _RECORDS_CACHE as it was: the next reader either
2208 hits the sidecar (if the file is append-physical and over threshold)
2209 or pays one full fold on a genuine cold/small-collection miss, same
2210 as any other cache miss elsewhere in this module.
2211 """
2212 signature = _stat_signature(path)
2213 if signature is None:
2214 return True # nothing to cache either way; nothing left to do
2215
2216 cache_key = str(path.resolve(strict=False))
2217 prior = _RECORDS_CACHE.get(cache_key)
2218 if prior is None or prior[1] != fields:
2219 return False
2220
2221 _, _prior_fields, old_records, old_id_index, _prior_covered = prior
2222 if len(old_records) > _records_cache_max_rows():
2223 _RECORDS_CACHE.pop(cache_key, None)
2224 return True
2225
2226 if delta_op == OP_DELETE:
2227 idx = old_id_index.get(delta_id)
2228 if idx is None:
2229 return False
2230 new_records = old_records[:idx] + old_records[idx + 1:]
2231 new_id_index = {
2232 key: (value if value < idx else value - 1)
2233 for key, value in old_id_index.items()
2234 if key != delta_id
2235 }
2236 else:
2237 idx = old_id_index.get(delta_id)
2238 if idx is not None:
2239 new_records = list(old_records)
2240 new_records[idx] = projected_row
2241 new_id_index = old_id_index
2242 else:
2243 new_records = old_records + [projected_row]
2244 new_id_index = dict(old_id_index)
2245 new_id_index[delta_id] = len(old_records)
2246
2247 if len(new_records) > _records_cache_max_rows():
2248 _RECORDS_CACHE.pop(cache_key, None)
2249 return True
2250
2251 # This process just performed the append that produced `signature`, so
2252 # it authoritatively covers the whole current file, same as any other
2253 # fully-covering entry (see the covered_bytes bullet in the
2254 # _RECORDS_CACHE block comment above).
2255 _store_records_cache(cache_key, signature, list(fields), new_records, new_id_index, signature[1])
2256 return True
2257
2258
2259def _repair_torn_tail_lines(path: Path) -> None:
2260 """Trim a torn trailing LINE from a line-oriented file (the sidecar).
2261
2262 Backward rfind(b"\n") -- quote-BLIND, and correct HERE, for exactly one
2263 reason: an idx line can never contain a literal tab or newline inside a
2264 field (ids are restricted to _RECORD_ID_RE; offsets and op are plain
2265 digits/literals), so "the file ends with \n" really is a complete
2266 completeness signal for the sidecar.
2267
2268 It is NOT correct for records.tsv, whose quoted cells may hold
2269 newlines. That distinction is why this is now a separate function: the
2270 data file's repair below is quote-aware and costs a scan, and pointing
2271 the sidecar at it would spend that scan on every append to buy nothing.
2272 Do not "unify" these; they were one function once, and the shared body
2273 was wrong for one caller and slow for the other.
2274 """
2275 try:
2276 size = path.stat().st_size
2277 except OSError:
2278 return
2279 if size == 0:
2280 return
2281
2282 with path.open("rb+") as handle:
2283 handle.seek(size - 1)
2284 if handle.read(1) == b"\n":
2285 return
2286
2287 window = 8192
2288 read_from = size
2289 while read_from > 0:
2290 read_from = max(0, read_from - window)
2291 handle.seek(read_from)
2292 chunk = handle.read(size - read_from)
2293 idx = chunk.rfind(b"\n")
2294 if idx >= 0:
2295 handle.truncate(read_from + idx + 1)
2296 return
2297 window *= 2
2298 handle.truncate(0)
2299
2300
2301def _repair_torn_tail(path: Path) -> None:
2302 """Ensure records.tsv ends at a committed CSV row boundary before an
2303 append lands. QUOTE-AWARE -- substrate bug #2's write side, fixed.
2304
2305 A write interrupted mid-row leaves a fragment; appending after it
2306 would concatenate the next row onto the fragment -- and if the
2307 fragment holds an unclosed quoted field, silently swallow that next
2308 row. The previous body found "the last row boundary" with
2309 rfind(b"\n"), which mistakes a newline INSIDE a quoted multi-line
2310 field for a boundary; worse, its fast path returned immediately
2311 whenever the file's last byte was "\n" -- including a newline inside
2312 an unclosed quote -- so the very case that needed repair was the case
2313 it skipped. Exposure was ~97% of a multi-line row's write window.
2314
2315 ## Why the first fix was reverted, and what this one does differently
2316
2317 The 2026-07-20 attempt gated correctness on _RECORDS_CACHE's
2318 covered_bytes. That value lives ONLY in process memory -- the on-disk
2319 sidecar header records just the inode -- so the gate read state the
2320 test harness happened to share: under pytest one interpreter runs the
2321 whole suite and the cache was warm from other tests' files; standalone
2322 it was cold every time. "Fails in pytest, passes standalone" was never
2323 a race; it was a correctness decision delegated to a cache. And a torn
2324 tail only exists after a process DIED, which is exactly when no cache
2325 exists.
2326
2327 So the rule here, stated so nobody re-learns it: **a cache may only
2328 ever SKIP this repair, never choose where it truncates.**
2329
2330 ## How it works
2331
2332 1. SKIP (sound): a warm _OIDX_CACHE entry, inode-matched, whose
2333 covered_bytes equals the file's size right now. Every covered value
2334 in that cache is a genuine committed row boundary (from
2335 _scan_append_tail, which excludes torn tails, or from this
2336 process's own completed append), so EOF landing exactly on one
2337 proves the file is clean. _RECORDS_CACHE is deliberately NOT
2338 consulted -- see the inline comment for why its covered_bytes lies
2339 about exactly the files that need repair, which is the probable
2340 mechanism of the original revert.
2341 2. ANCHOR (cheap): the same sidecar-cache covered, when it is short of
2342 EOF, is a boundary to scan forward from instead of from zero.
2343 Append-mode files only ever grow in place (any rewrite is an atomic
2344 rename and changes the inode), and a correct repair never truncates
2345 below the last committed boundary, so bytes before a recorded
2346 boundary are stable under a matching inode.
2347 3. SCAN: decode the tail with errors="surrogateescape" and take
2348 _committed_prefix_len -- THE quote-aware answer, the same single
2349 implementation the read path and object_backup_index use, because a
2350 second copy of this logic is how the backup module silently carried
2351 the bug for months. The char prefix is re-encoded to recover the
2352 exact byte length (surrogateescape round-trips arbitrary bytes).
2353 4. Truncate to the last committed boundary found. With no anchor this
2354 is a full O(n) scan -- paid on the first append after a restart,
2355 which is the one moment a torn tail can actually exist.
2356
2357 Deliberately NOT trusted: the on-disk sidecar's row_end offsets. They
2358 would be a durable anchor, but files damaged by the OLD repair may
2359 carry sidecar offsets that no longer sit on real boundaries, and
2360 anchoring the fix to state the bug may have corrupted would rebuild
2361 the bug. Process-local anchors are attested by this process's own
2362 reads and writes.
2363
2364 The three strict-xfail acceptance tests that held this behaviour map
2365 the trigger surface; they were rewritten into regression guards when
2366 this landed.
2367 """
2368 signature = _stat_signature(path)
2369 if signature is None:
2370 return
2371 size = signature[1]
2372 if size == 0:
2373 return
2374
2375 cache_key = str(path.resolve(strict=False))
2376 anchor = 0
2377
2378 # ONLY the sidecar cache. _RECORDS_CACHE is deliberately not consulted,
2379 # and the reason is the probable mechanism of the original revert: a
2380 # FULL parse stores covered_bytes = signature[1] -- the whole file size
2381 # -- even when that parse dropped a torn tail (the read fold "accounts
2382 # for" the torn bytes by knowing they are torn). So on precisely the
2383 # file that needs repair, a warm records-cache entry reports
2384 # covered == size and looks like proof of a clean boundary. Warm cache
2385 # -> skip -> regression; cold cache -> scan -> correct. Under pytest
2386 # the cache is warm; standalone it is cold. That is the whole
2387 # "fails in pytest, passes standalone".
2388 #
2389 # _OIDX_CACHE's covered comes from _scan_append_tail, which returns the
2390 # offset up to which parsing is AUTHORITATIVE -- torn tails excluded --
2391 # or from handle.tell() after this process's own completed append.
2392 # Every value in it is a genuine committed row boundary.
2393 oidx = _OIDX_CACHE.get(cache_key)
2394 if oidx is not None:
2395 oidx_ino, oidx_covered, _offsets = oidx
2396 if oidx_ino == signature[2] and 0 < oidx_covered <= size:
2397 if oidx_covered == size:
2398 return # EOF sits exactly on a recorded boundary
2399 anchor = oidx_covered
2400
2401 with path.open("rb+") as handle:
2402 handle.seek(anchor)
2403 data = handle.read(size - anchor)
2404 text = data.decode("utf-8", errors="surrogateescape")
2405 keep_chars = _committed_prefix_len(text)
2406 committed = anchor + len(
2407 text[:keep_chars].encode("utf-8", errors="surrogateescape")
2408 )
2409 if committed < size:
2410 handle.truncate(committed)
2411
2412
2413def _append_records_rows(
2414 path: Path,
2415 physical_fields: list[str],
2416 rows: list[dict[str, str]],
2417) -> list[tuple[int, int]]:
2418 """Append one or more pre-built physical rows to an append-format file.
2419
2420 Each row in `rows` must already carry an `_op` key (OP_UPSERT or
2421 OP_DELETE). Self-heals a torn tail first (_repair_torn_tail) so the
2422 new row(s) always land as clean, complete physical lines, then writes
2423 via csv.writer (not DictWriter -- rows are already fully projected)
2424 on a text-mode "a" handle, flushing once at the end. Same dialect as
2425 _write_collection_records: tab-delimited, "\\n" line terminator,
2426 QUOTE_MINIMAL.
2427
2428 Returns the (row_start_offset, row_end_offset) byte span written for
2429 each row, in order -- _persist_write hands these straight to the id
2430 sidecar (_update_oidx_after_append) so it never has to re-derive them
2431 with a separate stat or scan. `handle.tell()` on a text-mode stream is
2432 only reliably a byte offset when nothing has been read from it in
2433 this session (reading with a stateful decoder can return an opaque
2434 cookie instead); a pure sequential-write "a" session like this one
2435 never reads, so it's exact here -- verified empirically (UTF-8,
2436 non-ASCII content, mid-stream tell() calls all matched the file's
2437 actual byte length).
2438 """
2439 _repair_torn_tail(path)
2440 offsets: list[tuple[int, int]] = []
2441 with path.open("a", newline="") as handle:
2442 writer = csv.writer(
2443 handle, delimiter="\t", lineterminator="\n",
2444 quoting=(csv.QUOTE_ALL if _rows_need_full_quoting(rows) else csv.QUOTE_MINIMAL),
2445 )
2446 if handle.tell() == 0:
2447 # The file has no committed content -- most likely the repair
2448 # above found a TORN HEADER (intact bytes, no terminating
2449 # newline: external truncation clipping exactly the trailing
2450 # byte) and correctly truncated to zero. Appending a bare data
2451 # row here would leave a headerless file every subsequent read
2452 # rejects with an uncaught ValueError, having REPORTED SUCCESS
2453 # for this write -- the torn-header finding in
2454 # tests/test_durability_torn_write_characterization.py. The
2455 # header is fully determined by physical_fields, so restore it
2456 # rather than manufacturing a poisoned file.
2457 csv.writer(handle, delimiter="\t", lineterminator="\n").writerow(
2458 physical_fields
2459 )
2460 for row in rows:
2461 start = handle.tell()
2462 projected = _project_record(row, physical_fields)
2463 writer.writerow([projected[field] for field in physical_fields])
2464 offsets.append((start, handle.tell()))
2465 handle.flush()
2466 return offsets
2467
2468
2469# ---------------------------------------------------------------------------
2470# id -> byte-offset sidecar (docs/append-only-storage-design.md item 4).
2471#
2472# SCOPE: every function below is reached only from append-physical code
2473# paths -- _fast_record_lookup only looks past a _peek_records_cache miss
2474# into the sidecar when `_physical_header(path)[0] == OP_FIELD`, and
2475# _persist_write/_write_collection_records only call _update_oidx_after_
2476# append/_discard_oidx on the append-mode branches. A collection that has
2477# never been append-mode (no schema ever set "storage": "append") never
2478# executes a line past that header check: its _physical_header is a plain
2479# ["id", ...] with no leading `_op`, so the guard is false every time.
2480# ---------------------------------------------------------------------------
2481
2482
2483def _oidx_path(records_path: Path) -> Path:
2484 return records_path.with_name(OIDX_FILE)
2485
2486
2487def _store_oidx_cache(
2488 cache_key: str,
2489 data_ino: int,
2490 covered_bytes: int,
2491 id_offsets: dict[str, int],
2492) -> None:
2493 _OIDX_CACHE[cache_key] = (data_ino, covered_bytes, id_offsets)
2494 _OIDX_CACHE.move_to_end(cache_key)
2495 max_entries = _oidx_cache_max_entries()
2496 while len(_OIDX_CACHE) > max_entries:
2497 _OIDX_CACHE.popitem(last=False)
2498
2499
2500def _records_header_length(path: Path) -> int:
2501 """Byte length of records.tsv's own physical header line (newline
2502 included), or 0 if the file is empty/missing. Reads only the first
2503 physical line -- cheap regardless of file size."""
2504 try:
2505 with path.open("rb") as handle:
2506 return len(handle.readline())
2507 except OSError:
2508 return 0
2509
2510
2511def _scan_append_rows_for_offsets(
2512 path: Path,
2513 physical_fields: list[str],
2514 start_byte: int,
2515) -> tuple[list[tuple[int, int, str, str]], int, int]:
2516 """Scan records.tsv from `start_byte` to EOF for complete physical
2517 rows, returning (entries, covered_bytes, row_count).
2518
2519 `entries` is one (row_start, row_end, op, id) per row found with a
2520 non-empty id (blank-id rows -- reachable only via a hand-edited file,
2521 see _fold_append_rows -- can't be looked up by id, so are simply never
2522 indexed, though they DO count towards `row_count`, matching
2523 _parse_append_body's physical_row_count exactly); `covered_bytes` is
2524 the byte offset up to which scanning is authoritative (>= start_byte,
2525 equal to it when nothing new/complete was found). `start_byte` must
2526 land exactly on a row boundary -- every offset this module ever hands
2527 back here (a prior scan's covered_bytes, or the records-header
2528 length) always does, since it is always either 0 or a previous
2529 row_end.
2530
2531 Torn-tail tolerant exactly like _parse_append_body: a trailing
2532 unterminated physical line is dropped first (_drop_torn_tail), and a
2533 row that still fails to parse (or reports more fields than the header
2534 -- a real corruption, not a torn write) stops consumption at that
2535 point rather than raising. This function must NEVER raise: it is the
2536 sidecar's own scan, and the sidecar is never allowed to be less
2537 tolerant than the reader it exists to speed up (any row it can't
2538 make sense of just isn't indexed yet -- the caller's `coherent` flag,
2539 derived from comparing covered_bytes to the file's actual size,
2540 reports that honestly rather than guessing).
2541
2542 Delegates the actual scan to _scan_append_tail (shared with the
2543 records-cache's tail-delta incremental fold, _append_tail_delta_
2544 records) and projects each full physical row down to just (op, id)
2545 plus its byte span -- all this function's callers need.
2546 """
2547 scanned, covered_bytes = _scan_append_tail(path, physical_fields, start_byte)
2548 entries: list[tuple[int, int, str, str]] = []
2549 row_count = 0
2550 for row_start, row_end, record in scanned:
2551 row_count += 1
2552 record_id = record.get("id", "")
2553 if not record_id:
2554 continue
2555 op = record.get(OP_FIELD, OP_UPSERT)
2556 entries.append((row_start, row_end, op, record_id))
2557
2558 return entries, covered_bytes, row_count
2559
2560
2561def _fold_oidx_entries(
2562 entries: list[tuple[int, int, str, str]],
2563 *,
2564 base: dict[str, int] | None = None,
2565) -> dict[str, int]:
2566 """Fold sidecar entries last-wins into {id: row_start_offset},
2567 dropping deleted ids -- the same rule _fold_append_rows applies to
2568 full rows, applied here to bare offsets.
2569
2570 Mutates and returns `base` directly when given, rather than copying
2571 it first: every call site's `base` is either freshly built moments
2572 earlier with nothing else referencing it (_load_oidx_body's return),
2573 or is _OIDX_CACHE's own entry being brought up to date in a catch-up
2574 that is about to replace that same cache slot anyway (see the
2575 matching note on _update_oidx_after_append for why this dict is safe
2576 to mutate in place: it's never exposed outside this module). A copy
2577 here would be a one-time-per-catch-up cost rather than a per-write
2578 one, so it's lower-stakes than the bug that note describes -- but
2579 there's no reason to pay even that when nothing needs the old
2580 version preserved.
2581 """
2582 result = base if base is not None else {}
2583 for row_start, _row_end, op, record_id in entries:
2584 if op == OP_DELETE:
2585 result.pop(record_id, None)
2586 else:
2587 result[record_id] = row_start
2588 return result
2589
2590
2591def _write_oidx_file(
2592 oidx_path: Path, data_ino: int, entries: list[tuple[int, int, str, str]]
2593) -> None:
2594 """Write a fresh sidecar from scratch (REBUILD), atomically (temp file
2595 + replace) so a concurrent reader never observes a half-written file
2596 mid-rebuild -- unlike the steady-state append path
2597 (_append_oidx_lines), which is cheap specifically because it skips
2598 this."""
2599 oidx_path.parent.mkdir(parents=True, exist_ok=True)
2600 temp_path = oidx_path.with_name(f".{oidx_path.name}.{os.getpid()}.{threading.get_ident()}.tmp")
2601 try:
2602 with temp_path.open("w", newline="") as handle:
2603 handle.write(f"{OIDX_HEADER_TAG}\t{data_ino}\n")
2604 handle.writelines(
2605 f"{row_start}\t{row_end}\t{op}\t{record_id}\n"
2606 for row_start, row_end, op, record_id in entries
2607 )
2608 temp_path.replace(oidx_path)
2609 finally:
2610 if temp_path.exists():
2611 temp_path.unlink()
2612
2613
2614def _append_oidx_lines(
2615 oidx_path: Path, data_ino: int, entries: list[tuple[int, int, str, str]]
2616) -> None:
2617 """Append idx lines to an existing sidecar whose header ino the
2618 caller has already confirmed matches `data_ino`. Self-heals a torn
2619 idx tail first (same helper the data file uses -- the format needs no
2620 CSV awareness to repair: an idx line can never contain a literal tab
2621 or newline inside a field, since ids are restricted to _RECORD_ID_RE
2622 and offsets/op are plain digits/literals, so "does the file end with
2623 \\n" is exactly as reliable a completeness signal here as it is for
2624 records.tsv)."""
2625 if not entries:
2626 return
2627 _repair_torn_tail_lines(oidx_path)
2628 with oidx_path.open("a", newline="") as handle:
2629 handle.writelines(
2630 f"{row_start}\t{row_end}\t{op}\t{record_id}\n"
2631 for row_start, row_end, op, record_id in entries
2632 )
2633 handle.flush()
2634
2635
2636def _read_oidx_header(oidx_path: Path) -> int | None:
2637 """Return the data_ino recorded in the sidecar's header line, or None
2638 when the file is missing, empty, or its header doesn't parse (all
2639 treated identically by the caller: rebuild)."""
2640 try:
2641 with oidx_path.open("r", newline="") as handle:
2642 first_line = handle.readline()
2643 except (OSError, UnicodeDecodeError, ValueError):
2644 # A sidecar corrupted with non-UTF-8 bytes must read as "header
2645 # does not parse" -> rebuild, per _load_oidx's own contract
2646 # ("100% best-effort and NEVER raises"). Catching only OSError
2647 # here let a garbage sidecar crash every read of the collection
2648 # instead of self-healing -- the atomicity characterization's
2649 # sidecar finding.
2650 return None
2651 parts = first_line.rstrip("\n").split("\t")
2652 if len(parts) != 2 or parts[0] != OIDX_HEADER_TAG:
2653 return None
2654 try:
2655 return int(parts[1])
2656 except ValueError:
2657 return None
2658
2659
2660def _load_oidx_body(
2661 oidx_path: Path,
2662) -> tuple[dict[str, int], int | None, int] | None:
2663 """Parse a sidecar's data lines (everything after the header) into
2664 ({id: offset}, covered_bytes, indexed_row_count). Returns None when the
2665 file can't be trusted at all (missing, or a data line doesn't have
2666 exactly the 4 expected columns / non-integer offsets -- genuine
2667 corruption, not a torn write, so this doesn't try to salvage a prefix:
2668 the caller rebuilds instead of guessing).
2669
2670 covered_bytes is None when there are no data lines yet (a header-only
2671 sidecar, e.g. right after this collection's very first indexed
2672 write): the caller supplies the real baseline in that case
2673 (_records_header_length), since a brand-new sidecar covers exactly
2674 the data file's own header bytes, not zero.
2675
2676 indexed_row_count is the number of body lines parsed -- one per
2677 PHYSICAL row of records.tsv this sidecar has indexed (create, update,
2678 and delete rows all count; only the blank-id rows
2679 _scan_append_rows_for_offsets never indexes are excluded, per that
2680 function's docstring). Exists for object_records.append_collection_
2681 stats' cheap physical-row-count path (a "pure line-count read of the
2682 sidecar", avoiding a full fold of records.tsv) -- no other caller
2683 needs it, so it costs this function nothing beyond incrementing a
2684 counter already being iterated for the fold below.
2685
2686 A torn last line (the file doesn't end with "\\n") is dropped, same
2687 rule as everywhere else: an idx line, like a data row, is only
2688 trustworthy once its own newline lands.
2689 """
2690 try:
2691 text = oidx_path.read_text()
2692 except OSError:
2693 return None
2694 lines = text.split("\n")
2695 if not lines or not lines[0].startswith(f"{OIDX_HEADER_TAG}\t"):
2696 return None
2697
2698 body_lines = lines[1:]
2699 if body_lines and body_lines[-1] == "":
2700 body_lines = body_lines[:-1] # file ends with \n -- drop the trailing empty split
2701 elif body_lines:
2702 body_lines = body_lines[:-1] # torn last line -- drop it, unindexed
2703
2704 result: dict[str, int] = {}
2705 covered_bytes: int | None = None
2706 indexed_row_count = 0
2707 for line in body_lines:
2708 if not line:
2709 continue
2710 parts = line.split("\t")
2711 if len(parts) != 4:
2712 return None
2713 try:
2714 row_start = int(parts[0])
2715 row_end = int(parts[1])
2716 except ValueError:
2717 return None
2718 op, record_id = parts[2], parts[3]
2719 if op == OP_DELETE:
2720 result.pop(record_id, None)
2721 else:
2722 result[record_id] = row_start
2723 covered_bytes = row_end
2724 indexed_row_count += 1
2725
2726 return result, covered_bytes, indexed_row_count
2727
2728
2729def _oidx_get_row(
2730 path: Path, physical_fields: list[str], offset: int
2731) -> dict[str, str] | None:
2732 """Seek to `offset` in records.tsv and parse exactly the one physical
2733 row starting there, returning the LOGICAL record (id + fields, `_op`
2734 stripped) -- or None if the row can't be read/parsed cleanly, which
2735 the caller treats as "sidecar inconclusive, fall back to a full
2736 fold" rather than as "record missing"."""
2737 # Read the full LOGICAL row via csv, not a raw readline(). A row whose
2738 # value contains an embedded newline (a quoted field spanning several
2739 # physical lines -- e.g. multi-line text, or pretty-printed JSON) must be
2740 # consumed to its real row terminator; a readline() would stop at the
2741 # first '\n' byte and silently return a truncated (often unparseable)
2742 # value. csv.reader over the file from `offset` handles the quoted-field
2743 # continuation exactly as the full-fold path (_parse_append_body) does,
2744 # keeping the by-id read consistent with it. Any read/parse trouble ->
2745 # None, so the caller falls back to a full fold rather than trusting a
2746 # partial row.
2747 try:
2748 handle = path.open("rb")
2749 except OSError:
2750 return None
2751 try:
2752 handle.seek(offset)
2753 text = io.TextIOWrapper(handle, encoding="utf-8", newline="")
2754 row = next(csv.reader(text, delimiter="\t"))
2755 except (OSError, UnicodeDecodeError, StopIteration, csv.Error):
2756 return None
2757 finally:
2758 handle.close()
2759 field_count = len(physical_fields)
2760 if not row or len(row) > field_count:
2761 return None
2762 record = {
2763 physical_fields[i]: (row[i] if i < len(row) else "")
2764 for i in range(field_count)
2765 }
2766 return {key: value for key, value in record.items() if key != OP_FIELD}
2767
2768
2769def _load_oidx(path: Path) -> tuple[dict[str, int], bool]:
2770 """Return ({id: row_start_offset}, coherent) for `path`'s CURRENT
2771 append-mode content, using and maintaining the on-disk sidecar plus
2772 the small in-memory _OIDX_CACHE LRU.
2773
2774 `coherent` is True exactly when the returned dict accounts for every
2775 byte of the file as observed by this call's own stat -- i.e. it is
2776 safe to treat "id not in the dict" as proof the id doesn't currently
2777 exist. False means only a best-effort snapshot could be produced
2778 (normally: a torn tail at EOF from a write that crashed before
2779 self-healing) and callers must not treat an absence -- or, to be
2780 conservative, even a presence -- as authoritative; see
2781 _fast_record_lookup, the only caller, which falls all the way back to
2782 a full fold whenever this is False.
2783
2784 100% best-effort and NEVER raises: any inconsistency (missing sidecar,
2785 corrupt sidecar, inode mismatch, a data file that shrank under a
2786 stable inode -- e.g. a torn-tail repair truncation) triggers a
2787 rebuild (one sequential scan of records.tsv) rather than propagating
2788 an error, per docs/append-only-storage-design.md: the sidecar is
2789 "never required for correctness."
2790 """
2791 try:
2792 st = path.stat()
2793 except OSError:
2794 return {}, False
2795 data_ino = st.st_ino
2796 data_size = st.st_size
2797 cache_key = str(path.resolve(strict=False))
2798
2799 cached = _OIDX_CACHE.get(cache_key)
2800 if cached is not None:
2801 cached_ino, cached_covered, cached_dict = cached
2802 if cached_ino == data_ino and cached_covered <= data_size:
2803 if cached_covered == data_size:
2804 _OIDX_CACHE.move_to_end(cache_key)
2805 return cached_dict, True
2806 physical_fields = _physical_header(path)
2807 if physical_fields and physical_fields[0] == OP_FIELD:
2808 entries, new_covered, _row_count = _scan_append_rows_for_offsets(
2809 path, physical_fields, cached_covered
2810 )
2811 new_dict = _fold_oidx_entries(entries, base=cached_dict)
2812 _append_oidx_lines(_oidx_path(path), data_ino, entries)
2813 _store_oidx_cache(cache_key, data_ino, new_covered, new_dict)
2814 return new_dict, new_covered == data_size
2815 # ino mismatch, or the file shrank under a stable inode: this
2816 # in-memory snapshot can't be trusted as a base -- drop it and
2817 # fall through to the on-disk / rebuild path below.
2818 _OIDX_CACHE.pop(cache_key, None)
2819
2820 physical_fields = _physical_header(path)
2821 if not physical_fields or physical_fields[0] != OP_FIELD:
2822 return {}, False
2823
2824 oidx_path = _oidx_path(path)
2825 header_ino = _read_oidx_header(oidx_path)
2826 if header_ino == data_ino:
2827 loaded = _load_oidx_body(oidx_path)
2828 if loaded is not None:
2829 body_dict, body_covered, _body_row_count = loaded
2830 covered = body_covered if body_covered is not None else _records_header_length(path)
2831 if covered <= data_size:
2832 if covered < data_size:
2833 entries, covered, _row_count = _scan_append_rows_for_offsets(
2834 path, physical_fields, covered
2835 )
2836 body_dict = _fold_oidx_entries(entries, base=body_dict)
2837 _append_oidx_lines(oidx_path, data_ino, entries)
2838 _store_oidx_cache(cache_key, data_ino, covered, body_dict)
2839 return body_dict, covered == data_size
2840 # covered > data_size: the data file shrank under a stable
2841 # ino -- fall through to a full rebuild rather than trust a
2842 # sidecar that claims to know about bytes no longer there.
2843
2844 # REBUILD: a full sequential scan of records.tsv from its own header,
2845 # functionally equivalent to the classic full-fold parse this branch
2846 # replaces for an over-threshold collection -- so, like that parse
2847 # (_parse_records_file -> _maybe_flag_auto_compact), it's also the
2848 # right place to (re-)evaluate auto-compaction. Incremental catch-up
2849 # above deliberately does NOT re-flag: this is a documented, narrower
2850 # trigger window than the pre-sidecar behavior (every full fold used
2851 # to re-check on every cold read) -- a collection gets an accurate
2852 # check whenever its sidecar needs rebuilding (first use past the
2853 # cache threshold, or any time compaction/a mode switch/corruption
2854 # invalidates it) but not on every subsequent catch-up scan. Flagging
2855 # on every catch-up too would need a persisted running physical-row
2856 # count in the sidecar itself; not worth the format complexity for a
2857 # threshold heuristic that self-corrects the next time a rebuild does
2858 # happen.
2859 header_bytes = _records_header_length(path)
2860 entries, covered, row_count = _scan_append_rows_for_offsets(path, physical_fields, header_bytes)
2861 fresh_dict = _fold_oidx_entries(entries)
2862 _write_oidx_file(oidx_path, data_ino, entries)
2863 _store_oidx_cache(cache_key, data_ino, covered, fresh_dict)
2864 if covered == data_size:
2865 _maybe_flag_auto_compact(path, physical_row_count=row_count, live_row_count=len(fresh_dict))
2866 return fresh_dict, covered == data_size
2867
2868
2869def _discard_oidx(path: Path) -> None:
2870 """Drop any sidecar tracking (in-memory and on-disk) for `path` --
2871 called after a full rewrite of an append-physical file (compaction, a
2872 mode-transition rewrite in either direction, or the new-field
2873 fallback rewrite): the rewrite gives the file a new inode via
2874 temp+rename, which would make any existing sidecar's header ino stale
2875 even without this (the next _load_oidx would detect the mismatch and
2876 rebuild on its own) -- this just does it eagerly so a compacted or
2877 mode-switched collection's directory doesn't carry a permanently
2878 orphaned `.records.oidx` around
2879 (docs/append-only-storage-design.md Sidecars: "delete or rebuild
2880 sidecar after the rewrite"). Never called for a collection with no
2881 append-mode history -- see the SCOPE note above this section -- so a
2882 purely classic collection's write path never reaches here. Best-
2883 effort: an unlink failure is not a correctness problem, so this never
2884 raises.
2885 """
2886 cache_key = str(path.resolve(strict=False))
2887 _OIDX_CACHE.pop(cache_key, None)
2888 try:
2889 _oidx_path(path).unlink()
2890 except OSError:
2891 pass
2892
2893
2894def _update_oidx_after_append(
2895 path: Path,
2896 rows: list[dict[str, str]],
2897 offsets: list[tuple[int, int]],
2898) -> None:
2899 """Keep the sidecar in sync with a fast append, in O(1): appends
2900 matching idx line(s) to the on-disk sidecar and updates the in-memory
2901 _OIDX_CACHE entry -- but ONLY when that entry is already warm for
2902 this path. When it isn't, this does nothing at all: a write must
2903 never pay to build or maintain a sidecar nobody is currently reading
2904 through (see _load_oidx, which lazily builds/catches-up on the next
2905 point op that actually needs it -- "Sidecar builds lazily on first
2906 over-threshold op").
2907
2908 `offsets` are exactly what _append_records_rows just returned for
2909 `rows` (same order, same length) -- no re-derivation (stat or scan)
2910 needed to keep this O(1) regardless of collection size.
2911
2912 Mutates `cached_dict` IN PLACE rather than copying it first, UNLIKE
2913 _refresh_records_cache_after_append's equivalent step for
2914 _RECORDS_CACHE (which always copies, even on its "fast" path -- see
2915 that function's docstring). That copy is safe there because
2916 _RECORDS_CACHE's records/id_index are handed across the public API
2917 (get/list return values, directly or via one more copy) and so must
2918 never be mutated out from under an earlier caller -- and it's AFFORD-
2919 ABLE there because _RECORDS_CACHE only ever holds collections at or
2920 under DBBASIC_RECORDS_CACHE_MAX_ROWS. Neither holds for this dict: it
2921 is never exposed outside this module (every public entry point that
2922 might have sourced data from it -- get/create/update/delete via
2923 _fast_record_lookup -- only ever calls dict.get() on it and returns a
2924 freshly-parsed ROW, never the dict itself), and it exists specifically
2925 FOR collections too large for a copy to be the cheap operation it is
2926 for _RECORDS_CACHE (a 1M-entry dict copy alone was measured to add
2927 ~35-40ms to every single write here -- see the sidecar benchmark
2928 report -- reintroducing real per-write cost in exactly the size
2929 regime this feature exists to make O(1) again). A concurrent
2930 lock-free reader's dict.get() racing this mutation is benign under
2931 the GIL (each op is atomic; a race can only return a value from
2932 just-before or just-after the write, never corrupt memory or raise)
2933 -- the same class of eventual-consistency already documented and
2934 accepted for _RECORDS_CACHE's own CONCURRENCY note above.
2935 """
2936 cache_key = str(path.resolve(strict=False))
2937 cached = _OIDX_CACHE.get(cache_key)
2938 if cached is None:
2939 return
2940 cached_ino, cached_covered, cached_dict = cached
2941 if cached_covered != offsets[0][0]:
2942 # The cached view doesn't start exactly where this append starts
2943 # (should not happen -- writes are serialized under the caller's
2944 # file lock and this cache is only ever advanced by this same
2945 # process's own writes/reads) -- rather than risk recording wrong
2946 # offsets, drop it; the next _load_oidx call resyncs or rebuilds.
2947 _OIDX_CACHE.pop(cache_key, None)
2948 return
2949
2950 entries: list[tuple[int, int, str, str]] = []
2951 for row, (start, end) in zip(rows, offsets, strict=True):
2952 op = row.get(OP_FIELD, OP_UPSERT)
2953 record_id = row.get("id", "")
2954 entries.append((start, end, op, record_id))
2955 if op == OP_DELETE:
2956 cached_dict.pop(record_id, None)
2957 else:
2958 cached_dict[record_id] = start
2959
2960 _append_oidx_lines(_oidx_path(path), cached_ino, entries)
2961 _store_oidx_cache(cache_key, cached_ino, offsets[-1][1], cached_dict)
2962
2963
2964def _fast_record_lookup(
2965 collection: str,
2966 path: Path,
2967 record_id: str,
2968 *,
2969 need_row: bool,
2970) -> tuple[
2971 list[str],
2972 bool,
2973 dict[str, str] | None,
2974 list[dict[str, str]] | None,
2975 dict[str, int] | None,
2976]:
2977 """Resolve (fields, exists, existing_row, records_ref, id_index_ref)
2978 for one record id -- the shared point-op resolver behind get/create/
2979 update/delete, in increasing order of cost:
2980
2981 1. A warm _RECORDS_CACHE hit (_peek_records_cache) -- the ordinary
2982 fast path for any collection at or under
2983 DBBASIC_RECORDS_CACHE_MAX_ROWS once touched once. records_ref/
2984 id_index_ref are the module cache's own objects (never mutate --
2985 see the _RECORDS_CACHE ALIASING SAFETY note).
2986 2. On a miss, if the file is append-physical: the id->offset
2987 sidecar (_load_oidx). Answers in O(1) without ever folding the
2988 file -- this is the whole point of the sidecar (docs/append-
2989 only-storage-design.md item 4): a collection too large for
2990 _RECORDS_CACHE no longer pays an O(n) fold on every write.
2991 Trusted only when _load_oidx reports `coherent` -- otherwise
2992 (a torn tail at EOF from an unfinished write) this falls all the
2993 way through to step 3 rather than risk a wrong answer.
2994 records_ref/id_index_ref are None in this case: the fast-append
2995 write path this enables never needs the full record list (see
2996 _persist_write's folded_records_fn, which forces its own full
2997 fold via _cache_entry lazily, only if a rare full-rewrite path
2998 is chosen instead of a fast append).
2999 3. _cache_entry -- the original full fold, unchanged, used for
3000 classic-mode files, a genuinely cold/small collection's first
3001 touch, and any sidecar-inconclusive case above.
3002
3003 fields is always the LOGICAL field list (no `_op`). `existing_row`,
3004 when requested via need_row and the id exists, is always a fresh,
3005 independent copy safe to hand across the public API or mutate.
3006 """
3007 cached = _peek_records_cache(path)
3008 if cached is not None:
3009 fields, records_ref, id_index_ref = cached
3010 index = id_index_ref.get(record_id)
3011 existing_row = dict(records_ref[index]) if (need_row and index is not None) else None
3012 return fields, index is not None, existing_row, records_ref, id_index_ref
3013
3014 physical_header = _physical_header(path)
3015 if physical_header and physical_header[0] == OP_FIELD:
3016 id_offsets, coherent = _load_oidx(path)
3017 if coherent:
3018 offset = id_offsets.get(record_id)
3019 if offset is None:
3020 return physical_header[1:], False, None, None, None
3021 if not need_row:
3022 return physical_header[1:], True, None, None, None
3023 row = _oidx_get_row(path, physical_header, offset)
3024 if row is not None:
3025 return physical_header[1:], True, row, None, None
3026 # Row unreadable at its recorded offset despite a coherent
3027 # sidecar -- shouldn't happen; be conservative and fall
3028 # through to the authoritative full fold below.
3029
3030 fields, records_ref, id_index_ref = _cache_entry(collection, path)
3031 index = id_index_ref.get(record_id)
3032 existing_row = dict(records_ref[index]) if (need_row and index is not None) else None
3033 return fields, index is not None, existing_row, records_ref, id_index_ref
3034
3035
3036def _collection_storage_mode(
3037 collection: str,
3038 *,
3039 base_dir: Path | str,
3040 roots: Iterable[Path] | None,
3041) -> str:
3042 """Return a collection's storage mode from its schema's `storage` key.
3043
3044 Defaults to classic when the schema has no `storage` key, is derived
3045 (no manual schema file), or doesn't exist yet -- see
3046 object_schemas.normalize_schema, which validates the key at schema-
3047 write time, so any value reaching here is already known-good, but this
3048 stays defensive rather than trusting that unconditionally.
3049 """
3050 try:
3051 schema = object_schemas.get_schema(collection, base_dir=base_dir, roots=roots)
3052 except object_schemas.SchemaNotFoundError:
3053 return object_schemas.STORAGE_CLASSIC
3054 mode = schema.get("storage", object_schemas.STORAGE_CLASSIC)
3055 return mode if mode in object_schemas.VALID_STORAGE_MODES else object_schemas.STORAGE_CLASSIC
3056
3057
3058def _persist_write(
3059 collection: str,
3060 path: Path,
3061 *,
3062 base_dir: Path | str,
3063 roots: Iterable[Path] | None,
3064 prior_fields: list[str],
3065 merged_fields: list[str],
3066 folded_records_fn,
3067 delta_row: dict[str, str] | None,
3068 delta_op: str,
3069 delta_id: str,
3070) -> None:
3071 """Persist one create/update/delete, choosing the write strategy.
3072
3073 `folded_records_fn` is a zero-argument callable that builds the FULL
3074 folded record set after this op (a fresh, safe-to-mutate copy -- see
3075 the callers in create/update/delete_collection_record). It is called
3076 LAZILY, only on a full-rewrite path, and never on the fast-append
3077 path: the whole point of a fast append is O(1) work, and building
3078 that full copy is an O(n) cost callers can otherwise skip entirely
3079 (see _refresh_records_cache_after_append, which updates the cache
3080 from its own prior entry plus this one delta instead). `delta_row`/
3081 `delta_op`/`delta_id` describe just this one op's row, used only by
3082 the fast-append path. Chooses among (see
3083 docs/append-only-storage-design.md and the Decisions this module's
3084 implementation is bound by):
3085
3086 - FAST APPEND: storage is "append", the file is already physically
3087 in append format, this write doesn't introduce a field the
3088 current header lacks, and no auto-compaction is pending for this
3089 collection -- append one row, then update whichever of
3090 _RECORDS_CACHE / the id->offset sidecar (docs/append-only-
3091 storage-design.md item 4) is already warm for this path, in O(1),
3092 via _refresh_records_cache_after_append / _update_oidx_after_
3093 append. Neither is FORCED warm here: a collection too large for
3094 _RECORDS_CACHE (the case the sidecar exists for) is left
3095 uncached by design, and a sidecar nobody has read through yet is
3096 left unbuilt (see _update_oidx_after_append) -- either would mean
3097 paying the exact O(n) fold-per-write cost this branch exists to
3098 avoid, just moved from "on every write" to "on every write that
3099 happens to miss the cache," which is most of them for a
3100 write-hot, over-threshold collection.
3101 - TRANSITION-IN: storage is "append" but the file isn't (yet) in
3102 append format (including a brand new collection) -- one full
3103 rewrite that adds the `_op` header column, applying this op in
3104 the same pass.
3105 - NEW-FIELD FALLBACK / AUTO-COMPACT: storage is "append", the file
3106 is already append-format, but this write needs a header column
3107 the file doesn't have yet, or a prior read flagged this
3108 collection for compaction -- a full rewrite that folds current
3109 content, applies this op, and (in the new-field case) extends the
3110 header. Still `_op`-columned.
3111 - COMPACT-TO-CLASSIC: storage is "classic" but the file is still
3112 physically in append format (opt-out just happened) -- one full
3113 rewrite that folds current content, applies this op, and drops
3114 the `_op` column.
3115 - CLASSIC: storage is "classic" and the file already is (or never
3116 was anything but) classic format -- the original full-rewrite
3117 behavior, completely unchanged.
3118 """
3119 desired_mode = _collection_storage_mode(collection, base_dir=base_dir, roots=roots)
3120 physical_header = _physical_header(path)
3121 file_is_append_physical = bool(physical_header) and physical_header[0] == OP_FIELD
3122 cache_key = str(path.resolve(strict=False))
3123 new_field_introduced = merged_fields != prior_fields
3124 pending_compaction = cache_key in _PENDING_COMPACTION
3125
3126 if (
3127 desired_mode == object_schemas.STORAGE_APPEND
3128 and file_is_append_physical
3129 and not new_field_introduced
3130 and not pending_compaction
3131 ):
3132 physical_fields = [OP_FIELD, *merged_fields]
3133 row = dict(delta_row) if delta_row is not None else {}
3134 row["id"] = delta_id
3135 row[OP_FIELD] = delta_op
3136 offsets = _append_records_rows(path, physical_fields, [row])
3137 projected_row = _project_record(row, merged_fields)
3138 _refresh_records_cache_after_append(
3139 collection, path, merged_fields, delta_op, delta_id, projected_row
3140 )
3141 _update_oidx_after_append(path, [row], offsets)
3142 return
3143
3144 if file_is_append_physical:
3145 _discard_oidx(path)
3146 # folded_records_fn() is called BEFORE the discard below (not after,
3147 # as it may look more natural) because, on the sidecar-sourced lookup
3148 # path, it can itself force a fresh full fold (_cache_entry ->
3149 # _parse_records_file), which re-evaluates and can re-set auto-compact
3150 # flagging for this same cache_key from the PRE-write content. This
3151 # write is about to make that stale regardless of what it flagged
3152 # (append: compacting; classic/transition: rewriting from scratch
3153 # either way), so the discard must be the last word.
3154 folded_records = folded_records_fn()
3155 _PENDING_COMPACTION.discard(cache_key)
3156 if desired_mode == object_schemas.STORAGE_APPEND:
3157 physical_fields = [OP_FIELD, *merged_fields]
3158 _write_collection_records(
3159 collection, path, physical_fields, folded_records, cache_fields=merged_fields
3160 )
3161 else:
3162 _write_collection_records(collection, path, merged_fields, folded_records)
3163
3164
3165@contextmanager
3166def _records_file_lock(records_file: Path):
3167 """Use a best-effort advisory lock for collection record mutations."""
3168 lock_path = records_file.with_name(f".{records_file.name}.lock")
3169 lock_path.parent.mkdir(parents=True, exist_ok=True)
3170
3171 with lock_path.open("a") as lock_file:
3172 try:
3173 import fcntl
3174 except ImportError:
3175 fcntl = None
3176
3177 if fcntl is not None:
3178 fcntl.flock(lock_file, fcntl.LOCK_EX)
3179
3180 try:
3181 yield
3182 finally:
3183 if fcntl is not None:
3184 fcntl.flock(lock_file, fcntl.LOCK_UN)
3185
3186
3187def _validate_header(collection: str, fields: list[str] | None) -> None:
3188 if not fields:
3189 raise ValueError(f"Collection records file is missing a header: {collection}")
3190
3191 clean = [field.strip() for field in fields]
3192 if any(not field for field in clean):
3193 raise ValueError(f"Collection records file has an empty field name: {collection}")
3194 if clean != fields:
3195 raise ValueError(f"Collection records file has whitespace in field names: {collection}")
3196 if len(set(clean)) != len(clean):
3197 raise ValueError(f"Collection records file has duplicate field names: {collection}")
3198 if "id" not in clean:
3199 raise ValueError(f"Collection records file must include an id column: {collection}")
3200
3201
3202def _normalize_record_payload(payload: dict[str, Any], *, require_id: bool) -> dict[str, str]:
3203 if not isinstance(payload, dict):
3204 raise InvalidRecordPayloadError("Record payload must be an object")
3205 generate_id = require_id and (
3206 "id" not in payload
3207 or payload.get("id") is None
3208 or payload.get("id") == ""
3209 )
3210
3211 clean: dict[str, str] = {}
3212 for key, value in payload.items():
3213 if not isinstance(key, str) or not key:
3214 raise InvalidRecordPayloadError("Record field names must be non-empty strings")
3215 if key.strip() != key:
3216 raise InvalidRecordPayloadError(f"Record field name has whitespace: {key!r}")
3217 if "\t" in key or "\n" in key or "\r" in key:
3218 raise InvalidRecordPayloadError(f"Record field name is not TSV-safe: {key!r}")
3219 if key == OP_FIELD:
3220 raise InvalidRecordPayloadError(
3221 f"Record field name is reserved for internal storage use: {key!r}"
3222 )
3223 if key == EXTRA_FIELD:
3224 clean[key] = _normalize_extra_value(value)
3225 continue
3226 if value is None:
3227 clean[key] = ""
3228 elif isinstance(value, (str, int, float, bool)):
3229 clean[key] = str(value)
3230 else:
3231 raise InvalidRecordPayloadError(
3232 f"Record field value must be scalar or null: {key}"
3233 )
3234
3235 if generate_id:
3236 clean["id"] = object_ids.new_uuid4()
3237 elif require_id and "id" not in clean:
3238 raise InvalidRecordPayloadError("Record payload must include an id")
3239
3240 return clean
3241
3242
3243def _normalize_extra_value(value: Any) -> str:
3244 """Return a TSV-safe JSON-object string for the ``extra`` field.
3245
3246 Unlike other record fields (scalar or null only), ``extra`` may be
3247 submitted as a dict (serialized here) or as a JSON string that must
3248 already parse to an object. Anything else is rejected.
3249 """
3250 if isinstance(value, dict):
3251 return json.dumps(value, sort_keys=True)
3252 if isinstance(value, str):
3253 try:
3254 parsed = json.loads(value)
3255 except (TypeError, ValueError) as exc:
3256 raise InvalidRecordPayloadError(
3257 "Record field 'extra' must be a JSON object"
3258 ) from exc
3259 if not isinstance(parsed, dict):
3260 raise InvalidRecordPayloadError("Record field 'extra' must be a JSON object")
3261 return value
3262 raise InvalidRecordPayloadError(
3263 "Record field 'extra' must be an object or a JSON object string"
3264 )
3265
3266
3267def _apply_auto_created_at(
3268 collection: str,
3269 record: dict[str, str],
3270 submitted_fields: frozenset[str],
3271 *,
3272 base_dir: Path | str,
3273 roots: Iterable[Path] | None,
3274) -> dict[str, str]:
3275 """Server-set ``created_at`` on create when the schema declares it.
3276
3277 A schema field named ``created_at`` (date/datetime) is filled with the
3278 current UTC time on create, unless the client supplied it. Because the
3279 server fills it (not the client), it can be ``read_only`` — clients can
3280 neither omit it nor spoof it — which is what lets generated lists show
3281 a trustworthy relative timestamp.
3282 """
3283 if "created_at" in submitted_fields:
3284 return record
3285 for field in _schema_fields(collection, base_dir=base_dir, roots=roots):
3286 if field.get("name") != "created_at":
3287 continue
3288 field_type = str(field.get("type") or "").lower()
3289 now = datetime.now(timezone.utc)
3290 if field_type == "date":
3291 value = now.date().isoformat()
3292 elif field_type in {"datetime", "timestamp"}:
3293 value = now.isoformat().replace("+00:00", "Z")
3294 else:
3295 return record
3296 clean = dict(record)
3297 clean["created_at"] = value
3298 return clean
3299 return record
3300
3301
3302def _apply_schema_defaults(
3303 collection: str,
3304 record: dict[str, str],
3305 *,
3306 base_dir: Path | str,
3307 roots: Iterable[Path] | None,
3308) -> dict[str, str]:
3309 fields = _schema_fields(collection, base_dir=base_dir, roots=roots)
3310 if not fields:
3311 return record
3312
3313 clean = dict(record)
3314 for field in fields:
3315 name = field["name"]
3316 if name in clean and clean[name] != "":
3317 continue
3318 if _is_computed_or_read_only(field):
3319 continue
3320 if "default" not in field:
3321 continue
3322 clean[name] = _schema_scalar_to_string(field["default"], field_name=name)
3323 return clean
3324
3325
3326# ---------------------------------------------------------------------------
3327# Formula & rollup fields (plan/formula-rollup-spec.md): derived values,
3328# MATERIALIZED at this storage layer so every writer (HTTP, daemon, seeds,
3329# imports) keeps them consistent and every surface reads plain stored values.
3330# Formulas recompute on writes of the record itself; rollups recompute on
3331# writes/deletes in the source collection (single-hop, thread-local guarded).
3332# A broken formula stores "" and never fails the write -- derived values are
3333# non-authoritative, the opposite posture of pre-write hooks.
3334# ---------------------------------------------------------------------------
3335
3336_ROLLUP_LOCAL = threading.local()
3337
3338
3339def _apply_computed_fields(
3340 collection: str,
3341 record: dict[str, str],
3342 *,
3343 base_dir: Path | str,
3344 roots: Iterable[Path] | None,
3345) -> dict[str, str]:
3346 """Recompute this record's own derived values: rollups first (so a parent
3347 created/edited after its children exist is immediately correct), then
3348 formulas (which may reference the fresh rollup values)."""
3349 fields = _schema_fields(collection, base_dir=base_dir, roots=roots)
3350 if not fields:
3351 return record
3352 rollups = object_computed.rollup_fields(fields)
3353 formulas = object_computed.formula_fields(fields)
3354 if not rollups and not formulas:
3355 return record
3356 out = dict(record)
3357 for field in rollups:
3358 spec = field["rollup"]
3359 try:
3360 children = [
3361 row
3362 for row in read_collection_records(
3363 spec["collection"], base_dir=base_dir, roots=roots
3364 )
3365 if row.get(spec["fk_field"]) == out.get("id")
3366 ]
3367 except (object_collections.CollectionNotFoundError, OSError, ValueError):
3368 continue
3369 out[field["name"]] = object_computed.compute_rollup(spec, children)
3370 return object_computed.apply_formulas(fields, out)
3371
3372
3373def _rollup_parents_for_source(
3374 source_collection: str,
3375 *,
3376 base_dir: Path | str,
3377 roots: Iterable[Path] | None,
3378) -> list[tuple[str, list[dict[str, Any]]]]:
3379 """(parent_collection, rollup_fields) pairs whose rollups aggregate the
3380 given source collection. Declared schemas only (the schemas dir); a
3381 collection cannot roll up itself (v1 loop guard)."""
3382 parents: list[tuple[str, list[dict[str, Any]]]] = []
3383 schema_dir = Path(base_dir) / "schemas"
3384 try:
3385 names = sorted(path.stem for path in schema_dir.glob("*.json"))
3386 except OSError:
3387 return parents
3388 for name in names:
3389 if name == source_collection:
3390 continue
3391 fields = _schema_fields(name, base_dir=base_dir, roots=roots)
3392 specs = [
3393 field
3394 for field in object_computed.rollup_fields(fields)
3395 if field["rollup"]["collection"] == source_collection
3396 ]
3397 if specs:
3398 parents.append((name, specs))
3399 return parents
3400
3401
3402def _recompute_rollups_for_source(
3403 source_collection: str,
3404 affected_rows: list[dict[str, str] | None],
3405 *,
3406 base_dir: Path | str,
3407 roots: Iterable[Path] | None,
3408) -> None:
3409 """After a write/delete in source_collection, bring affected parents'
3410 rollup fields current. Single-hop: the parent update this performs never
3411 triggers further rollup scans (thread-local guard). Best-effort by design:
3412 a failed parent refresh must never fail the child's own write."""
3413 if getattr(_ROLLUP_LOCAL, "active", False):
3414 return
3415 parents = _rollup_parents_for_source(source_collection, base_dir=base_dir, roots=roots)
3416 if not parents:
3417 return
3418 _ROLLUP_LOCAL.active = True
3419 try:
3420 for parent_collection, specs in parents:
3421 fk_values: set[str] = set()
3422 for field in specs:
3423 fk_field = field["rollup"]["fk_field"]
3424 for row in affected_rows:
3425 if row and row.get(fk_field):
3426 fk_values.add(str(row[fk_field]))
3427 if not fk_values:
3428 continue
3429 try:
3430 source_rows = read_collection_records(
3431 source_collection, base_dir=base_dir, roots=roots
3432 )
3433 except (object_collections.CollectionNotFoundError, OSError, ValueError):
3434 continue
3435 for parent_id in sorted(fk_values):
3436 try:
3437 parent = get_collection_record(
3438 parent_collection, parent_id, base_dir=base_dir, roots=roots
3439 )
3440 except (RecordNotFoundError, object_collections.CollectionNotFoundError,
3441 InvalidRecordIdError, OSError, ValueError):
3442 continue
3443 changes: dict[str, str] = {}
3444 for field in specs:
3445 spec = field["rollup"]
3446 children = [
3447 row for row in source_rows if row.get(spec["fk_field"]) == parent_id
3448 ]
3449 changes[field["name"]] = object_computed.compute_rollup(spec, children)
3450 if all(str(parent.get(k, "")) == v for k, v in changes.items()):
3451 continue
3452 try:
3453 update_collection_record(
3454 parent_collection,
3455 parent_id,
3456 changes,
3457 base_dir=base_dir,
3458 roots=roots,
3459 actor="system:rollup",
3460 allow_computed_submission=True,
3461 )
3462 except Exception:
3463 continue
3464 finally:
3465 _ROLLUP_LOCAL.active = False
3466
3467
3468def recompute_computed_fields(
3469 collection: str,
3470 *,
3471 base_dir: Path | str = DEFAULT_DATA_DIR,
3472 roots: Iterable[Path] | None = None,
3473) -> int:
3474 """Backfill/refresh every derived value in a collection; returns how many
3475 records changed. For definition changes and first adoption -- called
3476 explicitly (install migration, admin action, cron), never automatically in
3477 a runtime loop (docs: no runtime object rewriting; same spirit for data)."""
3478 fields = _schema_fields(collection, base_dir=base_dir, roots=roots)
3479 computed_names = {
3480 field["name"]
3481 for field in object_computed.rollup_fields(fields)
3482 + object_computed.formula_fields(fields)
3483 }
3484 if not computed_names:
3485 return 0
3486 changed = 0
3487 for row in read_collection_records(collection, base_dir=base_dir, roots=roots):
3488 desired = _apply_computed_fields(collection, dict(row), base_dir=base_dir, roots=roots)
3489 delta = {
3490 name: desired.get(name, "")
3491 for name in computed_names
3492 if str(row.get(name, "")) != str(desired.get(name, ""))
3493 }
3494 if not delta:
3495 continue
3496 update_collection_record(
3497 collection,
3498 row["id"],
3499 delta,
3500 base_dir=base_dir,
3501 roots=roots,
3502 actor="system:recompute",
3503 allow_computed_submission=True,
3504 )
3505 changed += 1
3506 return changed
3507
3508
3509def _validate_field_transitions(
3510 collection: str,
3511 existing: dict[str, str],
3512 updated: dict[str, str],
3513 *,
3514 base_dir: Path | str,
3515 roots: Iterable[Path] | None,
3516 subject: object_permissions.PermissionSubject | None = None,
3517) -> None:
3518 """Enforce declared value transitions on update.
3519
3520 A field may declare which values each current value can move to. Each
3521 list entry is either a plain string, allowed for any caller who may
3522 update the record, or a guarded object:
3523
3524 {"name": "status", "type": "enum", "enum": [...],
3525 "transitions": {
3526 "open": ["assigned", {"to": "cancelled", "when": {"owner_id": "$user_id"}}],
3527 ...
3528 }}
3529
3530 A guarded move is allowed only once every ``when`` clause matches the
3531 record's CURRENT stored values (``existing``, before this update)
3532 against the resolved subject variable or literal -- the same closed
3533 set and matching rules row filters use (see
3534 object_permissions.record_matches_filter).
3535
3536 This is deliberately data plus one check — not a state machine
3537 framework: no hooks, no side effects, no transition callbacks. A
3538 current value missing from the map cannot change; an empty existing
3539 value may move anywhere.
3540
3541 Guards are enforced only when ``subject`` is supplied. The HTTP update
3542 path resolves and plumbs the request subject once permissions have
3543 already run (see update_collection_record); direct library callers
3544 with no subject -- daemon, CLI, tests -- are trusted callers: a
3545 guarded move is still checked for validity (its "to" is in the list)
3546 but the "when" clause is not enforced.
3547 """
3548 fields = _schema_fields(collection, base_dir=base_dir, roots=roots)
3549 for field in fields:
3550 transitions = field.get("transitions")
3551 if not isinstance(transitions, dict):
3552 continue
3553 name = field["name"]
3554 old_value = existing.get(name, "")
3555 new_value = updated.get(name, "")
3556 if old_value == new_value or _is_empty(old_value):
3557 continue
3558
3559 entries = transitions.get(old_value)
3560 entries = entries if isinstance(entries, list) else []
3561
3562 allowed_values: list[str] = []
3563 move_is_valid = False
3564 guard_failed = False
3565 for entry in entries:
3566 if isinstance(entry, str):
3567 to_value, when = entry, None
3568 elif isinstance(entry, dict):
3569 to_value, when = str(entry.get("to", "")), entry.get("when") or {}
3570 else:
3571 continue
3572
3573 if to_value and to_value not in allowed_values:
3574 allowed_values.append(to_value)
3575 if to_value != new_value:
3576 continue
3577
3578 if when is None or subject is None:
3579 move_is_valid = True
3580 elif object_permissions.record_matches_filter(existing, when, subject):
3581 move_is_valid = True
3582 else:
3583 guard_failed = True
3584
3585 if move_is_valid:
3586 continue
3587
3588 options = ", ".join(allowed_values) if allowed_values else "none"
3589 if guard_failed:
3590 raise TransitionNotAllowedError(
3591 f"Record field '{name}' cannot move from '{old_value}' to "
3592 f"'{new_value}' for this subject (allowed: {options})"
3593 )
3594 raise InvalidRecordPayloadError(
3595 f"Record field '{name}' cannot move from '{old_value}' to "
3596 f"'{new_value}' (allowed: {options})"
3597 )
3598
3599
3600def _canonicalize_schema_values(
3601 collection: str,
3602 record: dict[str, str],
3603 *,
3604 base_dir: Path | str,
3605 roots: Iterable[Path] | None,
3606) -> dict[str, str]:
3607 """Store schema-typed values in one canonical form.
3608
3609 Boolean fields accept several spellings on input ("True", "1", "yes")
3610 but must be stored as "true"/"false" so permission row filters like
3611 {"is_public": "true"} match by string comparison. Runs after
3612 validation, so every value here is known to parse.
3613 """
3614 fields = _schema_fields(collection, base_dir=base_dir, roots=roots)
3615 if not fields:
3616 return record
3617
3618 clean = dict(record)
3619 for field in fields:
3620 name = field["name"]
3621 value = clean.get(name)
3622 if _is_empty(value):
3623 continue
3624 field_type = str(field.get("type") or "text").lower()
3625 if field_type in _BOOLEAN_TYPES:
3626 clean[name] = "true" if _parse_boolean(value, field_name=name) else "false"
3627 return clean
3628
3629
3630def _validate_record_against_schema(
3631 collection: str,
3632 record: dict[str, str],
3633 *,
3634 submitted_fields: frozenset[str],
3635 base_dir: Path | str,
3636 roots: Iterable[Path] | None,
3637 allow_read_only_submission: bool = False,
3638 allow_computed_submission: bool = False,
3639) -> None:
3640 fields = _schema_fields(collection, base_dir=base_dir, roots=roots)
3641 if not fields:
3642 return
3643
3644 for field in fields:
3645 name = field["name"]
3646 value = record.get(name, "")
3647
3648 if name in submitted_fields and _is_computed_or_read_only(field):
3649 is_computed = _is_computed_field(field)
3650 permitted = (
3651 (allow_computed_submission and is_computed)
3652 or (allow_read_only_submission and not is_computed)
3653 )
3654 if not permitted:
3655 raise InvalidRecordPayloadError(
3656 f"Record field '{name}' is computed or read-only and cannot be written"
3657 )
3658
3659 if _field_is_required(field) and not _is_computed_or_read_only(field) and _is_empty(value):
3660 raise InvalidRecordPayloadError(f"Record field '{name}' is required")
3661
3662 if _is_empty(value):
3663 continue
3664
3665 _validate_field_type(field, value)
3666 _validate_field_enum(field, value)
3667 _validate_field_rules(field, value)
3668 _validate_field_relation(field, value, base_dir=base_dir)
3669
3670
3671def _check_field_storable(record: dict[str, Any]) -> None:
3672 """Reject a write whose any field value cannot be safely stored in the
3673 canonical plain-text TSV substrate, surfacing every case as a clean
3674 InvalidRecordPayloadError (a 4xx) rather than letting a raw stdlib
3675 _csv.Error or UnicodeEncodeError escape the public API. Three unstorable
3676 classes:
3677
3678 - **Too large** (> MAX_TSV_FIELD_BYTES): without this, an oversize cell
3679 silently empties an append-mode collection on read.
3680 - **NUL (0x00)**: Python's csv writer literally cannot represent a NUL
3681 under QUOTE_ALL/MINIMAL ("need to escape, but no escapechar set"), and a
3682 NUL corrupts the text tooling the plain-text-durability guarantee rests
3683 on -- it is not text. Legitimate JSON escapes NUL to \\u0000, so real
3684 structured data never carries a raw one.
3685 - **Unencodable UTF-8** (a lone surrogate, e.g. from bad input or
3686 json.dumps(ensure_ascii=False) over broken data): can't be written at
3687 all.
3688
3689 Enforced on create/update (the entry points for new/changed data);
3690 internal rewrites (compaction, transition) don't pass here -- they only
3691 re-serialize data that already passed this gate. Iterates the record's own
3692 values, so it guards schemaless collections and the packed `extra` blob
3693 too.
3694 """
3695 for name, value in record.items():
3696 if value is None:
3697 continue
3698 text = str(value)
3699 if "\x00" in text:
3700 raise InvalidRecordPayloadError(
3701 f"Record field '{name}' contains a NUL byte, which cannot be "
3702 f"stored in the plain-text substrate (JSON encodes NUL as "
3703 f"\\u0000)"
3704 )
3705 try:
3706 size = len(text.encode("utf-8"))
3707 except UnicodeEncodeError as exc:
3708 raise InvalidRecordPayloadError(
3709 f"Record field '{name}' is not valid UTF-8 (unencodable "
3710 f"surrogate) and cannot be stored: {exc}"
3711 ) from exc
3712 if size > MAX_TSV_FIELD_BYTES:
3713 raise InvalidRecordPayloadError(
3714 f"Record field '{name}' is {size} bytes, exceeding the "
3715 f"{MAX_TSV_FIELD_BYTES}-byte per-field maximum"
3716 )
3717
3718
3719# A bare carriage return -- "\r" NOT immediately followed by "\n". csv's
3720# QUOTE_MINIMAL does not quote a field for a lone CR (it quotes only for the
3721# delimiter, the quotechar, and "\n"), but csv.reader treats a bare CR in an
3722# UNQUOTED field as a row terminator -- silently splitting one row into two
3723# (and, in append mode, hiding every later record from the fold path). "\r\n"
3724# and lone "\n" are already handled by QUOTE_MINIMAL (both contain "\n").
3725_BARE_CR_RE = re.compile(r"\r(?!\n)")
3726
3727
3728def _rows_need_full_quoting(rows: Iterable[dict[str, Any]]) -> bool:
3729 """True if any field value in `rows` contains a bare CR (see _BARE_CR_RE).
3730
3731 When it does, that write uses csv.QUOTE_ALL so every field is quoted and
3732 the CR round-trips intact; the overwhelming common case (no bare CR) stays
3733 QUOTE_MINIMAL and byte-identical to before. This keeps the compact
3734 plain-text format for normal data while making a lone CR lossless instead
3735 of silently corrupting -- no format change for anyone who never stores a
3736 bare CR. Readers parse quoted and unquoted rows alike, so a file may mix
3737 both. Note: values destined for the packed `extra` JSON blob have their CR
3738 escaped by json.dumps, so scanning raw values here only ever OVER-detects
3739 (a harmless extra-quoted write), never misses a CR that reaches a column.
3740 """
3741 for row in rows:
3742 for value in row.values():
3743 if value is None:
3744 continue
3745 text = str(value)
3746 if "\r" in text and _BARE_CR_RE.search(text):
3747 return True
3748 return False
3749
3750
3751def _schema_fields(
3752 collection: str,
3753 *,
3754 base_dir: Path | str,
3755 roots: Iterable[Path] | None,
3756) -> list[dict[str, Any]]:
3757 try:
3758 schema = object_schemas.get_schema(collection, base_dir=base_dir, roots=roots)
3759 except object_schemas.SchemaNotFoundError:
3760 return []
3761 fields = schema.get("fields", [])
3762 if not isinstance(fields, list):
3763 return []
3764 return [field for field in fields if isinstance(field, dict) and "name" in field]
3765
3766
3767def _field_is_required(field: dict[str, Any]) -> bool:
3768 validation = field.get("validation") if isinstance(field.get("validation"), dict) else {}
3769 return bool(field.get("required") or validation.get("required") or validation.get("not_null"))
3770
3771
3772def _is_computed_or_read_only(field: dict[str, Any]) -> bool:
3773 field_type = str(field.get("type", "")).lower()
3774 return bool(
3775 field_type == "computed"
3776 or field.get("computed")
3777 or field.get("read_only")
3778 or field.get("readonly")
3779 or field.get("readOnly")
3780 )
3781
3782
3783def _is_computed_field(field: dict[str, Any]) -> bool:
3784 """Return True for a server-derived formula field, as opposed to a
3785 field that is merely ``read_only`` (client can't set it, but nothing
3786 computes its value -- see ``preserve_read_only`` on
3787 ``create_collection_record``)."""
3788 field_type = str(field.get("type", "")).lower()
3789 return bool(field_type == "computed" or field.get("computed"))
3790
3791
3792def _is_empty(value: str | None) -> bool:
3793 return value is None or value == ""
3794
3795
3796def _schema_scalar_to_string(value: Any, *, field_name: str) -> str:
3797 if value is None:
3798 return ""
3799 if isinstance(value, (str, int, float, bool)):
3800 return str(value)
3801 raise InvalidRecordPayloadError(f"Schema default for '{field_name}' must be scalar or null")
3802
3803
3804def _validate_field_type(field: dict[str, Any], value: str) -> None:
3805 name = field["name"]
3806 field_type = str(field.get("type") or "text").lower()
3807 if field_type in _INTEGER_TYPES:
3808 _parse_integer(value, field_name=name)
3809 elif field_type in _FLOAT_TYPES:
3810 _parse_float(value, field_name=name)
3811 elif field_type in _BOOLEAN_TYPES:
3812 _parse_boolean(value, field_name=name)
3813 elif field_type == "date":
3814 _parse_date(value, field_name=name)
3815 elif field_type in {"datetime", "timestamp"}:
3816 _parse_datetime(value, field_name=name)
3817 elif field_type == "enum":
3818 _validate_field_enum(field, value, required=True)
3819
3820
3821def _validate_field_enum(field: dict[str, Any], value: str, *, required: bool = False) -> None:
3822 enum_payload = field.get("enum")
3823 validation = field.get("validation") if isinstance(field.get("validation"), dict) else {}
3824 if enum_payload is None:
3825 enum_payload = validation.get("choices", validation.get("in"))
3826 values = _enum_values(enum_payload)
3827 if not values:
3828 if required:
3829 raise InvalidRecordPayloadError(f"Record field '{field['name']}' enum has no values")
3830 return
3831 if value not in values:
3832 allowed = ", ".join(values)
3833 raise InvalidRecordPayloadError(
3834 f"Record field '{field['name']}' must be one of: {allowed}"
3835 )
3836
3837
3838def _enum_values(enum_payload: Any) -> list[str]:
3839 if enum_payload is None:
3840 return []
3841 if isinstance(enum_payload, dict):
3842 enum_payload = enum_payload.get("values")
3843 if not isinstance(enum_payload, list):
3844 return []
3845 values = []
3846 for item in enum_payload:
3847 if isinstance(item, (str, int, float, bool)):
3848 values.append(str(item))
3849 return values
3850
3851
3852def _validate_field_relation(
3853 field: dict[str, Any],
3854 value: str,
3855 *,
3856 base_dir: Path | str,
3857) -> None:
3858 """Require relation values to be existing record ids in the target collection.
3859
3860 A relation is a validated pointer plus a display hint — deliberately not
3861 an association framework: no joins, no lazy loading, no cascades.
3862 """
3863 relation = field.get("relation")
3864 if relation is None:
3865 return
3866
3867 name = field["name"]
3868 if isinstance(relation, str):
3869 target = relation
3870 elif isinstance(relation, dict):
3871 target = relation.get("collection")
3872 else:
3873 raise InvalidRecordPayloadError(
3874 f"Record field '{name}' relation must be a collection name or object"
3875 )
3876
3877 if not isinstance(target, str) or not object_collections.validate_collection_name(target):
3878 raise InvalidRecordPayloadError(
3879 f"Record field '{name}' relation has an invalid collection"
3880 )
3881
3882 try:
3883 get_collection_record(target, value, base_dir=base_dir)
3884 except (object_collections.CollectionNotFoundError, RecordNotFoundError) as exc:
3885 raise InvalidRecordPayloadError(
3886 f"Record field '{name}' references a missing record: {target}/{value}"
3887 ) from exc
3888 except InvalidRecordIdError as exc:
3889 raise InvalidRecordPayloadError(
3890 f"Record field '{name}' relation value is not a valid record id"
3891 ) from exc
3892
3893
3894def _validate_field_rules(field: dict[str, Any], value: str) -> None:
3895 validation = field.get("validation")
3896 if not isinstance(validation, dict):
3897 return
3898
3899 name = field["name"]
3900 if "min_length" in validation and len(value) < _rule_int(validation["min_length"], name, "min_length"):
3901 raise InvalidRecordPayloadError(f"Record field '{name}' is shorter than min_length")
3902 if "max_length" in validation and len(value) > _rule_int(validation["max_length"], name, "max_length"):
3903 raise InvalidRecordPayloadError(f"Record field '{name}' is longer than max_length")
3904
3905 pattern = validation.get("pattern", validation.get("regex"))
3906 if pattern is not None:
3907 if not isinstance(pattern, str):
3908 raise InvalidRecordPayloadError(f"Record field '{name}' validation pattern must be a string")
3909 try:
3910 matches = re.fullmatch(pattern, value)
3911 except re.error as exc:
3912 raise InvalidRecordPayloadError(
3913 f"Record field '{name}' validation pattern is invalid"
3914 ) from exc
3915 if not matches:
3916 raise InvalidRecordPayloadError(f"Record field '{name}' does not match pattern")
3917
3918 if "min" in validation:
3919 if _parse_float(value, field_name=name) < _rule_float(validation["min"], name, "min"):
3920 raise InvalidRecordPayloadError(f"Record field '{name}' is below min")
3921 if "max" in validation:
3922 if _parse_float(value, field_name=name) > _rule_float(validation["max"], name, "max"):
3923 raise InvalidRecordPayloadError(f"Record field '{name}' is above max")
3924
3925
3926def _parse_integer(value: str, *, field_name: str) -> int:
3927 try:
3928 if value.strip() != value or not re.fullmatch(r"[+-]?\d+", value):
3929 raise ValueError
3930 return int(value)
3931 except ValueError as exc:
3932 raise InvalidRecordPayloadError(f"Record field '{field_name}' must be an integer") from exc
3933
3934
3935def _parse_float(value: str, *, field_name: str) -> float:
3936 try:
3937 number = float(value)
3938 except ValueError as exc:
3939 raise InvalidRecordPayloadError(f"Record field '{field_name}' must be a number") from exc
3940 if not math.isfinite(number):
3941 raise InvalidRecordPayloadError(f"Record field '{field_name}' must be a finite number")
3942 return number
3943
3944
3945def _parse_boolean(value: str, *, field_name: str) -> bool:
3946 text = value.lower()
3947 if text in _TRUE_VALUES:
3948 return True
3949 if text in _FALSE_VALUES:
3950 return False
3951 raise InvalidRecordPayloadError(f"Record field '{field_name}' must be a boolean")
3952
3953
3954def _parse_date(value: str, *, field_name: str) -> date:
3955 try:
3956 return date.fromisoformat(value)
3957 except ValueError as exc:
3958 raise InvalidRecordPayloadError(f"Record field '{field_name}' must be a date") from exc
3959
3960
3961def _parse_datetime(value: str, *, field_name: str) -> datetime:
3962 try:
3963 return datetime.fromisoformat(value.replace("Z", "+00:00"))
3964 except ValueError as exc:
3965 raise InvalidRecordPayloadError(f"Record field '{field_name}' must be a datetime") from exc
3966
3967
3968def _rule_int(value: Any, field_name: str, rule_name: str) -> int:
3969 try:
3970 return int(value)
3971 except (TypeError, ValueError) as exc:
3972 raise InvalidRecordPayloadError(
3973 f"Record field '{field_name}' validation {rule_name} must be an integer"
3974 ) from exc
3975
3976
3977def _rule_float(value: Any, field_name: str, rule_name: str) -> float:
3978 try:
3979 number = float(value)
3980 except (TypeError, ValueError) as exc:
3981 raise InvalidRecordPayloadError(
3982 f"Record field '{field_name}' validation {rule_name} must be a number"
3983 ) from exc
3984 if not math.isfinite(number):
3985 raise InvalidRecordPayloadError(
3986 f"Record field '{field_name}' validation {rule_name} must be finite"
3987 )
3988 return number
3989
3990
3991def _extra_field_names(
3992 collection: str,
3993 *,
3994 base_dir: Path | str,
3995 roots: Iterable[Path] | None,
3996) -> set[str]:
3997 """Return the names of schema fields whose value lives inside ``extra``."""
3998 fields = _schema_fields(collection, base_dir=base_dir, roots=roots)
3999 return {field["name"] for field in fields if field.get("store") == EXTRA_FIELD}
4000
4001
4002def _parse_extra_blob(value: str | None) -> dict[str, Any]:
4003 """Parse an ``extra`` TSV cell into a dict, tolerating missing/invalid data."""
4004 if not value:
4005 return {}
4006 try:
4007 parsed = json.loads(value)
4008 except (TypeError, ValueError):
4009 return {}
4010 return parsed if isinstance(parsed, dict) else {}
4011
4012
4013def _route_extra(
4014 record: dict[str, str],
4015 *,
4016 existing_blob: dict[str, Any],
4017 extra_names: set[str],
4018) -> dict[str, str]:
4019 """Fold ``extra`` and any ``store: extra`` view fields into one JSON blob.
4020
4021 Pure: builds on ``existing_blob`` (so a partial update never wipes
4022 untouched blob keys), shallow-merges any client-submitted ``extra``
4023 object into it, then pulls each declared view field out of the
4024 top-level record and into the blob. The record keeps a single
4025 ``extra`` TSV column holding the serialized blob (or no column at all
4026 when the blob is empty), never the view field's own column.
4027 """
4028 blob = dict(existing_blob)
4029 if EXTRA_FIELD in record:
4030 blob.update(_parse_extra_blob(record.pop(EXTRA_FIELD)))
4031 for name in extra_names:
4032 if name in record:
4033 blob[name] = record.pop(name)
4034
4035 if blob:
4036 serialized = json.dumps(blob, sort_keys=True)
4037 if "\t" in serialized or "\n" in serialized or "\r" in serialized:
4038 raise InvalidRecordPayloadError("Record field 'extra' is not TSV-safe")
4039 record[EXTRA_FIELD] = serialized
4040 else:
4041 record.pop(EXTRA_FIELD, None)
4042 return record
4043
4044
4045def _surface_extra(record: dict[str, str], *, extra_names: set[str]) -> dict[str, str]:
4046 """Expose each declared ``store: extra`` view field at the top level.
4047
4048 Pure: ``record["extra"]`` is left as-is (still the raw JSON string) so
4049 undeclared blob data stays reachable. A no-op when there are no view
4050 fields and no ``extra`` column.
4051 """
4052 if not extra_names:
4053 return record
4054 blob = _parse_extra_blob(record.get(EXTRA_FIELD))
4055 for name in extra_names:
4056 record[name] = str(blob.get(name, ""))
4057 return record
4058
4059
4060def _merge_fields(existing_fields: list[str], record: dict[str, str]) -> list[str]:
4061 fields = list(existing_fields) if existing_fields else ["id"]
4062 if "id" not in fields:
4063 fields.insert(0, "id")
4064 for field in record:
4065 if field not in fields:
4066 fields.append(field)
4067 return fields
4068
4069
4070def _project_record(record: dict[str, str], fields: list[str]) -> dict[str, str]:
4071 return {field: record.get(field, "") for field in fields}
4072
4073
4074def _validate_page(*, limit: int, offset: int) -> None:
4075 if limit < 1:
4076 raise ValueError("Record limit must be at least 1")
4077 if limit > MAX_RECORD_LIMIT:
4078 raise ValueError(f"Record limit must be at most {MAX_RECORD_LIMIT}")
4079 if offset < 0:
4080 raise ValueError("Record offset must be at least 0")