askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_records.py · 175.9 KB · 4080 lines
| 1 | """TSV-backed collection records. |
| 2 | |
| 3 | Collection records are the simple data surface Scroll can point generated |
| 4 | tables and forms at. Records live in |
| 5 | ``data/collections/{collection}/records.tsv``. |
| 6 | """ |
| 7 | |
| 8 | from __future__ import annotations |
| 9 | |
| 10 | import csv |
| 11 | import hashlib |
| 12 | import io |
| 13 | import json |
| 14 | import math |
| 15 | import os |
| 16 | import re |
| 17 | import threading |
| 18 | from collections import OrderedDict |
| 19 | from contextlib import contextmanager |
| 20 | from datetime import date, datetime, timezone |
| 21 | from pathlib import Path |
| 22 | from typing import Any, Iterable, Mapping |
| 23 | |
| 24 | import object_collections |
| 25 | import object_computed |
| 26 | import object_correlation |
| 27 | import object_ids |
| 28 | import object_permissions |
| 29 | import object_record_changes |
| 30 | import object_schemas |
| 31 | from object_versions import DEFAULT_DATA_DIR |
| 32 | |
| 33 | COLLECTIONS_DIR = "collections" |
| 34 | RECORDS_FILE = "records.tsv" |
| 35 | DEFAULT_RECORD_LIMIT = 100 |
| 36 | MAX_RECORD_LIMIT = 1000 |
| 37 | EXTRA_FIELD = "extra" |
| 38 | |
| 39 | # Maximum bytes for a single TSV field value. Python's csv module defaults |
| 40 | # csv.field_size_limit() to 128 KiB; a single record field can legitimately |
| 41 | # exceed that (a long article body, a big template JSON, the embedded-items |
| 42 | # array of plan/vocabulary/66). Left unraised, a cell over 128 KiB is a |
| 43 | # silent data-loss bug: a classic-mode read raises an uncaught csv.Error, |
| 44 | # and an APPEND-mode read returns an EMPTY collection (the oversize row trips |
| 45 | # the torn-tail-tolerant `except csv.Error: break` in _parse_append_body, |
| 46 | # discarding every row from that point on). We raise the PARSE ceiling here |
| 47 | # and enforce the SAME ceiling on WRITE (_check_field_sizes, called from |
| 48 | # create/update) so no row is ever persisted that cannot be read back. 16 MiB |
| 49 | # matches the platform's max request body -- a field cannot exceed the |
| 50 | # request that wrote it. Per-surface limits (e.g. the 256 KiB line-items cap) |
| 51 | # are stricter and enforced above this hard floor. |
| 52 | MAX_TSV_FIELD_BYTES = 16 * 1024 * 1024 |
| 53 | csv.field_size_limit(MAX_TSV_FIELD_BYTES) |
| 54 | |
| 55 | _RECORD_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$") |
| 56 | |
| 57 | # Append-only storage (docs/append-only-storage-design.md). `OP_FIELD` is |
| 58 | # an INTERNAL, file-layer-only column: when present, it is always the |
| 59 | # first column of the physical header, and it never appears in a schema, |
| 60 | # an API response, or a record returned by any public function in this |
| 61 | # module -- it is stripped during parse-fold and only ever written by the |
| 62 | # append/rewrite helpers below. `OP_UPSERT` ("") marks a row that sets/ |
| 63 | # replaces a record; `OP_DELETE` ("del") marks a tombstone. |
| 64 | OP_FIELD = "_op" |
| 65 | OP_UPSERT = "" |
| 66 | OP_DELETE = "del" |
| 67 | |
| 68 | # Threshold (physical row count) above which a collection becomes eligible |
| 69 | # for auto-compaction: see _maybe_flag_auto_compact. |
| 70 | APPEND_COMPACT_MIN_ROWS_ENV = "DBBASIC_APPEND_COMPACT_MIN_ROWS" |
| 71 | _DEFAULT_APPEND_COMPACT_MIN_ROWS = 10_000 |
| 72 | |
| 73 | # id -> byte-offset sidecar (docs/append-only-storage-design.md item 4). |
| 74 | # Lives next to records.tsv as a dotfile so it is invisible to every glob |
| 75 | # this codebase uses to enumerate a collection's real content |
| 76 | # (iter_record_collections' `*/records.tsv` glob, object_collections' same |
| 77 | # pattern) and to backup (object_backup._should_skip drops any path with a |
| 78 | # dotfile part) -- both were checked, not just assumed. Format (text, |
| 79 | # greppable, torn-tail-tolerant like the data file itself): |
| 80 | # line 1 (header): "oidx1\t<data_ino>" -- the records.tsv inode this |
| 81 | # sidecar describes. No byte count lives in the header (a header |
| 82 | # rewrite-in-place would itself need the same torn-tail care as a data |
| 83 | # row, for no real benefit): "how much of the file is indexed" is |
| 84 | # instead DERIVED from the body, as the row_end_offset of the last |
| 85 | # complete data line -- see _load_oidx_body. |
| 86 | # subsequent lines: "<row_start_offset>\t<row_end_offset>\t<op>\t<id>", |
| 87 | # one per indexed PHYSICAL row of records.tsv (op "" or "del", exactly |
| 88 | # like OP_UPSERT/OP_DELETE). No CSV quoting is used or needed here: |
| 89 | # unlike a record's own field values, `id` is restricted to |
| 90 | # _RECORD_ID_RE (no tabs/newlines possible) and the offsets/op are |
| 91 | # plain digits/literals, so a bare tab-split is always unambiguous. |
| 92 | # The sidecar is 100% disposable and NEVER required for correctness: any |
| 93 | # reader unable to make sense of it (missing, corrupt, inode mismatch, |
| 94 | # torn tail beyond what a catch-up scan can resolve) rebuilds it with one |
| 95 | # sequential scan of records.tsv rather than raising -- see _load_oidx. |
| 96 | OIDX_FILE = ".records.oidx" |
| 97 | OIDX_HEADER_TAG = "oidx1" |
| 98 | |
| 99 | OIDX_CACHE_MAX_ENTRIES_ENV = "DBBASIC_OIDX_CACHE_MAX_ENTRIES" |
| 100 | _DEFAULT_OIDX_CACHE_MAX_ENTRIES = 2 |
| 101 | |
| 102 | # Env knobs for the records cache below. Read at call time (not cached in a |
| 103 | # module global) via _records_cache_max_entries()/_records_cache_max_rows() |
| 104 | # so tests can monkeypatch os.environ and see the change take effect on the |
| 105 | # next cache operation without reimporting the module. |
| 106 | RECORDS_CACHE_MAX_ENTRIES_ENV = "DBBASIC_RECORDS_CACHE_MAX_ENTRIES" |
| 107 | RECORDS_CACHE_MAX_ROWS_ENV = "DBBASIC_RECORDS_CACHE_MAX_ROWS" |
| 108 | _DEFAULT_RECORDS_CACHE_MAX_ENTRIES = 64 |
| 109 | _DEFAULT_RECORDS_CACHE_MAX_ROWS = 500_000 |
| 110 | |
| 111 | # Module-level cache for parsed records.tsv files, keyed by the resolved |
| 112 | # file path. An OrderedDict bounded to DBBASIC_RECORDS_CACHE_MAX_ENTRIES |
| 113 | # entries (default 64): every cache hit calls move_to_end() so the dict |
| 114 | # stays ordered oldest-to-newest by use, and storing a new entry evicts the |
| 115 | # oldest (popitem(last=False)) once the dict exceeds capacity -- see |
| 116 | # _store_records_cache. Without a bound, a long-running server that touches |
| 117 | # many collections over its lifetime (e.g. a no-downtime game server) would |
| 118 | # pin every collection it has ever parsed in RAM forever; the LRU keeps |
| 119 | # only the most-recently-used entries resident. |
| 120 | # |
| 121 | # A second knob, DBBASIC_RECORDS_CACHE_MAX_ROWS (default 500_000), bounds |
| 122 | # the cache by size rather than recency: a collection whose parse produces |
| 123 | # more rows than this is still returned to its caller normally, but is |
| 124 | # never stored in (or left in) the cache. Without this, reading one huge |
| 125 | # collection once would occupy a full LRU slot and could evict many small, |
| 126 | # hot collections just to make room for something that will likely never |
| 127 | # be reused whole. _cache_entry and _refresh_records_cache both apply this |
| 128 | # threshold before writing to _RECORDS_CACHE; _cache_entry additionally |
| 129 | # drops any stale entry already cached for a path that has since grown |
| 130 | # past the threshold, so an oversized collection never lingers there under |
| 131 | # an old signature. |
| 132 | # |
| 133 | # Value is (stat_signature, fields, records, id_index, covered_bytes): |
| 134 | # - stat_signature = (st_mtime_ns, st_size, st_ino). A CLASSIC-mode write |
| 135 | # goes through an atomic tempfile-then-replace, so a new inode backs the |
| 136 | # path after each write; including st_ino means a same-tick mtime |
| 137 | # collision (coarse clocks/filesystems) still isn't mistaken for |
| 138 | # "unchanged" as long as the replace produced a different inode, which |
| 139 | # is the common case for both our own writes and any other well-behaved |
| 140 | # writer using the same replace pattern. |
| 141 | # APPEND-mode writes (docs/append-only-storage-design.md) are the |
| 142 | # deliberate exception: a fast append opens the SAME inode in "a" mode |
| 143 | # and writes one more line, so st_ino is unchanged and only st_mtime_ns |
| 144 | # / st_size move. That's still sufficient to invalidate a stale cache |
| 145 | # entry (the signature tuple as a whole changes), it just means "same |
| 146 | # inode" is no longer proof of "unchanged content" the way it was |
| 147 | # before this feature -- st_size growing is what a reader actually |
| 148 | # relies on to notice a completed append. A torn tail (a final physical |
| 149 | # line with no trailing newline, from a write interrupted mid-append) |
| 150 | # is never cached as data: the append-mode parser drops it, and the |
| 151 | # next append's self-heal (_repair_torn_tail) removes it from disk |
| 152 | # before adding a new line, so it can never resurface as a stale-but- |
| 153 | # plausible cached row either. |
| 154 | # - fields/records mirror exactly what a fresh parse of that exact file |
| 155 | # content would produce (see _refresh_records_cache: written rows are |
| 156 | # projected to the full field set before caching, matching what |
| 157 | # csv.DictWriter/csv.reader would round-trip). For an append-mode file |
| 158 | # this means the FOLDED (last-wins-by-id, tombstones removed, `_op` |
| 159 | # stripped) view -- see _parse_append_body/_fold_append_rows -- never |
| 160 | # the raw physical rows; everything below this cache (get, list, |
| 161 | # window-copy, extra-field surfacing) operates on folded records with |
| 162 | # zero changes, exactly as it does for a classic-mode file. |
| 163 | # - id_index maps record id -> index into `records`, first-occurrence-wins |
| 164 | # (matching the original linear scan's behavior on hand-edited files |
| 165 | # with duplicate ids). For append mode this is built from the already- |
| 166 | # deduplicated folded records, so in practice every live id maps to |
| 167 | # exactly one position. |
| 168 | # - covered_bytes is the byte offset (from 0) that this entry's fold |
| 169 | # already accounts for -- always equal to the file's size at the |
| 170 | # stat this entry was built from (signature[1]), for every entry |
| 171 | # produced by a full parse or a full rewrite (_cache_entry's |
| 172 | # full-parse branch, _refresh_records_cache, _refresh_records_cache_ |
| 173 | # after_append). It exists so _cache_entry's INCREMENTAL branch |
| 174 | # (docs/append-only-storage-design.md Sidecars, bullet 2: "cache |
| 175 | # entries remember the byte offset they consumed; when a stat |
| 176 | # signature changes by growth alone, parse only the tail delta") can |
| 177 | # tell how much of a GROWN append-mode file is already folded into |
| 178 | # `records`, so only the new bytes need parsing -- see |
| 179 | # _try_incremental_cache_entry. A CLASSIC-mode entry carries this |
| 180 | # field too (harmless: it's just the file's size at cache time, |
| 181 | # unused by anything) since a classic rewrite always produces a new |
| 182 | # inode, which alone rules out the incremental branch regardless of |
| 183 | # covered_bytes. Only an incrementally-built entry can end up with |
| 184 | # covered_bytes < signature[1] -- when the tail delta itself ends in |
| 185 | # a torn (uncommitted) row, that fragment's bytes are correctly left |
| 186 | # out of both `records` and covered_bytes, exactly as a full parse |
| 187 | # would also drop them (_drop_torn_tail); a later append still |
| 188 | # compares against this same covered_bytes to find its own delta. |
| 189 | # |
| 190 | # ALIASING SAFETY: the `records` list and its dicts stored here are shared, |
| 191 | # mutable module state. Nothing in this module may mutate them in place. |
| 192 | # Every function that hands a record or records list back across the |
| 193 | # public API must copy first (see _read_collection_records, |
| 194 | # get_collection_record, list_collection_records). Internal callers that |
| 195 | # need to build on cached data for a write (create/update/delete) read |
| 196 | # `_cache_entry`'s tuple directly -- fine as long as they only ever READ |
| 197 | # from `records_ref`/`id_index_ref` (e.g. an O(1) duplicate/lookup check) |
| 198 | # or capture them in a closure that builds a fresh, independent copy on |
| 199 | # demand (see each function's `build_folded_records`, and _persist_write's |
| 200 | # `folded_records_fn`, which calls it only on a full-rewrite path -- never |
| 201 | # on the fast-append path, which is what keeps a steady-state append O(1) |
| 202 | # instead of paying an eager O(n) copy on every single write). |
| 203 | # |
| 204 | # CONCURRENCY (readers hold no lock; writers hold _records_file_lock only |
| 205 | # around their own read-modify-write): a reader's path is stat -> dict.get |
| 206 | # -> compare signature -> return (falling through to parse+store on a |
| 207 | # miss). A writer's path builds a complete new (signature, fields, records, |
| 208 | # id_index) tuple off to the side and installs it with a single dict |
| 209 | # assignment (see _store_records_cache). That assignment is one atomic |
| 210 | # store under the GIL, so a concurrent reader's dict.get() always observes |
| 211 | # either the entry from before the write or the entry from after it in |
| 212 | # full -- never a torn mix of old fields with new records, or similar. |
| 213 | # |
| 214 | # The only reachable skew is an old signature serving newer content: a |
| 215 | # reader stats the file (capturing the OLD signature), a writer's replace() |
| 216 | # lands, and only then does the reader open+parse the file via its own |
| 217 | # fresh fd. Because replace() is atomic and that fd is opened after the |
| 218 | # stat, the fd sees either the pre- or post-write content in full, never a |
| 219 | # mix -- so if it lands on the post-write content, what the reader parses |
| 220 | # is complete and correct, it's just paired with a signature it captured a |
| 221 | # moment too early. That mismatch means this parse won't be stored under a |
| 222 | # key matching what's already cached, so it simply fails every future |
| 223 | # signature comparison and self-heals with one extra reparse next time |
| 224 | # (typically immediately, since the writer's own _refresh_records_cache |
| 225 | # call installs the current signature right after the write completes). |
| 226 | # The reverse -- a new signature paired with old content -- is structurally |
| 227 | # impossible: content is always read at or after the stat that produced |
| 228 | # its signature, never before. |
| 229 | # |
| 230 | # Cross-process coherence relies on every writer in this codebase using the |
| 231 | # same temp-file + atomic-rename pattern, so a fresh inode backs every |
| 232 | # write (st_ino is part of the signature): this module's own |
| 233 | # _write_collection_records (below), object_packages.py's package-install |
| 234 | # writer (~line 1022, os.replace), and object_backup.py's restore writer |
| 235 | # (~line 607, Path.replace) were all checked and write via a temp path |
| 236 | # followed by an atomic replace, never an in-place open("w"). |
| 237 | _RECORDS_CACHE: "OrderedDict[str, tuple[tuple[int, int, int], list[str], list[dict[str, str]], dict[str, int], int]]" = OrderedDict() |
| 238 | |
| 239 | # Observability-only counter: incremented once per successful tail-delta |
| 240 | # incremental fold in _cache_entry (see _try_incremental_cache_entry). |
| 241 | # Never read by any runtime code path -- it exists purely so tests can |
| 242 | # assert the incremental path actually ran (rather than a full re-parse) |
| 243 | # without needing to monkeypatch internals just to observe that. |
| 244 | _INCREMENTAL_CACHE_FOLDS = 0 |
| 245 | |
| 246 | # In-memory mirror of the on-disk id->offset sidecar (see OIDX_FILE above), |
| 247 | # keyed by resolved records.tsv path. Value is (data_ino, covered_bytes, |
| 248 | # id->row_start_offset): `covered_bytes` is how much of the file (from |
| 249 | # byte 0) this dict accounts for -- see _load_oidx, the sole reader AND |
| 250 | # writer of this dict. A DELIBERATELY SEPARATE, much smaller LRU than |
| 251 | # _RECORDS_CACHE (default 2 entries, not 64): the two caches serve |
| 252 | # opposite size regimes -- _RECORDS_CACHE holds fully-parsed records and |
| 253 | # is capped by ROW COUNT (never stores anything over |
| 254 | # DBBASIC_RECORDS_CACHE_MAX_ROWS), while this sidecar is only ever |
| 255 | # consulted for collections ABOVE that same threshold (see |
| 256 | # _fast_record_lookup: it's tried only on a _RECORDS_CACHE miss), so an |
| 257 | # entry here is exactly the case _RECORDS_CACHE refuses to hold. A plain |
| 258 | # {id: int} dict is far lighter per row than a full record dict, but at |
| 259 | # millions of rows it is still tens of MB (roughly 80-120MB at 1M ids per |
| 260 | # the design doc) -- sharing _RECORDS_CACHE's 64-entry budget would let a |
| 261 | # few huge collections' sidecars alone occupy gigabytes. Bounded instead |
| 262 | # by DBBASIC_OIDX_CACHE_MAX_ENTRIES (default 2): a server touching more |
| 263 | # than a couple of huge append collections concurrently pays a reload |
| 264 | # (still just a sidecar-body read, not a records.tsv fold) rather than |
| 265 | # holding every one of them resident forever. |
| 266 | _OIDX_CACHE: "OrderedDict[str, tuple[int, int, dict[str, int]]]" = OrderedDict() |
| 267 | |
| 268 | # Collections (by resolved records.tsv path) flagged during a parse/fold as |
| 269 | # having enough superseded+deleted rows to be worth compacting -- see |
| 270 | # _maybe_flag_auto_compact. Consulted (and cleared) by _persist_write on |
| 271 | # that collection's NEXT write, which performs a compacting rewrite instead |
| 272 | # of a plain append. Never acted on from a read path: setting membership |
| 273 | # here is the only thing a read ever does, and it is not a disk write. |
| 274 | _PENDING_COMPACTION: set[str] = set() |
| 275 | _INTEGER_TYPES = {"int", "integer"} |
| 276 | _FLOAT_TYPES = {"float", "number", "currency"} |
| 277 | _BOOLEAN_TYPES = {"bool", "boolean"} |
| 278 | _TRUE_VALUES = {"true", "1", "yes", "on"} |
| 279 | _FALSE_VALUES = {"false", "0", "no", "off"} |
| 280 | |
| 281 | |
| 282 | class InvalidRecordIdError(ValueError): |
| 283 | """Raised when a record id is not safe for routes or lookup.""" |
| 284 | |
| 285 | |
| 286 | class RecordNotFoundError(LookupError): |
| 287 | """Raised when a record cannot be found in a collection.""" |
| 288 | |
| 289 | |
| 290 | class DuplicateRecordIdError(ValueError): |
| 291 | """Raised when a create would reuse an existing record id.""" |
| 292 | |
| 293 | |
| 294 | class InvalidRecordPayloadError(ValueError): |
| 295 | """Raised when a record write payload is not usable.""" |
| 296 | |
| 297 | |
| 298 | class TransitionNotAllowedError(InvalidRecordPayloadError): |
| 299 | """Raised when a transition's value is valid but its guard denies the subject. |
| 300 | |
| 301 | Subclasses InvalidRecordPayloadError so callers that don't distinguish |
| 302 | the two keep working (still a 400-shaped payload error); callers that |
| 303 | plumb a subject into update_collection_record (see |
| 304 | _validate_field_transitions) can catch this specifically to report a |
| 305 | permission-style 403 instead. |
| 306 | """ |
| 307 | |
| 308 | |
| 309 | class VersionConflictError(Exception): |
| 310 | """Raised when an update's ``expected_rev`` precondition does not match |
| 311 | the record's current content (63 optimistic concurrency, |
| 312 | plan/vocabulary/63-concurrency-spec.md). Deliberately NOT a subclass of |
| 313 | InvalidRecordPayloadError/ValueError so the generic write-error clauses |
| 314 | that map to 400 never swallow it -- the server maps it to a distinct |
| 315 | 409, mirroring how TransitionNotAllowedError maps to 403. |
| 316 | """ |
| 317 | |
| 318 | |
| 319 | # 63: the wire name for the content fingerprint. ETag-shaped -- opaque to |
| 320 | # callers, compared for equality only, never parsed, never a schema field. |
| 321 | REV_FIELD = "_rev" |
| 322 | |
| 323 | |
| 324 | def compute_record_rev(record: Mapping[str, Any]) -> str: |
| 325 | """Return an opaque content fingerprint of a record's field values. |
| 326 | |
| 327 | The version signal for optimistic concurrency (63): a deterministic |
| 328 | hash of the record's current fields, computed for free from the |
| 329 | surfaced/projected row already in hand on every read and, inside the |
| 330 | write lock, from the freshly-read row at update time -- no stored |
| 331 | column, no migration, works retroactively on every record. Excludes |
| 332 | REV_FIELD itself so echoing a read's ``_rev`` back into a full-record |
| 333 | round-trip never changes the fingerprint. Sorted keys + a stable JSON |
| 334 | encoding so serialization noise (key order, whitespace) never makes two |
| 335 | logically-identical records disagree. ``default=str`` keeps mixed-type |
| 336 | surfaced values (extra-blob JSON: ints, lists) hashable deterministically. |
| 337 | """ |
| 338 | payload = json.dumps( |
| 339 | {k: v for k, v in record.items() if k != REV_FIELD}, |
| 340 | sort_keys=True, |
| 341 | ensure_ascii=False, |
| 342 | separators=(",", ":"), |
| 343 | default=str, |
| 344 | ) |
| 345 | return hashlib.sha256(payload.encode("utf-8")).hexdigest() |
| 346 | |
| 347 | |
| 348 | def validate_record_id(record_id: str) -> bool: |
| 349 | """Return True when a record id is route-safe.""" |
| 350 | if not isinstance(record_id, str): |
| 351 | return False |
| 352 | return bool(_RECORD_ID_RE.fullmatch(record_id)) |
| 353 | |
| 354 | |
| 355 | def normalize_record_payload(payload: dict[str, Any], *, require_id: bool = False) -> dict[str, str]: |
| 356 | """Return a TSV-safe record payload with scalar values converted to strings.""" |
| 357 | return _normalize_record_payload(payload, require_id=require_id) |
| 358 | |
| 359 | |
| 360 | def list_collection_records( |
| 361 | collection: str, |
| 362 | *, |
| 363 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 364 | roots: Iterable[Path] | None = None, |
| 365 | limit: int = DEFAULT_RECORD_LIMIT, |
| 366 | offset: int = 0, |
| 367 | where: Mapping[str, Any] | None = None, |
| 368 | ) -> dict[str, Any]: |
| 369 | """Return a paginated record list for one collection. |
| 370 | |
| 371 | Copies only the window (records[offset:offset + limit]) rather than |
| 372 | the whole collection first: callers of a paginated list only ever |
| 373 | consume a small page, and copying every record before slicing it away |
| 374 | (the straightforward implementation) dominates cost on large |
| 375 | collections for a window that is thrown away 99% copied. `total` is |
| 376 | computed from the cache's own record count, not from the copied |
| 377 | window. |
| 378 | |
| 379 | `where` (plan/vocabulary/58-query-filter-spec.md) is an optional |
| 380 | normalized field filter, evaluated with `filter_records` -- see there |
| 381 | for the shape. It is purely a field filter with no permission |
| 382 | awareness of its own: callers that also need row-level permission |
| 383 | filtering (object_server's HTTP/MCP surfaces) apply that FIRST and |
| 384 | pass only the already-permitted records through the same evaluator |
| 385 | (`filter_records`), so a field filter can only narrow what a caller |
| 386 | already decided is readable, never widen it -- this function's own |
| 387 | `where` follows the identical narrow-only contract, just without a |
| 388 | row filter of its own to narrow. When `where` is given, the windowing |
| 389 | optimization above no longer applies (a filtered window can't be |
| 390 | known without first testing every row), so the whole collection is |
| 391 | read and filtered before slicing -- same O(rows) cost as any other |
| 392 | full-collection read (see 58's Storage section). |
| 393 | """ |
| 394 | _ensure_collection_known(collection, base_dir=base_dir, roots=roots) |
| 395 | _validate_page(limit=limit, offset=offset) |
| 396 | |
| 397 | records_ref = _cached_records_ref(collection, base_dir=base_dir) |
| 398 | extra_names = _extra_field_names(collection, base_dir=base_dir, roots=roots) |
| 399 | |
| 400 | if where: |
| 401 | matching = [ |
| 402 | _surface_extra(dict(record), extra_names=extra_names) |
| 403 | for record in records_ref |
| 404 | if object_permissions.record_matches_filter( |
| 405 | record, where, object_permissions.PermissionSubject.anonymous() |
| 406 | ) |
| 407 | ] |
| 408 | total = len(matching) |
| 409 | window = matching[offset:offset + limit] |
| 410 | return _build_records_payload(collection, window, total=total, limit=limit, offset=offset) |
| 411 | |
| 412 | total = len(records_ref) |
| 413 | # Copy + surface only the window's records: entries in `records_ref` |
| 414 | # are the module cache's own dicts (see _RECORDS_CACHE ALIASING |
| 415 | # SAFETY above) and _surface_extra mutates its argument in place, so |
| 416 | # each must be copied before use -- but only the rows inside the |
| 417 | # window, never the rest of the collection. |
| 418 | window = [ |
| 419 | _surface_extra(dict(record), extra_names=extra_names) |
| 420 | for record in records_ref[offset:offset + limit] |
| 421 | ] |
| 422 | return _build_records_payload(collection, window, total=total, limit=limit, offset=offset) |
| 423 | |
| 424 | |
| 425 | def filter_records( |
| 426 | records: Iterable[Mapping[str, Any]], |
| 427 | where: Mapping[str, Any] | None, |
| 428 | ) -> list[dict[str, Any]]: |
| 429 | """Return the subset of `records` matching a normalized field filter. |
| 430 | |
| 431 | Pure and in-memory: no read, no schema/permission awareness of its |
| 432 | own. Reuses `object_permissions.record_matches_filter` -- the same |
| 433 | flat-condition evaluator that backs guarded transitions and |
| 434 | permission row filters -- so there is exactly one filter evaluator in |
| 435 | the codebase (plan/vocabulary/58-query-filter-spec.md). |
| 436 | |
| 437 | Callers that also apply a permission row filter MUST do so first and |
| 438 | pass only its output here: this function can only remove rows from |
| 439 | what it is given, never add any back, so "row filter's output is |
| 440 | this function's input" is what structurally guarantees a field |
| 441 | filter can narrow the readable set but never widen it (58's |
| 442 | Permissions Posture). |
| 443 | """ |
| 444 | if not where: |
| 445 | return [dict(record) for record in records] |
| 446 | anonymous = object_permissions.PermissionSubject.anonymous() |
| 447 | return [ |
| 448 | dict(record) |
| 449 | for record in records |
| 450 | if object_permissions.record_matches_filter(record, where, anonymous) |
| 451 | ] |
| 452 | |
| 453 | |
| 454 | _ORDERED_FILTER_OPERATORS = frozenset({"gte", "lte", "gt", "lt"}) |
| 455 | _ORDERED_FILTERABLE_TYPES = _INTEGER_TYPES | _FLOAT_TYPES | {"date", "datetime", "timestamp"} |
| 456 | |
| 457 | |
| 458 | def normalize_filter_value(field: dict[str, Any] | None, op: str, value: str) -> str: |
| 459 | """Validate one query-filter condition's value against a schema field |
| 460 | and return its canonical stored-string form. |
| 461 | |
| 462 | `field` is the schema field definition (or None for a schemaless/ |
| 463 | derived collection, which has no type to validate against and is |
| 464 | treated as permissive -- consistent with every other schema-optional |
| 465 | check in this module). Raises InvalidRecordPayloadError -- the same |
| 466 | exception schema validation already raises -- naming the field on a |
| 467 | bad type, or on an ordered operator (gte/lte/gt/lt) against a field |
| 468 | type that doesn't support ordered comparison (58 restricts those to |
| 469 | date/datetime/integer/number fields). Reuses `_validate_field_type`, |
| 470 | the same per-type parser create/update already validates against, so |
| 471 | there is exactly one value-type validator, not a second one for |
| 472 | filters. |
| 473 | |
| 474 | A boolean field's value is normalized to "true"/"false" -- the same |
| 475 | canonical form `_canonicalize_schema_values` stores on write -- so an |
| 476 | `eq`/`ne` filter written as `?is_public=1` still string-matches a |
| 477 | stored "true" rather than silently never matching. |
| 478 | """ |
| 479 | if field is None: |
| 480 | return value |
| 481 | |
| 482 | name = field.get("name", "") |
| 483 | field_type = str(field.get("type") or "text").lower() |
| 484 | if op in _ORDERED_FILTER_OPERATORS and field_type not in _ORDERED_FILTERABLE_TYPES: |
| 485 | raise InvalidRecordPayloadError( |
| 486 | f"Record field '{name}' does not support operator '{op}' " |
| 487 | f"(only date, datetime, integer, and number fields support " |
| 488 | f"ordered comparison)" |
| 489 | ) |
| 490 | _validate_field_type(field, value) |
| 491 | if field_type in _BOOLEAN_TYPES: |
| 492 | return "true" if _parse_boolean(value, field_name=name) else "false" |
| 493 | return value |
| 494 | |
| 495 | |
| 496 | def read_collection_records( |
| 497 | collection: str, |
| 498 | *, |
| 499 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 500 | roots: Iterable[Path] | None = None, |
| 501 | ) -> list[dict[str, str]]: |
| 502 | """Return all records for one known collection.""" |
| 503 | _ensure_collection_known(collection, base_dir=base_dir, roots=roots) |
| 504 | records = _read_collection_records(collection, base_dir=base_dir) |
| 505 | extra_names = _extra_field_names(collection, base_dir=base_dir, roots=roots) |
| 506 | return [_surface_extra(record, extra_names=extra_names) for record in records] |
| 507 | |
| 508 | |
| 509 | def collection_records_payload( |
| 510 | collection: str, |
| 511 | records: list[dict[str, str]], |
| 512 | *, |
| 513 | limit: int = DEFAULT_RECORD_LIMIT, |
| 514 | offset: int = 0, |
| 515 | ) -> dict[str, Any]: |
| 516 | """Return the standard paginated record-list payload shape. |
| 517 | |
| 518 | Takes a full records list and slices the window out of it here -- |
| 519 | unlike list_collection_records, which slices before copying. This |
| 520 | public entry point's contract is "you already have the full list"; |
| 521 | callers (e.g. object_server's search/permission-filtered list paths) |
| 522 | rely on that. |
| 523 | """ |
| 524 | _validate_page(limit=limit, offset=offset) |
| 525 | total = len(records) |
| 526 | window = records[offset:offset + limit] |
| 527 | return _build_records_payload(collection, window, total=total, limit=limit, offset=offset) |
| 528 | |
| 529 | |
| 530 | def _build_records_payload( |
| 531 | collection: str, |
| 532 | window: list[dict[str, str]], |
| 533 | *, |
| 534 | total: int, |
| 535 | limit: int, |
| 536 | offset: int, |
| 537 | ) -> dict[str, Any]: |
| 538 | return { |
| 539 | "collection": collection, |
| 540 | "records": window, |
| 541 | "count": len(window), |
| 542 | "total": total, |
| 543 | "limit": limit, |
| 544 | "offset": offset, |
| 545 | "has_more": offset + len(window) < total, |
| 546 | } |
| 547 | |
| 548 | |
| 549 | def get_collection_record( |
| 550 | collection: str, |
| 551 | record_id: str, |
| 552 | *, |
| 553 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 554 | roots: Iterable[Path] | None = None, |
| 555 | ) -> dict[str, Any]: |
| 556 | """Return one record by its ``id`` column.""" |
| 557 | _ensure_collection_known(collection, base_dir=base_dir, roots=roots) |
| 558 | if not validate_record_id(record_id): |
| 559 | raise InvalidRecordIdError(f"Invalid record id: {record_id}") |
| 560 | |
| 561 | extra_names = _extra_field_names(collection, base_dir=base_dir, roots=roots) |
| 562 | path = collection_records_file(collection, base_dir=base_dir) |
| 563 | # _fast_record_lookup tries the warm cache, then (append mode, over |
| 564 | # the cache's row threshold) the id->offset sidecar, before falling |
| 565 | # back to a full fold -- see its docstring. `existing_row`, when |
| 566 | # returned, is always already a fresh, independent copy. |
| 567 | _, exists, existing_row, _, _ = _fast_record_lookup( |
| 568 | collection, path, record_id, need_row=True |
| 569 | ) |
| 570 | if not exists: |
| 571 | raise RecordNotFoundError(f"Record not found: {collection}/{record_id}") |
| 572 | |
| 573 | return _surface_extra(existing_row, extra_names=extra_names) |
| 574 | |
| 575 | |
| 576 | def create_collection_record( |
| 577 | collection: str, |
| 578 | record: dict[str, Any], |
| 579 | *, |
| 580 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 581 | roots: Iterable[Path] | None = None, |
| 582 | actor: str | None = None, |
| 583 | preserve_read_only: bool = False, |
| 584 | allow_computed_submission: bool = False, |
| 585 | record_changes: bool = True, |
| 586 | ) -> dict[str, str]: |
| 587 | """Append one record to a collection TSV and return the stored row. |
| 588 | |
| 589 | `record_changes=False` skips the change-log entry, and exists for |
| 590 | exactly one caller shape: a full recompute writing DERIVED rows. A |
| 591 | rollup target's history is not "who did what" -- it is the source |
| 592 | collection plus the definition, both of which are still there, so the |
| 593 | log adds nothing anyone could audit while costing one entry per row |
| 594 | per pass, forever. One such log reached 944MB describing a collection |
| 595 | of 1818 rows, and the activity feed that reads every log stopped |
| 596 | finishing. Do NOT reach for this to make an ordinary write quieter: |
| 597 | a write nobody can trace is the thing the log exists to prevent. |
| 598 | |
| 599 | Every successful write is durably attributed here (universal |
| 600 | attribution -- every mutation, on every path, emits a record change). |
| 601 | ``actor`` identifies who/what caused the write; callers that omit it |
| 602 | are logged as ``"unattributed"`` rather than silently skipped, so gaps |
| 603 | stay visible instead of disappearing. See object_record_changes for |
| 604 | the append itself and its recursion note. |
| 605 | |
| 606 | ``preserve_read_only``: an ordinary write (client-submitted payload) |
| 607 | may never set a ``read_only`` field (e.g. a schema's ``created_at``) -- |
| 608 | the server owns it, per ``_apply_auto_created_at``. A bulk loader |
| 609 | replaying another system's history (object_import.py) legitimately |
| 610 | needs to carry that system's own ``created_at`` through instead of |
| 611 | stamping "now", which is exactly what a hand-typed form must never be |
| 612 | able to do. Setting this narrows, not removes, the read-only |
| 613 | protection: a genuinely ``computed`` field (a server-derived formula, |
| 614 | not just a field the client shouldn't touch) still rejects a |
| 615 | client-submitted value regardless of this flag -- see |
| 616 | ``_validate_record_against_schema``. Trusted, explicit, opt-in callers |
| 617 | only; never wired to an HTTP request payload. |
| 618 | |
| 619 | ``allow_computed_submission`` is the narrower sibling ``preserve_read_only`` |
| 620 | deliberately does not grant: a field marked ``computed`` (e.g. a rollup |
| 621 | target's metric columns, ``plan/vocabulary/14-rollup-spec.md``) is meant |
| 622 | to hold a server-derived formula's result, and no HTTP-facing caller |
| 623 | should ever be able to write one directly. The one legitimate writer is |
| 624 | the block that owns the derivation itself (the rollup daemon pass |
| 625 | writing its own computed metric columns into its own target row) -- |
| 626 | trusted, explicit, opt-in, same posture as ``preserve_read_only``, never |
| 627 | wired to a request payload. |
| 628 | """ |
| 629 | _ensure_collection_known(collection, base_dir=base_dir, roots=roots) |
| 630 | extra_names = _extra_field_names(collection, base_dir=base_dir, roots=roots) |
| 631 | clean = _normalize_record_payload(record, require_id=True) |
| 632 | record_id = clean["id"] |
| 633 | if not validate_record_id(record_id): |
| 634 | raise InvalidRecordIdError(f"Invalid record id: {record_id}") |
| 635 | submitted_fields = frozenset(clean) |
| 636 | clean = _apply_schema_defaults(collection, clean, base_dir=base_dir, roots=roots) |
| 637 | _check_field_storable(clean) |
| 638 | clean = _apply_auto_created_at(collection, clean, submitted_fields, base_dir=base_dir, roots=roots) |
| 639 | clean = _apply_computed_fields(collection, clean, base_dir=base_dir, roots=roots) |
| 640 | _validate_record_against_schema( |
| 641 | collection, |
| 642 | clean, |
| 643 | submitted_fields=submitted_fields, |
| 644 | base_dir=base_dir, |
| 645 | roots=roots, |
| 646 | allow_read_only_submission=preserve_read_only, |
| 647 | allow_computed_submission=allow_computed_submission, |
| 648 | ) |
| 649 | clean = _canonicalize_schema_values(collection, clean, base_dir=base_dir, roots=roots) |
| 650 | clean = _route_extra(clean, existing_blob={}, extra_names=extra_names) |
| 651 | |
| 652 | path = collection_records_file(collection, base_dir=base_dir) |
| 653 | with _records_file_lock(path): |
| 654 | # _fast_record_lookup resolves the duplicate check via the |
| 655 | # cheapest source available: warm cache, then (append mode, over |
| 656 | # threshold) the id->offset sidecar -- neither of which needs a |
| 657 | # full fold or a copy of every existing record. `records_ref`/ |
| 658 | # `id_index_ref` are the module cache's own uncopied objects when |
| 659 | # sourced from the cache (or a forced full fold below) -- fine to |
| 660 | # read (the O(1) duplicate check already happened above) or |
| 661 | # capture in a closure that builds a fresh copy lazily, only if a |
| 662 | # full rewrite turns out to be needed (see _persist_write's |
| 663 | # folded_records_fn). Skipping an eager copy of every existing |
| 664 | # record is what keeps a fast-append create O(1) rather than |
| 665 | # O(n): an `any(row.get("id") == ... for row in records)` linear |
| 666 | # scan plus a full deep-copy of every existing record were, |
| 667 | # together, the dominant cost of a "fast" append in an earlier |
| 668 | # version of this function -- and, before the sidecar, the O(n) |
| 669 | # fold itself was the dominant cost above the cache's row |
| 670 | # threshold. |
| 671 | # |
| 672 | # In append mode, "exists" reflects the last-wins-by-id folded |
| 673 | # view either way (a deleted id is absent, so a create can reuse |
| 674 | # it -- resurrection), same as classic mode's index, which never |
| 675 | # indexes a stale/removed row. |
| 676 | fields, exists, _, records_ref, id_index_ref = _fast_record_lookup( |
| 677 | collection, path, record_id, need_row=False |
| 678 | ) |
| 679 | if exists: |
| 680 | raise DuplicateRecordIdError(f"Record already exists: {collection}/{record_id}") |
| 681 | |
| 682 | merged_fields = _merge_fields(fields, clean) |
| 683 | |
| 684 | def build_folded_records() -> list[dict[str, str]]: |
| 685 | if records_ref is not None: |
| 686 | records = [dict(existing) for existing in records_ref] |
| 687 | else: |
| 688 | # Sidecar-sourced lookup: no full record list was ever |
| 689 | # built. Only reached on a rare full-rewrite path (new |
| 690 | # field / pending auto-compaction / mode transition), so |
| 691 | # paying one full fold here -- unavoidable, a full |
| 692 | # rewrite needs the complete record set -- doesn't |
| 693 | # regress the fast steady-state append this whole path |
| 694 | # exists to keep O(1). |
| 695 | _, full_records, _ = _cache_entry(collection, path) |
| 696 | records = [dict(existing) for existing in full_records] |
| 697 | records.append(clean) |
| 698 | return records |
| 699 | |
| 700 | _persist_write( |
| 701 | collection, |
| 702 | path, |
| 703 | base_dir=base_dir, |
| 704 | roots=roots, |
| 705 | prior_fields=fields, |
| 706 | merged_fields=merged_fields, |
| 707 | folded_records_fn=build_folded_records, |
| 708 | delta_row=clean, |
| 709 | delta_op=OP_UPSERT, |
| 710 | delta_id=record_id, |
| 711 | ) |
| 712 | result = _surface_extra(_project_record(clean, merged_fields), extra_names=extra_names) |
| 713 | if record_changes: |
| 714 | object_record_changes.append_record_change( |
| 715 | collection=collection, |
| 716 | record_id=record_id, |
| 717 | action="create", |
| 718 | before=None, |
| 719 | after=result, |
| 720 | actor=actor or "unattributed", |
| 721 | correlation_id=object_correlation.current_correlation_id(), |
| 722 | base_dir=base_dir, |
| 723 | ) |
| 724 | _recompute_rollups_for_source(collection, [result], base_dir=base_dir, roots=roots) |
| 725 | return result |
| 726 | |
| 727 | |
| 728 | def update_collection_record( |
| 729 | collection: str, |
| 730 | record_id: str, |
| 731 | changes: dict[str, Any], |
| 732 | *, |
| 733 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 734 | roots: Iterable[Path] | None = None, |
| 735 | actor: str | None = None, |
| 736 | transition_subject: object_permissions.PermissionSubject | None = None, |
| 737 | preserve_read_only: bool = False, |
| 738 | allow_computed_submission: bool = False, |
| 739 | expected_rev: str | None = None, |
| 740 | record_changes: bool = True, |
| 741 | ) -> dict[str, str]: |
| 742 | """Update one existing record by id and return the stored row. |
| 743 | |
| 744 | See create_collection_record for the attribution contract: every |
| 745 | successful update is durably attributed, defaulting to |
| 746 | ``"unattributed"`` when the caller doesn't identify itself. |
| 747 | |
| 748 | ``transition_subject`` is optional and only affects schema-declared |
| 749 | transition guards (see _validate_field_transitions). The HTTP update |
| 750 | path resolves the request subject once permissions have already run |
| 751 | and passes it through here; direct library callers (daemon, CLI, |
| 752 | tests) that don't pass one are trusted callers -- guarded moves are |
| 753 | still checked for validity, just not for who is making them. |
| 754 | |
| 755 | ``preserve_read_only``/``allow_computed_submission`` mirror |
| 756 | create_collection_record's own flags, same "trusted, explicit, opt-in, |
| 757 | never wired to a request payload" posture -- an update needs them too, |
| 758 | e.g. the rollup daemon pass (plan/vocabulary/14-rollup-spec.md) stamping |
| 759 | a definition's read-only ``last_computed_at`` and a target row's |
| 760 | computed metric columns on every recompute. |
| 761 | |
| 762 | ``expected_rev`` is 63's optimistic-concurrency precondition |
| 763 | (plan/vocabulary/63-concurrency-spec.md): when supplied, the freshly-read |
| 764 | row's compute_record_rev is compared against it INSIDE the write lock, |
| 765 | before any validation or write; a mismatch raises VersionConflictError |
| 766 | and nothing is written. ``None`` (the default) means last-write-wins, |
| 767 | exactly as before -- the precondition is opt-in per write, never |
| 768 | mandatory. Because the check runs after the lock is held and reads the |
| 769 | same post-lock row every writer serializes behind, two racing updates |
| 770 | to the same record resolve as "first wins, second sees the changed row |
| 771 | and 409s" rather than a silent clobber -- see the spec's worked example. |
| 772 | """ |
| 773 | _ensure_collection_known(collection, base_dir=base_dir, roots=roots) |
| 774 | if not validate_record_id(record_id): |
| 775 | raise InvalidRecordIdError(f"Invalid record id: {record_id}") |
| 776 | extra_names = _extra_field_names(collection, base_dir=base_dir, roots=roots) |
| 777 | |
| 778 | clean = _normalize_record_payload(changes, require_id=False) |
| 779 | if "id" in clean and clean["id"] != record_id: |
| 780 | raise InvalidRecordPayloadError("Record id cannot be changed") |
| 781 | submitted_fields = frozenset(clean) |
| 782 | |
| 783 | path = collection_records_file(collection, base_dir=base_dir) |
| 784 | with _records_file_lock(path): |
| 785 | # See the matching comment in create_collection_record: this |
| 786 | # resolves via warm cache, then (append mode, over threshold) the |
| 787 | # id->offset sidecar, before a full fold -- see |
| 788 | # _fast_record_lookup. `existing_row` is always already a fresh, |
| 789 | # independent copy. When sourced from the cache (or a forced full |
| 790 | # fold below), id_index_ref is first-occurrence-wins |
| 791 | # (_build_id_index), same as the linear scan this replaces, so |
| 792 | # this changes nothing about WHICH row a duplicate-id file |
| 793 | # resolves to. |
| 794 | fields, exists, existing_row, records_ref, id_index_ref = _fast_record_lookup( |
| 795 | collection, path, record_id, need_row=True |
| 796 | ) |
| 797 | if not exists: |
| 798 | raise RecordNotFoundError(f"Record not found: {collection}/{record_id}") |
| 799 | |
| 800 | existing_blob = _parse_extra_blob(existing_row.get(EXTRA_FIELD)) |
| 801 | existing = _surface_extra(existing_row, extra_names=extra_names) |
| 802 | |
| 803 | # 63: the precondition, checked against the row just read under the |
| 804 | # lock (the freshest possible state -- not the caller's earlier |
| 805 | # pre-lock read, which may be stale by now). `existing` is the same |
| 806 | # surfaced representation get_collection_record returns and the read |
| 807 | # side fingerprints, so the two `_rev` values are computed over |
| 808 | # identical inputs. Runs before any validation/write: a mismatch is |
| 809 | # a clean no-op. |
| 810 | if expected_rev is not None and compute_record_rev(existing) != expected_rev: |
| 811 | raise VersionConflictError( |
| 812 | f"Record {collection}/{record_id} has changed since it was read " |
| 813 | f"(expected _rev {expected_rev!r})." |
| 814 | ) |
| 815 | |
| 816 | updated = dict(existing) |
| 817 | updated.update(clean) |
| 818 | updated["id"] = record_id |
| 819 | updated = _apply_computed_fields(collection, updated, base_dir=base_dir, roots=roots) |
| 820 | _validate_record_against_schema( |
| 821 | collection, |
| 822 | updated, |
| 823 | submitted_fields=submitted_fields, |
| 824 | base_dir=base_dir, |
| 825 | roots=roots, |
| 826 | allow_read_only_submission=preserve_read_only, |
| 827 | allow_computed_submission=allow_computed_submission, |
| 828 | ) |
| 829 | _check_field_storable(updated) |
| 830 | _validate_field_transitions( |
| 831 | collection, |
| 832 | existing, |
| 833 | updated, |
| 834 | base_dir=base_dir, |
| 835 | roots=roots, |
| 836 | subject=transition_subject, |
| 837 | ) |
| 838 | updated = _canonicalize_schema_values( |
| 839 | collection, updated, base_dir=base_dir, roots=roots |
| 840 | ) |
| 841 | updated = _route_extra(updated, existing_blob=existing_blob, extra_names=extra_names) |
| 842 | merged_fields = _merge_fields(fields, updated) |
| 843 | |
| 844 | def build_folded_records() -> list[dict[str, str]]: |
| 845 | if records_ref is not None: |
| 846 | records = [dict(existing_row) for existing_row in records_ref] |
| 847 | records[id_index_ref[record_id]] = updated |
| 848 | else: |
| 849 | # Sidecar-sourced lookup -- see the matching comment in |
| 850 | # create_collection_record's build_folded_records. |
| 851 | _, full_records, full_id_index = _cache_entry(collection, path) |
| 852 | records = [dict(existing_row) for existing_row in full_records] |
| 853 | records[full_id_index[record_id]] = updated |
| 854 | return records |
| 855 | |
| 856 | _persist_write( |
| 857 | collection, |
| 858 | path, |
| 859 | base_dir=base_dir, |
| 860 | roots=roots, |
| 861 | prior_fields=fields, |
| 862 | merged_fields=merged_fields, |
| 863 | folded_records_fn=build_folded_records, |
| 864 | delta_row=updated, |
| 865 | delta_op=OP_UPSERT, |
| 866 | delta_id=record_id, |
| 867 | ) |
| 868 | result = _surface_extra(_project_record(updated, merged_fields), extra_names=extra_names) |
| 869 | if record_changes: |
| 870 | object_record_changes.append_record_change( |
| 871 | collection=collection, |
| 872 | record_id=record_id, |
| 873 | action="update", |
| 874 | before=existing, |
| 875 | after=result, |
| 876 | actor=actor or "unattributed", |
| 877 | correlation_id=object_correlation.current_correlation_id(), |
| 878 | base_dir=base_dir, |
| 879 | ) |
| 880 | _recompute_rollups_for_source( |
| 881 | collection, [existing, result], base_dir=base_dir, roots=roots |
| 882 | ) |
| 883 | return result |
| 884 | |
| 885 | |
| 886 | def delete_collection_record( |
| 887 | collection: str, |
| 888 | record_id: str, |
| 889 | *, |
| 890 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 891 | roots: Iterable[Path] | None = None, |
| 892 | actor: str | None = None, |
| 893 | record_changes: bool = True, |
| 894 | ) -> dict[str, str]: |
| 895 | """Delete one existing record by id and return the removed row. |
| 896 | |
| 897 | See create_collection_record for the attribution contract: every |
| 898 | successful delete is durably attributed, defaulting to |
| 899 | ``"unattributed"`` when the caller doesn't identify itself. |
| 900 | """ |
| 901 | _ensure_collection_known(collection, base_dir=base_dir, roots=roots) |
| 902 | if not validate_record_id(record_id): |
| 903 | raise InvalidRecordIdError(f"Invalid record id: {record_id}") |
| 904 | |
| 905 | path = collection_records_file(collection, base_dir=base_dir) |
| 906 | with _records_file_lock(path): |
| 907 | # See the matching comment in create_collection_record. |
| 908 | # `existing_row` is already a fresh, independent copy -- safe to |
| 909 | # hand back across the API as-is. |
| 910 | fields, exists, existing_row, records_ref, id_index_ref = _fast_record_lookup( |
| 911 | collection, path, record_id, need_row=True |
| 912 | ) |
| 913 | if not exists: |
| 914 | raise RecordNotFoundError(f"Record not found: {collection}/{record_id}") |
| 915 | |
| 916 | removed = existing_row |
| 917 | |
| 918 | def build_folded_records() -> list[dict[str, str]]: |
| 919 | if records_ref is not None: |
| 920 | records = [dict(existing_row) for existing_row in records_ref] |
| 921 | records.pop(id_index_ref[record_id]) |
| 922 | else: |
| 923 | # Sidecar-sourced lookup -- see the matching comment in |
| 924 | # create_collection_record's build_folded_records. |
| 925 | _, full_records, full_id_index = _cache_entry(collection, path) |
| 926 | records = [dict(existing_row) for existing_row in full_records] |
| 927 | records.pop(full_id_index[record_id]) |
| 928 | return records |
| 929 | |
| 930 | _persist_write( |
| 931 | collection, |
| 932 | path, |
| 933 | base_dir=base_dir, |
| 934 | roots=roots, |
| 935 | prior_fields=fields, |
| 936 | merged_fields=fields, |
| 937 | folded_records_fn=build_folded_records, |
| 938 | delta_row=None, |
| 939 | delta_op=OP_DELETE, |
| 940 | delta_id=record_id, |
| 941 | ) |
| 942 | result = _project_record(removed, fields) |
| 943 | if record_changes: |
| 944 | object_record_changes.append_record_change( |
| 945 | collection=collection, |
| 946 | record_id=record_id, |
| 947 | action="delete", |
| 948 | before=result, |
| 949 | after=None, |
| 950 | actor=actor or "unattributed", |
| 951 | correlation_id=object_correlation.current_correlation_id(), |
| 952 | base_dir=base_dir, |
| 953 | ) |
| 954 | _recompute_rollups_for_source(collection, [result], base_dir=base_dir, roots=roots) |
| 955 | return result |
| 956 | |
| 957 | |
| 958 | def compact_collection( |
| 959 | collection: str, |
| 960 | *, |
| 961 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 962 | roots: Iterable[Path] | None = None, |
| 963 | ) -> dict[str, Any]: |
| 964 | """Fold an append-mode collection's records file to its live rows only. |
| 965 | |
| 966 | Rewrites atomically via the same _write_collection_records path a |
| 967 | plain write uses (temp file + replace), under the collection's file |
| 968 | lock, keeping the `_op` column (every row becomes an upsert). Callable |
| 969 | directly (this function), via POST /admin/collections/{collection}/ |
| 970 | compact (object_server.py), or automatically: Auto-compaction |
| 971 | (_maybe_flag_auto_compact) causes a collection's next ordinary write |
| 972 | to perform this same rewrite instead of a plain append, and the |
| 973 | daemon's process_compactions (object_daemon.py) polls append_ |
| 974 | collection_stats and compacts any collection over its bloat/row |
| 975 | thresholds on a timer (see docs/append-only-storage-design.md |
| 976 | Compaction: "run on a schedule or when the superseded-row ratio |
| 977 | passes a threshold — never inline in a request"). This function can |
| 978 | also be called directly at any time, including on a classic-mode or |
| 979 | never-written collection, where it is a correctly-reported no-op. |
| 980 | |
| 981 | Returns {"rows_before", "rows_after", "bytes_before", "bytes_after"} |
| 982 | (row counts are physical rows on disk before/after; for a collection |
| 983 | not currently in append physical format, before == after). |
| 984 | """ |
| 985 | _ensure_collection_known(collection, base_dir=base_dir, roots=roots) |
| 986 | path = collection_records_file(collection, base_dir=base_dir) |
| 987 | |
| 988 | with _records_file_lock(path): |
| 989 | try: |
| 990 | bytes_before = path.stat().st_size |
| 991 | except OSError: |
| 992 | bytes_before = 0 |
| 993 | |
| 994 | physical_header = _physical_header(path) |
| 995 | if not physical_header or physical_header[0] != OP_FIELD: |
| 996 | rows_before = len(_read_collection_records(collection, base_dir=base_dir)) |
| 997 | return { |
| 998 | "rows_before": rows_before, |
| 999 | "rows_after": rows_before, |
| 1000 | "bytes_before": bytes_before, |
| 1001 | "bytes_after": bytes_before, |
| 1002 | } |
| 1003 | |
| 1004 | with path.open(newline="") as handle: |
| 1005 | text = handle.read() |
| 1006 | folded_records, physical_row_count = _parse_append_body(collection, text, physical_header) |
| 1007 | logical_fields = physical_header[1:] |
| 1008 | |
| 1009 | _write_collection_records( |
| 1010 | collection, path, physical_header, folded_records, cache_fields=logical_fields |
| 1011 | ) |
| 1012 | _PENDING_COMPACTION.discard(str(path.resolve(strict=False))) |
| 1013 | _discard_oidx(path) |
| 1014 | |
| 1015 | bytes_after = path.stat().st_size |
| 1016 | return { |
| 1017 | "rows_before": physical_row_count, |
| 1018 | "rows_after": len(folded_records), |
| 1019 | "bytes_before": bytes_before, |
| 1020 | "bytes_after": bytes_after, |
| 1021 | } |
| 1022 | |
| 1023 | |
| 1024 | def prune_collection_records( |
| 1025 | collection: str, |
| 1026 | *, |
| 1027 | keep_newer_than: str, |
| 1028 | timestamp_field: str = "created_at", |
| 1029 | keep_last: int | None = None, |
| 1030 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 1031 | roots: Iterable[Path] | None = None, |
| 1032 | ) -> dict[str, Any]: |
| 1033 | """Retention / rotation: drop rows whose ``timestamp_field`` is strictly |
| 1034 | older than ``keep_newer_than`` (an ISO-8601 string), rewriting the file to |
| 1035 | only the kept rows in ONE atomic pass (temp + replace, under the file lock) |
| 1036 | -- the same rewrite path compaction uses. |
| 1037 | |
| 1038 | This is the tool for high-volume append-mode event logs (page_views) where |
| 1039 | id-fold compaction cannot help: every row is a distinct event, nothing is |
| 1040 | superseded, so the only way to bound the file is to age rows out. A row with |
| 1041 | a missing or unparseable timestamp is KEPT (never delete data we can't |
| 1042 | date). Only meaningful for append-format collections; a classic-mode or |
| 1043 | never-written collection is a correctly-reported no-op. |
| 1044 | |
| 1045 | ``keep_last`` additionally caps the row COUNT, keeping the newest N after |
| 1046 | the time filter. Time alone does not bound a log: a retention window says |
| 1047 | how far back to look, not how much can arrive inside it, so a traffic |
| 1048 | spike (or a bot storm) can fill a disk without aging a single row out. |
| 1049 | Small machines need the second bound, and a bound nobody set is a bound |
| 1050 | that fails on the worst day rather than an ordinary one. |
| 1051 | |
| 1052 | Returns ``{"rows_before", "rows_after", "removed", "pruned"}``. |
| 1053 | """ |
| 1054 | _ensure_collection_known(collection, base_dir=base_dir, roots=roots) |
| 1055 | path = collection_records_file(collection, base_dir=base_dir) |
| 1056 | |
| 1057 | with _records_file_lock(path): |
| 1058 | physical_header = _physical_header(path) |
| 1059 | if not physical_header or physical_header[0] != OP_FIELD: |
| 1060 | live = len(_read_collection_records(collection, base_dir=base_dir)) |
| 1061 | return {"rows_before": live, "rows_after": live, "removed": 0, "pruned": False} |
| 1062 | |
| 1063 | with path.open(newline="") as handle: |
| 1064 | text = handle.read() |
| 1065 | folded_records, _physical_row_count = _parse_append_body(collection, text, physical_header) |
| 1066 | logical_fields = physical_header[1:] |
| 1067 | |
| 1068 | cutoff = str(keep_newer_than) |
| 1069 | |
| 1070 | def _keep(record: dict[str, str]) -> bool: |
| 1071 | stamp = str(record.get(timestamp_field) or "") |
| 1072 | return True if not stamp else stamp >= cutoff |
| 1073 | |
| 1074 | kept = [record for record in folded_records if _keep(record)] |
| 1075 | if keep_last is not None and keep_last >= 0 and len(kept) > keep_last: |
| 1076 | # Newest N. The append log is written in arrival order, so the tail |
| 1077 | # IS the newest -- no sort, which matters because this runs against |
| 1078 | # the largest file the server owns. |
| 1079 | kept = kept[len(kept) - keep_last:] |
| 1080 | removed = len(folded_records) - len(kept) |
| 1081 | if removed == 0: |
| 1082 | return { |
| 1083 | "rows_before": len(folded_records), |
| 1084 | "rows_after": len(folded_records), |
| 1085 | "removed": 0, |
| 1086 | "pruned": False, |
| 1087 | } |
| 1088 | |
| 1089 | _write_collection_records( |
| 1090 | collection, path, physical_header, kept, cache_fields=logical_fields |
| 1091 | ) |
| 1092 | _PENDING_COMPACTION.discard(str(path.resolve(strict=False))) |
| 1093 | _discard_oidx(path) |
| 1094 | return { |
| 1095 | "rows_before": len(folded_records), |
| 1096 | "rows_after": len(kept), |
| 1097 | "removed": removed, |
| 1098 | "pruned": True, |
| 1099 | } |
| 1100 | |
| 1101 | |
| 1102 | def append_collection_stats( |
| 1103 | collection: str, |
| 1104 | *, |
| 1105 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 1106 | roots: Iterable[Path] | None = None, |
| 1107 | allow_fold: bool = True, |
| 1108 | ) -> dict[str, Any] | None: |
| 1109 | """Return compaction-observability stats for one collection, or None |
| 1110 | when it is not currently in append storage mode (docs/storage-modes.md |
| 1111 | "Compaction"). |
| 1112 | |
| 1113 | Returns: |
| 1114 | {"collection", "storage": "append", "physical_rows", "live_rows", |
| 1115 | "dead_rows", "bloat_ratio" (dead_rows / max(live_rows, 1), rounded |
| 1116 | 3dp), "file_bytes", "sidecar_present", "compaction_flagged"} |
| 1117 | |
| 1118 | Cheapest-source-first, same doctrine as _fast_record_lookup's point-op |
| 1119 | resolution (docs/append-only-storage-design.md Sidecars), tried in |
| 1120 | order and combined -- either can answer `live_rows`, but only the |
| 1121 | sidecar can answer `physical_rows` short of a fold: |
| 1122 | |
| 1123 | 1. A warm _RECORDS_CACHE hit (_peek_records_cache) answers |
| 1124 | `live_rows` for free -- it's already the folded record count, no |
| 1125 | I/O at all. |
| 1126 | 2. The id->offset sidecar (.records.oidx), read directly via |
| 1127 | _load_oidx_body -- a "pure line-count read" as opposed to a |
| 1128 | records.tsv parse: one line per physical row was written when |
| 1129 | that row was appended (OIDX_FILE's format), so the sidecar's own |
| 1130 | body length IS `physical_rows` without touching records.tsv at |
| 1131 | all, and the same read's last-wins fold gives `live_rows` too |
| 1132 | (used only when tier 1 didn't already answer it). Deliberately |
| 1133 | narrower than the hot-path loader _load_oidx: this never triggers |
| 1134 | a catch-up scan or a from-scratch rebuild when the sidecar is |
| 1135 | stale or missing (`covered_bytes` short of the file's current |
| 1136 | size, wrong header inode, or absent) -- an observability call |
| 1137 | should never be the reason records.tsv gets scanned; it just |
| 1138 | doesn't answer from this tier, and falls through to tier 3 (or |
| 1139 | the estimated shortcut below) instead. |
| 1140 | 3. Neither tier answered both numbers: a full fold via |
| 1141 | _parse_append_body -- the same O(file) cost compact_collection |
| 1142 | itself pays to read the file (docs/storage-modes.md: "cold reads |
| 1143 | of an append file cost ~1.6x a classic parse"). This is the ONLY |
| 1144 | expensive path in this function, and only runs when `allow_fold` |
| 1145 | is True (the default for direct/CLI callers, and for the daemon's |
| 1146 | polling pass, where an occasional full fold is an acceptable |
| 1147 | cost against an interval timer). Callers that must never pay it |
| 1148 | -- GET /admin/storage (object_server.py), which must stay O(1) |
| 1149 | regardless of any one collection's cache/sidecar state -- pass |
| 1150 | allow_fold=False and get an {"estimated": True, "physical_rows": |
| 1151 | None, "live_rows": None, "dead_rows": None, "bloat_ratio": None} |
| 1152 | entry instead (still with a real, cheap `file_bytes` from the |
| 1153 | initial stat). |
| 1154 | |
| 1155 | A collection whose records.tsv doesn't exist yet (schema declared |
| 1156 | "append" but never written) short-circuits to an honest all-zero |
| 1157 | result before any of the above -- that check (`path.exists()`, done as |
| 1158 | part of the initial stat) is itself O(1), so it's never worth |
| 1159 | representing as "estimated". |
| 1160 | """ |
| 1161 | _ensure_collection_known(collection, base_dir=base_dir, roots=roots) |
| 1162 | mode = _collection_storage_mode(collection, base_dir=base_dir, roots=roots) |
| 1163 | if mode != object_schemas.STORAGE_APPEND: |
| 1164 | return None |
| 1165 | |
| 1166 | path = collection_records_file(collection, base_dir=base_dir) |
| 1167 | try: |
| 1168 | st = path.stat() |
| 1169 | file_bytes = st.st_size |
| 1170 | data_ino: int | None = st.st_ino |
| 1171 | except OSError: |
| 1172 | file_bytes = 0 |
| 1173 | data_ino = None |
| 1174 | |
| 1175 | cache_key = str(path.resolve(strict=False)) |
| 1176 | sidecar_path = _oidx_path(path) |
| 1177 | sidecar_present = sidecar_path.exists() |
| 1178 | compaction_flagged = cache_key in _PENDING_COMPACTION |
| 1179 | |
| 1180 | if data_ino is None: |
| 1181 | # No records.tsv at all -- a schema can declare "append" storage |
| 1182 | # before its collection is ever written. Nothing to fold, sidecar |
| 1183 | # or otherwise; report honest zeros rather than routing this |
| 1184 | # through the fold/estimate machinery below. |
| 1185 | return { |
| 1186 | "collection": collection, |
| 1187 | "storage": object_schemas.STORAGE_APPEND, |
| 1188 | "physical_rows": 0, |
| 1189 | "live_rows": 0, |
| 1190 | "dead_rows": 0, |
| 1191 | "bloat_ratio": 0.0, |
| 1192 | "file_bytes": 0, |
| 1193 | "sidecar_present": sidecar_present, |
| 1194 | "compaction_flagged": compaction_flagged, |
| 1195 | } |
| 1196 | |
| 1197 | live_rows: int | None = None |
| 1198 | physical_rows: int | None = None |
| 1199 | |
| 1200 | peeked = _peek_records_cache(path) |
| 1201 | if peeked is not None: |
| 1202 | live_rows = len(peeked[1]) |
| 1203 | |
| 1204 | if sidecar_present: |
| 1205 | header_ino = _read_oidx_header(sidecar_path) |
| 1206 | if header_ino == data_ino: |
| 1207 | loaded = _load_oidx_body(sidecar_path) |
| 1208 | if loaded is not None: |
| 1209 | body_dict, body_covered, body_physical = loaded |
| 1210 | covered = ( |
| 1211 | body_covered if body_covered is not None else _records_header_length(path) |
| 1212 | ) |
| 1213 | if covered == file_bytes: |
| 1214 | physical_rows = body_physical |
| 1215 | if live_rows is None: |
| 1216 | live_rows = len(body_dict) |
| 1217 | |
| 1218 | if physical_rows is None or live_rows is None: |
| 1219 | if not allow_fold: |
| 1220 | return { |
| 1221 | "collection": collection, |
| 1222 | "storage": object_schemas.STORAGE_APPEND, |
| 1223 | "physical_rows": None, |
| 1224 | "live_rows": None, |
| 1225 | "dead_rows": None, |
| 1226 | "bloat_ratio": None, |
| 1227 | "file_bytes": file_bytes, |
| 1228 | "sidecar_present": sidecar_present, |
| 1229 | "compaction_flagged": compaction_flagged, |
| 1230 | "estimated": True, |
| 1231 | } |
| 1232 | |
| 1233 | physical_header = _physical_header(path) |
| 1234 | if not physical_header or physical_header[0] != OP_FIELD: |
| 1235 | # Exists on disk but never written in append physical format |
| 1236 | # (e.g. an empty file) -- nothing to fold. |
| 1237 | physical_rows = 0 |
| 1238 | live_rows = 0 |
| 1239 | else: |
| 1240 | with path.open(newline="") as handle: |
| 1241 | text = handle.read() |
| 1242 | folded_records, physical_row_count = _parse_append_body(collection, text, physical_header) |
| 1243 | physical_rows = physical_row_count |
| 1244 | live_rows = len(folded_records) |
| 1245 | |
| 1246 | dead_rows = max(physical_rows - live_rows, 0) |
| 1247 | bloat_ratio = round(dead_rows / max(live_rows, 1), 3) |
| 1248 | return { |
| 1249 | "collection": collection, |
| 1250 | "storage": object_schemas.STORAGE_APPEND, |
| 1251 | "physical_rows": physical_rows, |
| 1252 | "live_rows": live_rows, |
| 1253 | "dead_rows": dead_rows, |
| 1254 | "bloat_ratio": bloat_ratio, |
| 1255 | "file_bytes": file_bytes, |
| 1256 | "sidecar_present": sidecar_present, |
| 1257 | "compaction_flagged": compaction_flagged, |
| 1258 | } |
| 1259 | |
| 1260 | |
| 1261 | def list_append_collection_stats( |
| 1262 | *, |
| 1263 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 1264 | roots: Iterable[Path] | None = None, |
| 1265 | allow_fold: bool = True, |
| 1266 | ) -> list[dict[str, Any]]: |
| 1267 | """Return append_collection_stats() for every collection whose schema |
| 1268 | currently declares "storage": "append", sorted by collection name. |
| 1269 | |
| 1270 | Discovery (object_schemas.list_append_storage_collections) is cheap |
| 1271 | regardless of how many collections exist on disk: it globs the schemas |
| 1272 | directory and loads each manual schema through the same mtime/size- |
| 1273 | keyed cache every other schema read in this codebase already goes |
| 1274 | through, never touching records.tsv. A collection whose schema |
| 1275 | disappears between that listing and its own stats call (a benign race |
| 1276 | under concurrent schema edits) is skipped rather than raising -- same |
| 1277 | tolerance _ensure_collection_known's callers generally apply to |
| 1278 | concurrent schema/collection changes. |
| 1279 | """ |
| 1280 | names = object_schemas.list_append_storage_collections(base_dir=base_dir) |
| 1281 | stats: list[dict[str, Any]] = [] |
| 1282 | for name in names: |
| 1283 | try: |
| 1284 | entry = append_collection_stats( |
| 1285 | name, base_dir=base_dir, roots=roots, allow_fold=allow_fold |
| 1286 | ) |
| 1287 | except ( |
| 1288 | object_collections.InvalidCollectionNameError, |
| 1289 | object_collections.CollectionNotFoundError, |
| 1290 | ): |
| 1291 | continue |
| 1292 | if entry is not None: |
| 1293 | stats.append(entry) |
| 1294 | return stats |
| 1295 | |
| 1296 | |
| 1297 | def collection_records_file(collection: str, base_dir: Path | str = DEFAULT_DATA_DIR) -> Path: |
| 1298 | """Return the validated TSV file path for a collection.""" |
| 1299 | if not object_collections.validate_collection_name(collection): |
| 1300 | raise object_collections.InvalidCollectionNameError( |
| 1301 | f"Invalid collection name: {collection}" |
| 1302 | ) |
| 1303 | |
| 1304 | root = Path(base_dir) / COLLECTIONS_DIR |
| 1305 | path = root / collection / RECORDS_FILE |
| 1306 | resolved_root = root.resolve(strict=False) |
| 1307 | resolved_path = path.resolve(strict=False) |
| 1308 | |
| 1309 | try: |
| 1310 | resolved_path.relative_to(resolved_root) |
| 1311 | except ValueError as exc: |
| 1312 | raise object_collections.InvalidCollectionNameError( |
| 1313 | f"Collection path escapes collection directory: {collection}" |
| 1314 | ) from exc |
| 1315 | |
| 1316 | return path |
| 1317 | |
| 1318 | |
| 1319 | def collection_has_records(collection: str, base_dir: Path | str = DEFAULT_DATA_DIR) -> bool: |
| 1320 | """Return True when a collection has a records TSV file.""" |
| 1321 | try: |
| 1322 | path = collection_records_file(collection, base_dir=base_dir) |
| 1323 | except object_collections.InvalidCollectionNameError: |
| 1324 | return False |
| 1325 | return path.is_file() |
| 1326 | |
| 1327 | |
| 1328 | def iter_record_collections(base_dir: Path | str = DEFAULT_DATA_DIR) -> list[str]: |
| 1329 | """Return collection names with a ``records.tsv`` file.""" |
| 1330 | root = Path(base_dir) / COLLECTIONS_DIR |
| 1331 | if not root.exists() or not root.is_dir(): |
| 1332 | return [] |
| 1333 | |
| 1334 | names = [] |
| 1335 | for path in sorted(root.glob(f"*/{RECORDS_FILE}"), key=lambda item: item.as_posix()): |
| 1336 | name = path.parent.name |
| 1337 | if object_collections.validate_collection_name(name): |
| 1338 | names.append(name) |
| 1339 | return names |
| 1340 | |
| 1341 | |
| 1342 | def _ensure_collection_known( |
| 1343 | collection: str, |
| 1344 | *, |
| 1345 | base_dir: Path | str, |
| 1346 | roots: Iterable[Path] | None, |
| 1347 | ) -> None: |
| 1348 | if not object_collections.validate_collection_name(collection): |
| 1349 | raise object_collections.InvalidCollectionNameError( |
| 1350 | f"Invalid collection name: {collection}" |
| 1351 | ) |
| 1352 | |
| 1353 | if collection_has_records(collection, base_dir=base_dir): |
| 1354 | return |
| 1355 | |
| 1356 | try: |
| 1357 | object_schemas.get_schema(collection, base_dir=base_dir, roots=roots) |
| 1358 | return |
| 1359 | except object_schemas.SchemaNotFoundError: |
| 1360 | pass |
| 1361 | |
| 1362 | raise object_collections.CollectionNotFoundError(f"Collection not found: {collection}") |
| 1363 | |
| 1364 | |
| 1365 | def _cached_records_ref( |
| 1366 | collection: str, |
| 1367 | *, |
| 1368 | base_dir: Path | str, |
| 1369 | ) -> list[dict[str, str]]: |
| 1370 | """Return the module cache's own records list for a collection. |
| 1371 | |
| 1372 | Callers must not mutate the returned list or its dicts in place, and |
| 1373 | must copy before handing anything derived from it back across the |
| 1374 | public API -- see the _RECORDS_CACHE ALIASING SAFETY note above. |
| 1375 | """ |
| 1376 | path = collection_records_file(collection, base_dir=base_dir) |
| 1377 | _, records, _ = _cache_entry(collection, path) |
| 1378 | return records |
| 1379 | |
| 1380 | |
| 1381 | def _read_collection_records( |
| 1382 | collection: str, |
| 1383 | *, |
| 1384 | base_dir: Path | str, |
| 1385 | ) -> list[dict[str, str]]: |
| 1386 | records = _cached_records_ref(collection, base_dir=base_dir) |
| 1387 | # Copy: `records` is the module cache's own list of its own dicts. |
| 1388 | return [dict(record) for record in records] |
| 1389 | |
| 1390 | |
| 1391 | def _stat_signature(path: Path) -> tuple[int, int, int] | None: |
| 1392 | try: |
| 1393 | st = path.stat() |
| 1394 | except OSError: |
| 1395 | return None |
| 1396 | return (st.st_mtime_ns, st.st_size, st.st_ino) |
| 1397 | |
| 1398 | |
| 1399 | def _build_id_index(records: list[dict[str, str]]) -> dict[str, int]: |
| 1400 | index: dict[str, int] = {} |
| 1401 | for position, record in enumerate(records): |
| 1402 | record_id = record.get("id") |
| 1403 | if record_id and record_id not in index: |
| 1404 | index[record_id] = position |
| 1405 | return index |
| 1406 | |
| 1407 | |
| 1408 | def _env_int(name: str, default: int) -> int: |
| 1409 | """Read an int env var, falling back to `default` when unset or unparseable.""" |
| 1410 | raw = os.environ.get(name) |
| 1411 | if raw is None: |
| 1412 | return default |
| 1413 | try: |
| 1414 | return int(raw) |
| 1415 | except ValueError: |
| 1416 | return default |
| 1417 | |
| 1418 | |
| 1419 | def _records_cache_max_entries() -> int: |
| 1420 | return _env_int(RECORDS_CACHE_MAX_ENTRIES_ENV, _DEFAULT_RECORDS_CACHE_MAX_ENTRIES) |
| 1421 | |
| 1422 | |
| 1423 | def _records_cache_max_rows() -> int: |
| 1424 | return _env_int(RECORDS_CACHE_MAX_ROWS_ENV, _DEFAULT_RECORDS_CACHE_MAX_ROWS) |
| 1425 | |
| 1426 | |
| 1427 | def _oidx_cache_max_entries() -> int: |
| 1428 | return _env_int(OIDX_CACHE_MAX_ENTRIES_ENV, _DEFAULT_OIDX_CACHE_MAX_ENTRIES) |
| 1429 | |
| 1430 | |
| 1431 | def _store_records_cache( |
| 1432 | cache_key: str, |
| 1433 | signature: tuple[int, int, int], |
| 1434 | fields: list[str], |
| 1435 | records: list[dict[str, str]], |
| 1436 | id_index: dict[str, int], |
| 1437 | covered_bytes: int, |
| 1438 | ) -> None: |
| 1439 | """Insert/refresh one entry as most-recently-used, then evict over capacity. |
| 1440 | |
| 1441 | Eviction is pure LRU by entry count: once the dict exceeds |
| 1442 | DBBASIC_RECORDS_CACHE_MAX_ENTRIES, the oldest (least-recently-used) |
| 1443 | entries are dropped first via OrderedDict.popitem(last=False). |
| 1444 | |
| 1445 | `covered_bytes` is normally just `signature[1]` (every caller except |
| 1446 | _try_incremental_cache_entry's torn-tail case passes exactly that) -- |
| 1447 | see the _RECORDS_CACHE block comment above for what it means and the |
| 1448 | one case where it differs. |
| 1449 | """ |
| 1450 | _RECORDS_CACHE[cache_key] = (signature, fields, records, id_index, covered_bytes) |
| 1451 | _RECORDS_CACHE.move_to_end(cache_key) |
| 1452 | max_entries = _records_cache_max_entries() |
| 1453 | while len(_RECORDS_CACHE) > max_entries: |
| 1454 | _RECORDS_CACHE.popitem(last=False) |
| 1455 | |
| 1456 | |
| 1457 | def _peek_records_cache( |
| 1458 | path: Path, |
| 1459 | ) -> tuple[list[str], list[dict[str, str]], dict[str, int]] | None: |
| 1460 | """Return a warm _RECORDS_CACHE hit for `path`, or None on any miss. |
| 1461 | |
| 1462 | Never parses: this is the "is there already a fully-folded view of |
| 1463 | this file sitting in memory" check used to decide, in |
| 1464 | _fast_record_lookup, whether a point op can skip both a full fold AND |
| 1465 | the sidecar (cache hit -- the common case for any collection at or |
| 1466 | under DBBASIC_RECORDS_CACHE_MAX_ROWS once warm) or must try the |
| 1467 | cheaper-than-a-fold sidecar path next (a miss -- cold cache, an |
| 1468 | evicted LRU entry, or a collection that has never fit under the row |
| 1469 | threshold and so is never stored here at all; see _cache_entry, this |
| 1470 | function's parsing counterpart, which _fast_record_lookup and every |
| 1471 | non-fast-path caller fall back to when neither a cache hit nor the |
| 1472 | sidecar can answer). |
| 1473 | """ |
| 1474 | cache_key = str(path.resolve(strict=False)) |
| 1475 | signature = _stat_signature(path) |
| 1476 | if signature is None: |
| 1477 | return None |
| 1478 | cached = _RECORDS_CACHE.get(cache_key) |
| 1479 | if cached is not None and cached[0] == signature: |
| 1480 | _RECORDS_CACHE.move_to_end(cache_key) |
| 1481 | return cached[1], cached[2], cached[3] |
| 1482 | return None |
| 1483 | |
| 1484 | |
| 1485 | def _cache_entry( |
| 1486 | collection: str, |
| 1487 | path: Path, |
| 1488 | ) -> tuple[list[str], list[dict[str, str]], dict[str, int]]: |
| 1489 | """Return (fields, records, id_index) for path's current content. |
| 1490 | |
| 1491 | Serves from the module cache when the file's stat signature matches |
| 1492 | (moving the entry to most-recently-used); otherwise parses fresh and, |
| 1493 | when the file exists and its row count is within |
| 1494 | DBBASIC_RECORDS_CACHE_MAX_ROWS, stores the result for next time (a |
| 1495 | collection over that threshold is returned normally but left out of |
| 1496 | the cache -- see the _RECORDS_CACHE block comment above). The returned |
| 1497 | fields/records/id_index are the CACHED objects -- see the |
| 1498 | _RECORDS_CACHE docstring above. Callers within this module must copy |
| 1499 | before handing anything back across the public API. |
| 1500 | |
| 1501 | On a peek miss, tries one more thing before paying a full parse: if a |
| 1502 | STALE entry is sitting in _RECORDS_CACHE for this exact path and the |
| 1503 | file has purely grown in place (same inode, size beyond what that |
| 1504 | entry covered) on an append-mode file, _try_incremental_cache_entry |
| 1505 | folds only the new tail bytes onto the stale entry instead -- O(new |
| 1506 | rows), not O(file). See that function for the full precondition list |
| 1507 | and fallback rules; anything it can't handle falls through to the |
| 1508 | ordinary full parse below, unchanged. |
| 1509 | |
| 1510 | Absent that, this is the O(n) fold path: a full parse of the whole |
| 1511 | file whenever there's no warm cache hit. _fast_record_lookup (get/ |
| 1512 | create/update/delete's shared point-op resolver) tries |
| 1513 | _peek_records_cache and then the id->offset sidecar FIRST specifically |
| 1514 | to avoid paying this on every write to a collection too large to fit |
| 1515 | in _RECORDS_CACHE; this function remains the correctness fallback for |
| 1516 | everything else (list, read_collection_records, a cold/small |
| 1517 | collection's first touch, and any point op the sidecar can't answer |
| 1518 | authoritatively). |
| 1519 | """ |
| 1520 | if not path.exists(): |
| 1521 | return ["id"], [], {} |
| 1522 | if not path.is_file(): |
| 1523 | raise ValueError(f"Collection records path is not a file: {collection}") |
| 1524 | |
| 1525 | peeked = _peek_records_cache(path) |
| 1526 | if peeked is not None: |
| 1527 | return peeked |
| 1528 | |
| 1529 | cache_key = str(path.resolve(strict=False)) |
| 1530 | signature = _stat_signature(path) |
| 1531 | |
| 1532 | incremental = _try_incremental_cache_entry(path, cache_key, signature) |
| 1533 | if incremental is not None: |
| 1534 | return incremental |
| 1535 | |
| 1536 | fields, records = _parse_records_file(collection, path) |
| 1537 | id_index = _build_id_index(records) |
| 1538 | if signature is not None: |
| 1539 | if len(records) <= _records_cache_max_rows(): |
| 1540 | _store_records_cache(cache_key, signature, fields, records, id_index, signature[1]) |
| 1541 | else: |
| 1542 | # Too large to cache: don't pin it, and drop any stale entry |
| 1543 | # left over from before this collection grew past the |
| 1544 | # threshold, so it doesn't linger under an old signature. |
| 1545 | _RECORDS_CACHE.pop(cache_key, None) |
| 1546 | return fields, records, id_index |
| 1547 | |
| 1548 | |
| 1549 | def _try_incremental_cache_entry( |
| 1550 | path: Path, |
| 1551 | cache_key: str, |
| 1552 | signature: tuple[int, int, int] | None, |
| 1553 | ) -> tuple[list[str], list[dict[str, str]], dict[str, int]] | None: |
| 1554 | """Attempt the tail-delta incremental fold for an append-mode file that |
| 1555 | grew in place since it was last cached (docs/append-only-storage- |
| 1556 | design.md Sidecars, bullet 2: "when a stat signature changes by |
| 1557 | growth alone, parse only the tail delta"). Returns the fresh (fields, |
| 1558 | records, id_index) on success, or None when the fast path doesn't |
| 1559 | apply -- _cache_entry then falls back to its ordinary full parse, |
| 1560 | unchanged. |
| 1561 | |
| 1562 | This is what makes a WARM cache stay O(delta) instead of O(file) when |
| 1563 | the growth happened outside this process's own write path (another |
| 1564 | process's writer, the CLI, a sibling pool worker, or this process's |
| 1565 | cache having gone cold and been rebuilt from a stat mismatch) -- the |
| 1566 | in-process fast-append path (_refresh_records_cache_after_append) |
| 1567 | already keeps this process's OWN writes O(1); this covers the gap |
| 1568 | where the growth was observed on a read instead. |
| 1569 | |
| 1570 | Preconditions, ALL required (anything off -> None, no partial work is |
| 1571 | trusted): |
| 1572 | - a stat signature could be taken (file still exists, readable) |
| 1573 | - a PRIOR entry is warm in _RECORDS_CACHE for this exact path (a |
| 1574 | cold/evicted/never-cached collection has nothing to build on -- |
| 1575 | the ordinary full parse is the only option) |
| 1576 | - same inode as the prior entry's signature -- a classic-mode |
| 1577 | rewrite, and an append-mode compaction/transition/new-field |
| 1578 | rewrite, all produce a fresh inode via temp+replace |
| 1579 | (_write_collection_records), so this alone rules out every case |
| 1580 | except an in-place append-mode growth |
| 1581 | - the new size is strictly greater than the prior entry's |
| 1582 | covered_bytes -- pure growth, nothing rewritten in place |
| 1583 | - the file's CURRENT physical header is append-format (`_op` first |
| 1584 | column) and its logical fields match the prior entry's fields |
| 1585 | exactly (a header/field change under the same inode should be |
| 1586 | impossible given the point above, but this stays defensive |
| 1587 | rather than trusting that unconditionally) |
| 1588 | |
| 1589 | On success, parses only the new bytes (_append_tail_delta_records, |
| 1590 | the shared torn-tail-tolerant tail reader also used by the id->offset |
| 1591 | sidecar's catch-up scan -- see _scan_append_tail) and folds them onto |
| 1592 | the PRIOR entry's records/id_index -- never mutating the prior |
| 1593 | entry's own list/dict in place (a fresh list/dict is always built; |
| 1594 | other code in this same event-loop turn may still hold references to |
| 1595 | the prior entry's objects, same rule as every other cache-installing |
| 1596 | function in this module). |
| 1597 | |
| 1598 | A tombstone (`_op == "del"`) ANYWHERE in the delta bails out (returns |
| 1599 | None, falling back to the full fold) rather than special-casing |
| 1600 | position shifts: a delete removes a slot and shifts every later |
| 1601 | position down by one (see _fold_append_rows), which this cheap |
| 1602 | replace-or-append merge does not model. Deletes are the rare case for |
| 1603 | the append-shaped workloads this exists for (game-server state, |
| 1604 | impression logs, price history), so this is an accepted, documented |
| 1605 | trade, not a gap -- correctness always wins over speed here. A |
| 1606 | blank-id row in the delta (only reachable via a hand-edited file -- |
| 1607 | see _fold_append_rows) bails out for the same reason: its fold |
| 1608 | position depends on the FULL row history, which an incremental merge |
| 1609 | doesn't have. |
| 1610 | |
| 1611 | The row-count cache threshold (DBBASIC_RECORDS_CACHE_MAX_ROWS) is |
| 1612 | still honored: a successful merge that grows the collection past it |
| 1613 | is still returned to the caller (correct, and far cheaper than a |
| 1614 | full parse either way), but is evicted rather than (re)cached -- the |
| 1615 | same rule _cache_entry's full-parse branch applies just above. |
| 1616 | """ |
| 1617 | if signature is None: |
| 1618 | return None |
| 1619 | |
| 1620 | prior = _RECORDS_CACHE.get(cache_key) |
| 1621 | if prior is None: |
| 1622 | return None |
| 1623 | old_signature, old_fields, old_records, old_id_index, old_covered = prior |
| 1624 | |
| 1625 | if old_signature[2] != signature[2]: |
| 1626 | return None # different inode: not an in-place append |
| 1627 | if signature[1] <= old_covered: |
| 1628 | return None # no growth beyond what's already folded |
| 1629 | |
| 1630 | physical_header = _physical_header(path) |
| 1631 | if not physical_header or physical_header[0] != OP_FIELD: |
| 1632 | return None # not (or no longer) append-physical |
| 1633 | if physical_header[1:] != old_fields: |
| 1634 | return None # defensive: header shouldn't change under the same inode |
| 1635 | |
| 1636 | delta_rows, new_covered = _append_tail_delta_records(path, physical_header, old_covered) |
| 1637 | if not delta_rows: |
| 1638 | return None # nothing complete to fold yet (e.g. a torn tail only) |
| 1639 | |
| 1640 | if any( |
| 1641 | row.get(OP_FIELD, OP_UPSERT) == OP_DELETE or not row.get("id") |
| 1642 | for row in delta_rows |
| 1643 | ): |
| 1644 | return None # tombstone or blank id: fall back to a full fold |
| 1645 | |
| 1646 | new_records = list(old_records) |
| 1647 | new_id_index = dict(old_id_index) |
| 1648 | for row in delta_rows: |
| 1649 | record_id = row["id"] |
| 1650 | clean = {key: value for key, value in row.items() if key != OP_FIELD} |
| 1651 | idx = new_id_index.get(record_id) |
| 1652 | if idx is not None: |
| 1653 | new_records[idx] = clean |
| 1654 | else: |
| 1655 | new_id_index[record_id] = len(new_records) |
| 1656 | new_records.append(clean) |
| 1657 | |
| 1658 | global _INCREMENTAL_CACHE_FOLDS |
| 1659 | _INCREMENTAL_CACHE_FOLDS += 1 |
| 1660 | |
| 1661 | if len(new_records) <= _records_cache_max_rows(): |
| 1662 | _store_records_cache(cache_key, signature, old_fields, new_records, new_id_index, new_covered) |
| 1663 | else: |
| 1664 | _RECORDS_CACHE.pop(cache_key, None) |
| 1665 | return old_fields, new_records, new_id_index |
| 1666 | |
| 1667 | |
| 1668 | def _parse_records_file(collection: str, path: Path) -> tuple[list[str], list[dict[str, str]]]: |
| 1669 | """Parse one records.tsv file into (fields, records). |
| 1670 | |
| 1671 | Uses csv.reader rather than csv.DictReader: csv.reader is the same |
| 1672 | C-accelerated tokenizer DictReader wraps, but skips DictReader's |
| 1673 | per-row dict-construction overhead (zip + restkey/restval handling) in |
| 1674 | favor of a direct index walk here. Using csv.reader (not a naive |
| 1675 | line.split("\t")) is required for correctness, not just speed: a value |
| 1676 | written by csv.DictWriter's default QUOTE_MINIMAL may legitimately |
| 1677 | contain a quoted tab or embedded newline, and only real CSV/TSV parsing |
| 1678 | reconstructs that value correctly. |
| 1679 | |
| 1680 | Dispatches on the physical header (docs/append-only-storage-design.md): |
| 1681 | a file whose first column is `_op` is in append-only physical format |
| 1682 | and is parsed+folded by _parse_append_body; every other file is parsed |
| 1683 | by the original classic-mode routine, unchanged. This dispatch is by |
| 1684 | the FILE's own header, not the collection's current schema `storage` |
| 1685 | setting -- a collection that just switched storage modes keeps reading |
| 1686 | correctly until its next write physically transitions the file (see |
| 1687 | _persist_write), and a collection that switched back to classic still |
| 1688 | reads correctly (via the append path) until its next write compacts |
| 1689 | the `_op` column away. |
| 1690 | """ |
| 1691 | with path.open(newline="") as handle: |
| 1692 | reader = csv.reader(handle, delimiter="\t") |
| 1693 | try: |
| 1694 | header = next(reader) |
| 1695 | except StopIteration: |
| 1696 | header = None |
| 1697 | _validate_header(collection, header) |
| 1698 | physical_fields = list(header) |
| 1699 | |
| 1700 | if physical_fields and physical_fields[0] == OP_FIELD: |
| 1701 | with path.open(newline="") as handle: |
| 1702 | text = handle.read() |
| 1703 | folded_records, physical_row_count = _parse_append_body(collection, text, physical_fields) |
| 1704 | _maybe_flag_auto_compact( |
| 1705 | path, physical_row_count=physical_row_count, live_row_count=len(folded_records) |
| 1706 | ) |
| 1707 | return physical_fields[1:], folded_records |
| 1708 | |
| 1709 | return physical_fields, _parse_classic_records(collection, path, physical_fields) |
| 1710 | |
| 1711 | |
| 1712 | def _parse_classic_records( |
| 1713 | collection: str, path: Path, fields: list[str] |
| 1714 | ) -> list[dict[str, str]]: |
| 1715 | """Parse the data rows of a classic-mode (no `_op` column) file. |
| 1716 | |
| 1717 | Replicates csv.DictReader's exact semantics for short/long rows and |
| 1718 | blank-line skipping -- see the block comment above _RECORDS_CACHE and |
| 1719 | the tests in tests/test_object_records.py for the specific cases this |
| 1720 | was checked against on the pre-change code. Unchanged from before |
| 1721 | append-only storage existed: a strict parse, no torn-tail tolerance. |
| 1722 | """ |
| 1723 | field_count = len(fields) |
| 1724 | with path.open(newline="") as handle: |
| 1725 | reader = csv.reader(handle, delimiter="\t") |
| 1726 | next(reader, None) # header, already parsed/validated by the caller |
| 1727 | |
| 1728 | records: list[dict[str, str]] = [] |
| 1729 | row_number = 1 # the header is row 1; DictReader's enumerate started data rows at 2 |
| 1730 | for row in reader: |
| 1731 | # csv.DictReader's __next__ silently skips rows the underlying |
| 1732 | # csv.reader returns as [] (blank physical lines) rather than |
| 1733 | # emitting a record for them; replicate that here so row_number |
| 1734 | # (used only in the error message below) still lines up with |
| 1735 | # DictReader's numbering of *emitted* rows, not physical lines. |
| 1736 | if not row: |
| 1737 | continue |
| 1738 | row_number += 1 |
| 1739 | if len(row) > field_count: |
| 1740 | # Mirrors DictReader putting overflow values under the |
| 1741 | # `None` restkey, which the pre-change code detected via |
| 1742 | # `None in row`. |
| 1743 | raise ValueError( |
| 1744 | f"Collection records file has extra fields on row {row_number}: {collection}" |
| 1745 | ) |
| 1746 | # Mirrors DictReader's restval=None for a short row, which the |
| 1747 | # pre-change code then normalized to "" (`value if value is not |
| 1748 | # None else ""`) -- do that normalization directly here. |
| 1749 | record = { |
| 1750 | fields[i]: (row[i] if i < len(row) else "") |
| 1751 | for i in range(field_count) |
| 1752 | } |
| 1753 | records.append(record) |
| 1754 | |
| 1755 | return records |
| 1756 | |
| 1757 | |
| 1758 | def _committed_prefix_len(text: str) -> int: |
| 1759 | """Char length of the prefix of `text` ending at the last CSV ROW |
| 1760 | TERMINATOR -- a "\\n" seen OUTSIDE a quoted field. |
| 1761 | |
| 1762 | QUOTE-AWARE, single O(len) forward pass tracking csv quote state: a "\\n" |
| 1763 | INSIDE a quoted field (a multi-line cell) is field content, never a row |
| 1764 | boundary. "" inside a quoted field is an escaped quote (stays quoted), |
| 1765 | matching csv QUOTE_MINIMAL/QUOTE_ALL. Everything after the last real |
| 1766 | terminator is an incomplete (torn) trailing row -- an interrupted write -- |
| 1767 | and is excluded. This can only ever trim a torn TAIL, never bisect a |
| 1768 | committed row. |
| 1769 | """ |
| 1770 | in_quote = False |
| 1771 | last_row_end = 0 |
| 1772 | i = 0 |
| 1773 | n = len(text) |
| 1774 | while i < n: |
| 1775 | c = text[i] |
| 1776 | if in_quote: |
| 1777 | if c == '"': |
| 1778 | if i + 1 < n and text[i + 1] == '"': |
| 1779 | i += 2 # escaped "" -- stays inside the quoted field |
| 1780 | continue |
| 1781 | in_quote = False |
| 1782 | i += 1 |
| 1783 | else: |
| 1784 | if c == '"': |
| 1785 | in_quote = True |
| 1786 | elif c == "\n": |
| 1787 | last_row_end = i + 1 |
| 1788 | i += 1 |
| 1789 | return last_row_end |
| 1790 | |
| 1791 | |
| 1792 | # Public alias. `_committed_prefix_len` is the one correct answer to "where |
| 1793 | # does the last complete CSV row end", and it is needed in more than one |
| 1794 | # module: object_backup_index parses records.tsv out of a backup archive on |
| 1795 | # the restore path and must reach the same verdict. |
| 1796 | # |
| 1797 | # It is exported rather than copied because the copy is what went wrong. |
| 1798 | # object_backup_index kept its own `_drop_torn_tail` with a docstring saying |
| 1799 | # it "mirrors object_records._drop_torn_tail" -- and when this module's copy |
| 1800 | # was upgraded to be quote-aware, the mirror was not, silently, for months. |
| 1801 | # Two implementations of one invariant drift; one implementation cannot. |
| 1802 | committed_prefix_len = _committed_prefix_len |
| 1803 | |
| 1804 | |
| 1805 | def _drop_torn_tail(text: str) -> str: |
| 1806 | """Return `text` truncated to the end of its last COMPLETE csv row. |
| 1807 | |
| 1808 | QUOTE-AWARE. This was previously a quote-BLIND check ("ends with \\n", else |
| 1809 | rfind the last \\n) which mistook a "\\n" INSIDE a quoted multi-line field |
| 1810 | for a row boundary -- silently resurrecting a torn multi-line row and, in |
| 1811 | append mode, cascading to swallow later writes (the torn-write durability |
| 1812 | characterization mapped this to ~97% of a multi-line row's write window). |
| 1813 | A row is committed only when a row-terminating "\\n" (outside any quoted |
| 1814 | field) follows it; anything after the last such terminator is an |
| 1815 | interrupted write and is dropped. |
| 1816 | |
| 1817 | Fast path: a text containing no '"' at all has no quoted fields, so every |
| 1818 | "\\n" is a row terminator and the cheap check is exact -- this keeps |
| 1819 | collections whose cells never need quoting (no tab/quote/newline in any |
| 1820 | value) on the original O(1)/rfind path. Only text with quoting pays the |
| 1821 | O(len) quote-aware scan, and that path is already parsing the same text. |
| 1822 | """ |
| 1823 | if text == "": |
| 1824 | return text |
| 1825 | if '"' not in text: |
| 1826 | if text.endswith("\n"): |
| 1827 | return text |
| 1828 | cut = text.rfind("\n") |
| 1829 | return text[: cut + 1] if cut >= 0 else "" |
| 1830 | return text[:_committed_prefix_len(text)] |
| 1831 | |
| 1832 | |
| 1833 | def _parse_append_body( |
| 1834 | collection: str, text: str, physical_fields: list[str] |
| 1835 | ) -> tuple[list[dict[str, str]], int]: |
| 1836 | """Parse+fold an append-mode file's body. Returns (folded_records, physical_row_count). |
| 1837 | |
| 1838 | Torn-tail tolerant: a trailing unterminated line is dropped first |
| 1839 | (_drop_torn_tail); a row that still fails to parse (e.g. a value with |
| 1840 | an embedded newline whose closing quote never arrived, so its |
| 1841 | apparent line boundary landed mid-field) stops consumption at that |
| 1842 | point rather than raising -- everything after an unparseable point is, |
| 1843 | by construction, either that same torn write or unreachable. This |
| 1844 | differs from classic mode, which keeps raising on malformed content |
| 1845 | (see _parse_classic_records): append mode's contract is specifically |
| 1846 | that in-flight writes are tolerated, not silently-corrupt files. |
| 1847 | """ |
| 1848 | field_count = len(physical_fields) |
| 1849 | reader = csv.reader(io.StringIO(_drop_torn_tail(text)), delimiter="\t") |
| 1850 | next(reader, None) # header, already parsed/validated by the caller |
| 1851 | |
| 1852 | physical_rows: list[dict[str, str]] = [] |
| 1853 | while True: |
| 1854 | try: |
| 1855 | row = next(reader) |
| 1856 | except StopIteration: |
| 1857 | break |
| 1858 | except csv.Error: |
| 1859 | break |
| 1860 | if not row: |
| 1861 | continue |
| 1862 | if len(row) > field_count: |
| 1863 | raise ValueError( |
| 1864 | f"Collection records file has extra fields: {collection}" |
| 1865 | ) |
| 1866 | record = { |
| 1867 | physical_fields[i]: (row[i] if i < len(row) else "") |
| 1868 | for i in range(field_count) |
| 1869 | } |
| 1870 | physical_rows.append(record) |
| 1871 | |
| 1872 | return _fold_append_rows(physical_rows), len(physical_rows) |
| 1873 | |
| 1874 | |
| 1875 | def _fold_append_rows(physical_rows: list[dict[str, str]]) -> list[dict[str, str]]: |
| 1876 | """Fold physical append-mode rows into live records, last-wins per id. |
| 1877 | |
| 1878 | An id's position in the returned list is where it was inserted since |
| 1879 | it was last live: an upsert for an id already present updates it in |
| 1880 | place (same position, matching classic mode's update-in-place); an |
| 1881 | upsert for a not-currently-live id inserts at the current end |
| 1882 | (matching classic mode's create-appends); a delete (`_op == "del"`) |
| 1883 | removes the id's slot entirely (matching classic mode's delete |
| 1884 | popping the row) -- so a later upsert for that same id is a fresh |
| 1885 | insertion at the end, not a return to its old position. Built with an |
| 1886 | OrderedDict so this falls out of normal dict semantics rather than |
| 1887 | needing to be modeled by hand; this is what gives append mode the |
| 1888 | same list order as classic mode for the same operation sequence (see |
| 1889 | the equivalence tests in tests/test_object_records.py). `_op` is |
| 1890 | stripped from every folded record. |
| 1891 | |
| 1892 | Rows with a blank/missing id (only reachable via a hand-edited file -- |
| 1893 | every write path in this module requires a valid id) cannot be |
| 1894 | last-wins folded against each other for lack of a key; each is kept as |
| 1895 | its own entry and appended after the id-keyed records, which does not |
| 1896 | preserve their original interleaving with id-keyed rows. This is a |
| 1897 | documented limitation of a pre-existing edge case, not a behavior this |
| 1898 | feature needs to support: normalize_record_payload never lets a create |
| 1899 | omit its id, and update always operates on an already-valid id. |
| 1900 | """ |
| 1901 | folded: "OrderedDict[str, dict[str, str]]" = OrderedDict() |
| 1902 | blank_id_rows: list[dict[str, str]] = [] |
| 1903 | blank_counter = 0 |
| 1904 | for row in physical_rows: |
| 1905 | op = row.get(OP_FIELD, OP_UPSERT) |
| 1906 | record_id = row.get("id", "") |
| 1907 | clean = {key: value for key, value in row.items() if key != OP_FIELD} |
| 1908 | if not record_id: |
| 1909 | if op == OP_DELETE: |
| 1910 | continue |
| 1911 | blank_id_rows.append((f"__blank__{blank_counter}", clean)) |
| 1912 | blank_counter += 1 |
| 1913 | continue |
| 1914 | if op == OP_DELETE: |
| 1915 | folded.pop(record_id, None) |
| 1916 | continue |
| 1917 | folded[record_id] = clean |
| 1918 | |
| 1919 | return list(folded.values()) + [record for _, record in blank_id_rows] |
| 1920 | |
| 1921 | |
| 1922 | def _scan_append_tail( |
| 1923 | path: Path, physical_fields: list[str], start_byte: int |
| 1924 | ) -> tuple[list[tuple[int, int, dict[str, str]]], int]: |
| 1925 | """Read and parse complete physical rows of an append-mode file from |
| 1926 | `start_byte` to EOF, returning (rows, covered_bytes) where each row is |
| 1927 | (row_start_offset, row_end_offset, physical_record) -- `physical_record` |
| 1928 | keyed by every column in `physical_fields`, `_op` included, exactly |
| 1929 | like a row from _parse_append_body before folding. `covered_bytes` is |
| 1930 | the byte offset up to which parsing is authoritative (>= start_byte, |
| 1931 | equal to it when nothing new/complete was found). `start_byte` must |
| 1932 | land exactly on a row boundary -- every offset any caller ever hands |
| 1933 | back here (a prior scan/cache entry's covered_bytes, or the |
| 1934 | records-header length) always does, since it is always either 0 or a |
| 1935 | previous row's end. |
| 1936 | |
| 1937 | Torn-tail tolerant exactly like _parse_append_body: a trailing |
| 1938 | unterminated physical line is dropped first (_drop_torn_tail), and a |
| 1939 | row that still fails to parse (or reports more fields than the header |
| 1940 | -- real corruption, not a torn write) stops consumption at that point |
| 1941 | rather than raising. NEVER raises: this is the shared low-level scan |
| 1942 | behind two callers that both need "read complete new rows appended |
| 1943 | since an earlier point, without re-reading or re-folding the whole |
| 1944 | file" -- the id->offset sidecar's incremental catch-up |
| 1945 | (_scan_append_rows_for_offsets, which projects each row down to just |
| 1946 | its (op, id) for the offset index) and the records-cache's tail-delta |
| 1947 | incremental fold (_append_tail_delta_records, which keeps every field |
| 1948 | since it must produce real records). Neither caller may raise either, |
| 1949 | so this doesn't. |
| 1950 | |
| 1951 | Byte offsets are computed by decoding `path`'s bytes from `start_byte` |
| 1952 | once, then, after each row csv.reader consumes, converting the |
| 1953 | now-consumed prefix of that decoded text back to its exact byte |
| 1954 | length (`str.encode` on the newly consumed slice only, not the whole |
| 1955 | prefix each time, so total work stays O(scanned bytes) instead of |
| 1956 | O(scanned bytes squared)). `start_byte` itself is always a byte |
| 1957 | offset immediately after a "\\n", which is a single ASCII byte and |
| 1958 | therefore never a UTF-8 continuation byte -- so slicing raw bytes from |
| 1959 | there and decoding is always safe. |
| 1960 | """ |
| 1961 | try: |
| 1962 | with path.open("rb") as handle: |
| 1963 | handle.seek(start_byte) |
| 1964 | raw = handle.read() |
| 1965 | except OSError: |
| 1966 | return [], start_byte |
| 1967 | if not raw: |
| 1968 | return [], start_byte |
| 1969 | |
| 1970 | try: |
| 1971 | text = raw.decode("utf-8") |
| 1972 | except UnicodeDecodeError: |
| 1973 | return [], start_byte |
| 1974 | text = _drop_torn_tail(text) |
| 1975 | if not text: |
| 1976 | return [], start_byte |
| 1977 | |
| 1978 | field_count = len(physical_fields) |
| 1979 | stream = io.StringIO(text) |
| 1980 | reader = csv.reader(stream, delimiter="\t") |
| 1981 | rows: list[tuple[int, int, dict[str, str]]] = [] |
| 1982 | prev_char = 0 |
| 1983 | byte_cursor = start_byte |
| 1984 | while True: |
| 1985 | try: |
| 1986 | row = next(reader) |
| 1987 | except StopIteration: |
| 1988 | break |
| 1989 | except csv.Error: |
| 1990 | break |
| 1991 | cur_char = stream.tell() |
| 1992 | row_bytes = len(text[prev_char:cur_char].encode("utf-8")) |
| 1993 | prev_char = cur_char |
| 1994 | row_start = byte_cursor |
| 1995 | row_end = byte_cursor + row_bytes |
| 1996 | byte_cursor = row_end |
| 1997 | if not row: |
| 1998 | continue |
| 1999 | if len(row) > field_count: |
| 2000 | break |
| 2001 | record = { |
| 2002 | physical_fields[i]: (row[i] if i < len(row) else "") |
| 2003 | for i in range(field_count) |
| 2004 | } |
| 2005 | rows.append((row_start, row_end, record)) |
| 2006 | |
| 2007 | return rows, byte_cursor |
| 2008 | |
| 2009 | |
| 2010 | def _append_tail_delta_records( |
| 2011 | path: Path, physical_fields: list[str], start_byte: int |
| 2012 | ) -> tuple[list[dict[str, str]], int]: |
| 2013 | """Parse complete physical rows appended to an append-mode file since |
| 2014 | `start_byte`, returning (physical_rows, covered_bytes) -- a thin |
| 2015 | projection of _scan_append_tail down to just the row dicts (`_op` |
| 2016 | included, unfolded), for _try_incremental_cache_entry's tail-delta |
| 2017 | fold. See _scan_append_tail for the shared torn-tail-tolerant scan |
| 2018 | this builds on: never raises, stops at the first row it can't make |
| 2019 | sense of. |
| 2020 | """ |
| 2021 | scanned, covered_bytes = _scan_append_tail(path, physical_fields, start_byte) |
| 2022 | return [record for _, _, record in scanned], covered_bytes |
| 2023 | |
| 2024 | |
| 2025 | def _append_compact_min_rows() -> int: |
| 2026 | return _env_int(APPEND_COMPACT_MIN_ROWS_ENV, _DEFAULT_APPEND_COMPACT_MIN_ROWS) |
| 2027 | |
| 2028 | |
| 2029 | def append_compact_min_rows() -> int: |
| 2030 | """Public read of DBBASIC_APPEND_COMPACT_MIN_ROWS (default 10_000). |
| 2031 | |
| 2032 | Exists so callers outside this module (object_daemon.py's |
| 2033 | process_compactions) can apply the exact same physical-row-count floor |
| 2034 | _maybe_flag_auto_compact uses for its own auto-compaction trigger, |
| 2035 | without duplicating the env var's default value. |
| 2036 | """ |
| 2037 | return _append_compact_min_rows() |
| 2038 | |
| 2039 | |
| 2040 | def _maybe_flag_auto_compact(path: Path, *, physical_row_count: int, live_row_count: int) -> None: |
| 2041 | """Flag a collection for compaction on its NEXT WRITE. |
| 2042 | |
| 2043 | Never compacts here -- this runs from the read/parse path, and reads |
| 2044 | must stay read-only (docs/append-only-storage-design.md Compaction). |
| 2045 | Triggers when the physical file is at least DBBASIC_APPEND_COMPACT_ |
| 2046 | MIN_ROWS rows (so a small collection's ordinary churn never matters) |
| 2047 | AND superseded-or-deleted rows outnumber live rows. |
| 2048 | """ |
| 2049 | if physical_row_count <= _append_compact_min_rows(): |
| 2050 | return |
| 2051 | stale_row_count = physical_row_count - live_row_count |
| 2052 | if stale_row_count > live_row_count: |
| 2053 | _PENDING_COMPACTION.add(str(path.resolve(strict=False))) |
| 2054 | |
| 2055 | |
| 2056 | def _physical_header(path: Path) -> list[str] | None: |
| 2057 | """Return the raw on-disk header (leading `_op` included, if present). |
| 2058 | |
| 2059 | None when the file doesn't exist or is empty. Cheap: reads only the |
| 2060 | first physical line, not the whole file. |
| 2061 | """ |
| 2062 | if not path.exists(): |
| 2063 | return None |
| 2064 | with path.open(newline="") as handle: |
| 2065 | reader = csv.reader(handle, delimiter="\t") |
| 2066 | try: |
| 2067 | return next(reader) |
| 2068 | except StopIteration: |
| 2069 | return None |
| 2070 | |
| 2071 | |
| 2072 | def _write_collection_records( |
| 2073 | collection: str, |
| 2074 | path: Path, |
| 2075 | fields: list[str], |
| 2076 | records: list[dict[str, str]], |
| 2077 | *, |
| 2078 | cache_fields: list[str] | None = None, |
| 2079 | ) -> None: |
| 2080 | """Rewrite the whole records file atomically (temp file + replace). |
| 2081 | |
| 2082 | `fields` is the PHYSICAL header to write -- for an append-mode full |
| 2083 | rewrite (transition-in, compaction, or the new-field-fallback rewrite; |
| 2084 | see _persist_write) this includes a leading `_op` column, which every |
| 2085 | record in `records` implicitly projects to "" (upsert) since none of |
| 2086 | them carry an `_op` key (docs/append-only-storage-design.md). |
| 2087 | `cache_fields`, when given, is the LOGICAL field list (without `_op`) |
| 2088 | to store in the module cache instead of `fields` -- callers writing an |
| 2089 | append-format file must pass this, or `_op` would leak into the cache |
| 2090 | and, from there, into API responses (get/list/create/update all |
| 2091 | return cache-derived data). Classic-mode callers omit it and the |
| 2092 | written `fields` doubles as the cache's fields, exactly as before this |
| 2093 | parameter existed. |
| 2094 | """ |
| 2095 | if "id" not in fields: |
| 2096 | fields = ["id", *fields] |
| 2097 | |
| 2098 | path.parent.mkdir(parents=True, exist_ok=True) |
| 2099 | temp_path = path.with_name(f".{path.name}.{os.getpid()}.{threading.get_ident()}.tmp") |
| 2100 | try: |
| 2101 | with temp_path.open("w", newline="") as handle: |
| 2102 | writer = csv.DictWriter( |
| 2103 | handle, |
| 2104 | fieldnames=fields, |
| 2105 | delimiter="\t", |
| 2106 | lineterminator="\n", |
| 2107 | extrasaction="ignore", |
| 2108 | quoting=(csv.QUOTE_ALL if _rows_need_full_quoting(records) else csv.QUOTE_MINIMAL), |
| 2109 | ) |
| 2110 | writer.writeheader() |
| 2111 | for record in records: |
| 2112 | writer.writerow(_project_record(record, fields)) |
| 2113 | temp_path.replace(path) |
| 2114 | finally: |
| 2115 | if temp_path.exists(): |
| 2116 | temp_path.unlink() |
| 2117 | |
| 2118 | _refresh_records_cache(collection, path, cache_fields if cache_fields is not None else fields, records) |
| 2119 | |
| 2120 | |
| 2121 | def _refresh_records_cache( |
| 2122 | collection: str, |
| 2123 | path: Path, |
| 2124 | fields: list[str], |
| 2125 | records: list[dict[str, str]], |
| 2126 | ) -> None: |
| 2127 | """Populate the module cache with the content just written to `path`. |
| 2128 | |
| 2129 | Called immediately after the atomic replace above, under the caller's |
| 2130 | _records_file_lock, so this process's view of the file is authoritative |
| 2131 | right now. Storing the new content (rather than merely invalidating) |
| 2132 | means the next read for this collection -- often the same request, |
| 2133 | returning its own write -- skips re-parsing the file it just wrote. |
| 2134 | |
| 2135 | If the post-write stat can't be trusted (the file vanished, or some |
| 2136 | other OSError), skip caching entirely rather than caching something |
| 2137 | stale or guessed: the next reader will just parse fresh, same as |
| 2138 | before this change existed. |
| 2139 | |
| 2140 | Honors the same DBBASIC_RECORDS_CACHE_MAX_ROWS threshold as |
| 2141 | _cache_entry: a write that grows a collection past the threshold is |
| 2142 | not cached (and any stale smaller-collection entry under this key is |
| 2143 | dropped), so a single giant write can't occupy an LRU slot. |
| 2144 | """ |
| 2145 | signature = _stat_signature(path) |
| 2146 | if signature is None: |
| 2147 | return |
| 2148 | |
| 2149 | cache_key = str(path.resolve(strict=False)) |
| 2150 | if len(records) > _records_cache_max_rows(): |
| 2151 | _RECORDS_CACHE.pop(cache_key, None) |
| 2152 | return |
| 2153 | |
| 2154 | # Project every record to the full field set before caching, so a |
| 2155 | # cache hit yields exactly what a fresh parse of this file would |
| 2156 | # yield (same as what was just written: `_project_record` fills any |
| 2157 | # field missing from a given record with "", matching how |
| 2158 | # csv.DictWriter/extrasaction wrote that row). |
| 2159 | cached_records = [_project_record(record, fields) for record in records] |
| 2160 | id_index = _build_id_index(cached_records) |
| 2161 | _store_records_cache(cache_key, signature, list(fields), cached_records, id_index, signature[1]) |
| 2162 | |
| 2163 | |
| 2164 | def _refresh_records_cache_after_append( |
| 2165 | collection: str, |
| 2166 | path: Path, |
| 2167 | fields: list[str], |
| 2168 | delta_op: str, |
| 2169 | delta_id: str, |
| 2170 | projected_row: dict[str, str], |
| 2171 | ) -> bool: |
| 2172 | """Incrementally update the cache after a fast append. Returns True if |
| 2173 | it did (the caller then skips the full _refresh_records_cache rebuild). |
| 2174 | |
| 2175 | The entire point of the fast-append path is O(1) disk work -- calling |
| 2176 | the general _refresh_records_cache after every append would silently |
| 2177 | reintroduce an O(n) cost per write, just moved from disk I/O to cache |
| 2178 | bookkeeping (a full re-projection of every record plus a full |
| 2179 | id_index rebuild). This instead derives the new cache entry from the |
| 2180 | PRIOR entry plus this one delta: |
| 2181 | |
| 2182 | - create (an id not already in the index): `old_records + [row]` |
| 2183 | and `dict(old_id_index)` plus one new key -- still technically |
| 2184 | O(n) (a fresh list/dict must be built; nothing cached is ever |
| 2185 | mutated in place, see the ALIASING SAFETY note on |
| 2186 | _RECORDS_CACHE), but as bulk C-level copies (list concat, dict |
| 2187 | copy) rather than a Python-level per-record rebuild. |
| 2188 | - update (an id already in the index -- always at the SAME |
| 2189 | position, since _fold_append_rows updates in place): `list( |
| 2190 | old_records)` with one element replaced; the id_index is REUSED |
| 2191 | UNCHANGED (no copy at all), since no id's position moved. |
| 2192 | - delete: a genuine O(n) Python-level index rebuild (removing a |
| 2193 | position means every later position shifts down by one) -- the |
| 2194 | one case this doesn't make cheap. Deletes are the rare case for |
| 2195 | the append-shaped workloads this feature targets (game-server |
| 2196 | state, impression logs, price history), so this is an accepted, |
| 2197 | documented trade rather than a gap. |
| 2198 | |
| 2199 | Returns False when there is no usable prior entry to build on -- cold |
| 2200 | cache, evicted, over the row-count cache threshold, or (defensively) |
| 2201 | a fields mismatch. The caller (_persist_write's fast-append branch) |
| 2202 | does NOT force a full rebuild in that case (an earlier version of |
| 2203 | this code did, via folded_records_fn() + _refresh_records_cache -- |
| 2204 | exactly the O(n)-per-write cost the id->offset sidecar, docs/append- |
| 2205 | only-storage-design.md item 4, exists to eliminate for a collection |
| 2206 | too large to be a "usable prior entry" in the first place). A False |
| 2207 | return simply leaves _RECORDS_CACHE as it was: the next reader either |
| 2208 | hits the sidecar (if the file is append-physical and over threshold) |
| 2209 | or pays one full fold on a genuine cold/small-collection miss, same |
| 2210 | as any other cache miss elsewhere in this module. |
| 2211 | """ |
| 2212 | signature = _stat_signature(path) |
| 2213 | if signature is None: |
| 2214 | return True # nothing to cache either way; nothing left to do |
| 2215 | |
| 2216 | cache_key = str(path.resolve(strict=False)) |
| 2217 | prior = _RECORDS_CACHE.get(cache_key) |
| 2218 | if prior is None or prior[1] != fields: |
| 2219 | return False |
| 2220 | |
| 2221 | _, _prior_fields, old_records, old_id_index, _prior_covered = prior |
| 2222 | if len(old_records) > _records_cache_max_rows(): |
| 2223 | _RECORDS_CACHE.pop(cache_key, None) |
| 2224 | return True |
| 2225 | |
| 2226 | if delta_op == OP_DELETE: |
| 2227 | idx = old_id_index.get(delta_id) |
| 2228 | if idx is None: |
| 2229 | return False |
| 2230 | new_records = old_records[:idx] + old_records[idx + 1:] |
| 2231 | new_id_index = { |
| 2232 | key: (value if value < idx else value - 1) |
| 2233 | for key, value in old_id_index.items() |
| 2234 | if key != delta_id |
| 2235 | } |
| 2236 | else: |
| 2237 | idx = old_id_index.get(delta_id) |
| 2238 | if idx is not None: |
| 2239 | new_records = list(old_records) |
| 2240 | new_records[idx] = projected_row |
| 2241 | new_id_index = old_id_index |
| 2242 | else: |
| 2243 | new_records = old_records + [projected_row] |
| 2244 | new_id_index = dict(old_id_index) |
| 2245 | new_id_index[delta_id] = len(old_records) |
| 2246 | |
| 2247 | if len(new_records) > _records_cache_max_rows(): |
| 2248 | _RECORDS_CACHE.pop(cache_key, None) |
| 2249 | return True |
| 2250 | |
| 2251 | # This process just performed the append that produced `signature`, so |
| 2252 | # it authoritatively covers the whole current file, same as any other |
| 2253 | # fully-covering entry (see the covered_bytes bullet in the |
| 2254 | # _RECORDS_CACHE block comment above). |
| 2255 | _store_records_cache(cache_key, signature, list(fields), new_records, new_id_index, signature[1]) |
| 2256 | return True |
| 2257 | |
| 2258 | |
| 2259 | def _repair_torn_tail_lines(path: Path) -> None: |
| 2260 | """Trim a torn trailing LINE from a line-oriented file (the sidecar). |
| 2261 | |
| 2262 | Backward rfind(b"\n") -- quote-BLIND, and correct HERE, for exactly one |
| 2263 | reason: an idx line can never contain a literal tab or newline inside a |
| 2264 | field (ids are restricted to _RECORD_ID_RE; offsets and op are plain |
| 2265 | digits/literals), so "the file ends with \n" really is a complete |
| 2266 | completeness signal for the sidecar. |
| 2267 | |
| 2268 | It is NOT correct for records.tsv, whose quoted cells may hold |
| 2269 | newlines. That distinction is why this is now a separate function: the |
| 2270 | data file's repair below is quote-aware and costs a scan, and pointing |
| 2271 | the sidecar at it would spend that scan on every append to buy nothing. |
| 2272 | Do not "unify" these; they were one function once, and the shared body |
| 2273 | was wrong for one caller and slow for the other. |
| 2274 | """ |
| 2275 | try: |
| 2276 | size = path.stat().st_size |
| 2277 | except OSError: |
| 2278 | return |
| 2279 | if size == 0: |
| 2280 | return |
| 2281 | |
| 2282 | with path.open("rb+") as handle: |
| 2283 | handle.seek(size - 1) |
| 2284 | if handle.read(1) == b"\n": |
| 2285 | return |
| 2286 | |
| 2287 | window = 8192 |
| 2288 | read_from = size |
| 2289 | while read_from > 0: |
| 2290 | read_from = max(0, read_from - window) |
| 2291 | handle.seek(read_from) |
| 2292 | chunk = handle.read(size - read_from) |
| 2293 | idx = chunk.rfind(b"\n") |
| 2294 | if idx >= 0: |
| 2295 | handle.truncate(read_from + idx + 1) |
| 2296 | return |
| 2297 | window *= 2 |
| 2298 | handle.truncate(0) |
| 2299 | |
| 2300 | |
| 2301 | def _repair_torn_tail(path: Path) -> None: |
| 2302 | """Ensure records.tsv ends at a committed CSV row boundary before an |
| 2303 | append lands. QUOTE-AWARE -- substrate bug #2's write side, fixed. |
| 2304 | |
| 2305 | A write interrupted mid-row leaves a fragment; appending after it |
| 2306 | would concatenate the next row onto the fragment -- and if the |
| 2307 | fragment holds an unclosed quoted field, silently swallow that next |
| 2308 | row. The previous body found "the last row boundary" with |
| 2309 | rfind(b"\n"), which mistakes a newline INSIDE a quoted multi-line |
| 2310 | field for a boundary; worse, its fast path returned immediately |
| 2311 | whenever the file's last byte was "\n" -- including a newline inside |
| 2312 | an unclosed quote -- so the very case that needed repair was the case |
| 2313 | it skipped. Exposure was ~97% of a multi-line row's write window. |
| 2314 | |
| 2315 | ## Why the first fix was reverted, and what this one does differently |
| 2316 | |
| 2317 | The 2026-07-20 attempt gated correctness on _RECORDS_CACHE's |
| 2318 | covered_bytes. That value lives ONLY in process memory -- the on-disk |
| 2319 | sidecar header records just the inode -- so the gate read state the |
| 2320 | test harness happened to share: under pytest one interpreter runs the |
| 2321 | whole suite and the cache was warm from other tests' files; standalone |
| 2322 | it was cold every time. "Fails in pytest, passes standalone" was never |
| 2323 | a race; it was a correctness decision delegated to a cache. And a torn |
| 2324 | tail only exists after a process DIED, which is exactly when no cache |
| 2325 | exists. |
| 2326 | |
| 2327 | So the rule here, stated so nobody re-learns it: **a cache may only |
| 2328 | ever SKIP this repair, never choose where it truncates.** |
| 2329 | |
| 2330 | ## How it works |
| 2331 | |
| 2332 | 1. SKIP (sound): a warm _OIDX_CACHE entry, inode-matched, whose |
| 2333 | covered_bytes equals the file's size right now. Every covered value |
| 2334 | in that cache is a genuine committed row boundary (from |
| 2335 | _scan_append_tail, which excludes torn tails, or from this |
| 2336 | process's own completed append), so EOF landing exactly on one |
| 2337 | proves the file is clean. _RECORDS_CACHE is deliberately NOT |
| 2338 | consulted -- see the inline comment for why its covered_bytes lies |
| 2339 | about exactly the files that need repair, which is the probable |
| 2340 | mechanism of the original revert. |
| 2341 | 2. ANCHOR (cheap): the same sidecar-cache covered, when it is short of |
| 2342 | EOF, is a boundary to scan forward from instead of from zero. |
| 2343 | Append-mode files only ever grow in place (any rewrite is an atomic |
| 2344 | rename and changes the inode), and a correct repair never truncates |
| 2345 | below the last committed boundary, so bytes before a recorded |
| 2346 | boundary are stable under a matching inode. |
| 2347 | 3. SCAN: decode the tail with errors="surrogateescape" and take |
| 2348 | _committed_prefix_len -- THE quote-aware answer, the same single |
| 2349 | implementation the read path and object_backup_index use, because a |
| 2350 | second copy of this logic is how the backup module silently carried |
| 2351 | the bug for months. The char prefix is re-encoded to recover the |
| 2352 | exact byte length (surrogateescape round-trips arbitrary bytes). |
| 2353 | 4. Truncate to the last committed boundary found. With no anchor this |
| 2354 | is a full O(n) scan -- paid on the first append after a restart, |
| 2355 | which is the one moment a torn tail can actually exist. |
| 2356 | |
| 2357 | Deliberately NOT trusted: the on-disk sidecar's row_end offsets. They |
| 2358 | would be a durable anchor, but files damaged by the OLD repair may |
| 2359 | carry sidecar offsets that no longer sit on real boundaries, and |
| 2360 | anchoring the fix to state the bug may have corrupted would rebuild |
| 2361 | the bug. Process-local anchors are attested by this process's own |
| 2362 | reads and writes. |
| 2363 | |
| 2364 | The three strict-xfail acceptance tests that held this behaviour map |
| 2365 | the trigger surface; they were rewritten into regression guards when |
| 2366 | this landed. |
| 2367 | """ |
| 2368 | signature = _stat_signature(path) |
| 2369 | if signature is None: |
| 2370 | return |
| 2371 | size = signature[1] |
| 2372 | if size == 0: |
| 2373 | return |
| 2374 | |
| 2375 | cache_key = str(path.resolve(strict=False)) |
| 2376 | anchor = 0 |
| 2377 | |
| 2378 | # ONLY the sidecar cache. _RECORDS_CACHE is deliberately not consulted, |
| 2379 | # and the reason is the probable mechanism of the original revert: a |
| 2380 | # FULL parse stores covered_bytes = signature[1] -- the whole file size |
| 2381 | # -- even when that parse dropped a torn tail (the read fold "accounts |
| 2382 | # for" the torn bytes by knowing they are torn). So on precisely the |
| 2383 | # file that needs repair, a warm records-cache entry reports |
| 2384 | # covered == size and looks like proof of a clean boundary. Warm cache |
| 2385 | # -> skip -> regression; cold cache -> scan -> correct. Under pytest |
| 2386 | # the cache is warm; standalone it is cold. That is the whole |
| 2387 | # "fails in pytest, passes standalone". |
| 2388 | # |
| 2389 | # _OIDX_CACHE's covered comes from _scan_append_tail, which returns the |
| 2390 | # offset up to which parsing is AUTHORITATIVE -- torn tails excluded -- |
| 2391 | # or from handle.tell() after this process's own completed append. |
| 2392 | # Every value in it is a genuine committed row boundary. |
| 2393 | oidx = _OIDX_CACHE.get(cache_key) |
| 2394 | if oidx is not None: |
| 2395 | oidx_ino, oidx_covered, _offsets = oidx |
| 2396 | if oidx_ino == signature[2] and 0 < oidx_covered <= size: |
| 2397 | if oidx_covered == size: |
| 2398 | return # EOF sits exactly on a recorded boundary |
| 2399 | anchor = oidx_covered |
| 2400 | |
| 2401 | with path.open("rb+") as handle: |
| 2402 | handle.seek(anchor) |
| 2403 | data = handle.read(size - anchor) |
| 2404 | text = data.decode("utf-8", errors="surrogateescape") |
| 2405 | keep_chars = _committed_prefix_len(text) |
| 2406 | committed = anchor + len( |
| 2407 | text[:keep_chars].encode("utf-8", errors="surrogateescape") |
| 2408 | ) |
| 2409 | if committed < size: |
| 2410 | handle.truncate(committed) |
| 2411 | |
| 2412 | |
| 2413 | def _append_records_rows( |
| 2414 | path: Path, |
| 2415 | physical_fields: list[str], |
| 2416 | rows: list[dict[str, str]], |
| 2417 | ) -> list[tuple[int, int]]: |
| 2418 | """Append one or more pre-built physical rows to an append-format file. |
| 2419 | |
| 2420 | Each row in `rows` must already carry an `_op` key (OP_UPSERT or |
| 2421 | OP_DELETE). Self-heals a torn tail first (_repair_torn_tail) so the |
| 2422 | new row(s) always land as clean, complete physical lines, then writes |
| 2423 | via csv.writer (not DictWriter -- rows are already fully projected) |
| 2424 | on a text-mode "a" handle, flushing once at the end. Same dialect as |
| 2425 | _write_collection_records: tab-delimited, "\\n" line terminator, |
| 2426 | QUOTE_MINIMAL. |
| 2427 | |
| 2428 | Returns the (row_start_offset, row_end_offset) byte span written for |
| 2429 | each row, in order -- _persist_write hands these straight to the id |
| 2430 | sidecar (_update_oidx_after_append) so it never has to re-derive them |
| 2431 | with a separate stat or scan. `handle.tell()` on a text-mode stream is |
| 2432 | only reliably a byte offset when nothing has been read from it in |
| 2433 | this session (reading with a stateful decoder can return an opaque |
| 2434 | cookie instead); a pure sequential-write "a" session like this one |
| 2435 | never reads, so it's exact here -- verified empirically (UTF-8, |
| 2436 | non-ASCII content, mid-stream tell() calls all matched the file's |
| 2437 | actual byte length). |
| 2438 | """ |
| 2439 | _repair_torn_tail(path) |
| 2440 | offsets: list[tuple[int, int]] = [] |
| 2441 | with path.open("a", newline="") as handle: |
| 2442 | writer = csv.writer( |
| 2443 | handle, delimiter="\t", lineterminator="\n", |
| 2444 | quoting=(csv.QUOTE_ALL if _rows_need_full_quoting(rows) else csv.QUOTE_MINIMAL), |
| 2445 | ) |
| 2446 | if handle.tell() == 0: |
| 2447 | # The file has no committed content -- most likely the repair |
| 2448 | # above found a TORN HEADER (intact bytes, no terminating |
| 2449 | # newline: external truncation clipping exactly the trailing |
| 2450 | # byte) and correctly truncated to zero. Appending a bare data |
| 2451 | # row here would leave a headerless file every subsequent read |
| 2452 | # rejects with an uncaught ValueError, having REPORTED SUCCESS |
| 2453 | # for this write -- the torn-header finding in |
| 2454 | # tests/test_durability_torn_write_characterization.py. The |
| 2455 | # header is fully determined by physical_fields, so restore it |
| 2456 | # rather than manufacturing a poisoned file. |
| 2457 | csv.writer(handle, delimiter="\t", lineterminator="\n").writerow( |
| 2458 | physical_fields |
| 2459 | ) |
| 2460 | for row in rows: |
| 2461 | start = handle.tell() |
| 2462 | projected = _project_record(row, physical_fields) |
| 2463 | writer.writerow([projected[field] for field in physical_fields]) |
| 2464 | offsets.append((start, handle.tell())) |
| 2465 | handle.flush() |
| 2466 | return offsets |
| 2467 | |
| 2468 | |
| 2469 | # --------------------------------------------------------------------------- |
| 2470 | # id -> byte-offset sidecar (docs/append-only-storage-design.md item 4). |
| 2471 | # |
| 2472 | # SCOPE: every function below is reached only from append-physical code |
| 2473 | # paths -- _fast_record_lookup only looks past a _peek_records_cache miss |
| 2474 | # into the sidecar when `_physical_header(path)[0] == OP_FIELD`, and |
| 2475 | # _persist_write/_write_collection_records only call _update_oidx_after_ |
| 2476 | # append/_discard_oidx on the append-mode branches. A collection that has |
| 2477 | # never been append-mode (no schema ever set "storage": "append") never |
| 2478 | # executes a line past that header check: its _physical_header is a plain |
| 2479 | # ["id", ...] with no leading `_op`, so the guard is false every time. |
| 2480 | # --------------------------------------------------------------------------- |
| 2481 | |
| 2482 | |
| 2483 | def _oidx_path(records_path: Path) -> Path: |
| 2484 | return records_path.with_name(OIDX_FILE) |
| 2485 | |
| 2486 | |
| 2487 | def _store_oidx_cache( |
| 2488 | cache_key: str, |
| 2489 | data_ino: int, |
| 2490 | covered_bytes: int, |
| 2491 | id_offsets: dict[str, int], |
| 2492 | ) -> None: |
| 2493 | _OIDX_CACHE[cache_key] = (data_ino, covered_bytes, id_offsets) |
| 2494 | _OIDX_CACHE.move_to_end(cache_key) |
| 2495 | max_entries = _oidx_cache_max_entries() |
| 2496 | while len(_OIDX_CACHE) > max_entries: |
| 2497 | _OIDX_CACHE.popitem(last=False) |
| 2498 | |
| 2499 | |
| 2500 | def _records_header_length(path: Path) -> int: |
| 2501 | """Byte length of records.tsv's own physical header line (newline |
| 2502 | included), or 0 if the file is empty/missing. Reads only the first |
| 2503 | physical line -- cheap regardless of file size.""" |
| 2504 | try: |
| 2505 | with path.open("rb") as handle: |
| 2506 | return len(handle.readline()) |
| 2507 | except OSError: |
| 2508 | return 0 |
| 2509 | |
| 2510 | |
| 2511 | def _scan_append_rows_for_offsets( |
| 2512 | path: Path, |
| 2513 | physical_fields: list[str], |
| 2514 | start_byte: int, |
| 2515 | ) -> tuple[list[tuple[int, int, str, str]], int, int]: |
| 2516 | """Scan records.tsv from `start_byte` to EOF for complete physical |
| 2517 | rows, returning (entries, covered_bytes, row_count). |
| 2518 | |
| 2519 | `entries` is one (row_start, row_end, op, id) per row found with a |
| 2520 | non-empty id (blank-id rows -- reachable only via a hand-edited file, |
| 2521 | see _fold_append_rows -- can't be looked up by id, so are simply never |
| 2522 | indexed, though they DO count towards `row_count`, matching |
| 2523 | _parse_append_body's physical_row_count exactly); `covered_bytes` is |
| 2524 | the byte offset up to which scanning is authoritative (>= start_byte, |
| 2525 | equal to it when nothing new/complete was found). `start_byte` must |
| 2526 | land exactly on a row boundary -- every offset this module ever hands |
| 2527 | back here (a prior scan's covered_bytes, or the records-header |
| 2528 | length) always does, since it is always either 0 or a previous |
| 2529 | row_end. |
| 2530 | |
| 2531 | Torn-tail tolerant exactly like _parse_append_body: a trailing |
| 2532 | unterminated physical line is dropped first (_drop_torn_tail), and a |
| 2533 | row that still fails to parse (or reports more fields than the header |
| 2534 | -- a real corruption, not a torn write) stops consumption at that |
| 2535 | point rather than raising. This function must NEVER raise: it is the |
| 2536 | sidecar's own scan, and the sidecar is never allowed to be less |
| 2537 | tolerant than the reader it exists to speed up (any row it can't |
| 2538 | make sense of just isn't indexed yet -- the caller's `coherent` flag, |
| 2539 | derived from comparing covered_bytes to the file's actual size, |
| 2540 | reports that honestly rather than guessing). |
| 2541 | |
| 2542 | Delegates the actual scan to _scan_append_tail (shared with the |
| 2543 | records-cache's tail-delta incremental fold, _append_tail_delta_ |
| 2544 | records) and projects each full physical row down to just (op, id) |
| 2545 | plus its byte span -- all this function's callers need. |
| 2546 | """ |
| 2547 | scanned, covered_bytes = _scan_append_tail(path, physical_fields, start_byte) |
| 2548 | entries: list[tuple[int, int, str, str]] = [] |
| 2549 | row_count = 0 |
| 2550 | for row_start, row_end, record in scanned: |
| 2551 | row_count += 1 |
| 2552 | record_id = record.get("id", "") |
| 2553 | if not record_id: |
| 2554 | continue |
| 2555 | op = record.get(OP_FIELD, OP_UPSERT) |
| 2556 | entries.append((row_start, row_end, op, record_id)) |
| 2557 | |
| 2558 | return entries, covered_bytes, row_count |
| 2559 | |
| 2560 | |
| 2561 | def _fold_oidx_entries( |
| 2562 | entries: list[tuple[int, int, str, str]], |
| 2563 | *, |
| 2564 | base: dict[str, int] | None = None, |
| 2565 | ) -> dict[str, int]: |
| 2566 | """Fold sidecar entries last-wins into {id: row_start_offset}, |
| 2567 | dropping deleted ids -- the same rule _fold_append_rows applies to |
| 2568 | full rows, applied here to bare offsets. |
| 2569 | |
| 2570 | Mutates and returns `base` directly when given, rather than copying |
| 2571 | it first: every call site's `base` is either freshly built moments |
| 2572 | earlier with nothing else referencing it (_load_oidx_body's return), |
| 2573 | or is _OIDX_CACHE's own entry being brought up to date in a catch-up |
| 2574 | that is about to replace that same cache slot anyway (see the |
| 2575 | matching note on _update_oidx_after_append for why this dict is safe |
| 2576 | to mutate in place: it's never exposed outside this module). A copy |
| 2577 | here would be a one-time-per-catch-up cost rather than a per-write |
| 2578 | one, so it's lower-stakes than the bug that note describes -- but |
| 2579 | there's no reason to pay even that when nothing needs the old |
| 2580 | version preserved. |
| 2581 | """ |
| 2582 | result = base if base is not None else {} |
| 2583 | for row_start, _row_end, op, record_id in entries: |
| 2584 | if op == OP_DELETE: |
| 2585 | result.pop(record_id, None) |
| 2586 | else: |
| 2587 | result[record_id] = row_start |
| 2588 | return result |
| 2589 | |
| 2590 | |
| 2591 | def _write_oidx_file( |
| 2592 | oidx_path: Path, data_ino: int, entries: list[tuple[int, int, str, str]] |
| 2593 | ) -> None: |
| 2594 | """Write a fresh sidecar from scratch (REBUILD), atomically (temp file |
| 2595 | + replace) so a concurrent reader never observes a half-written file |
| 2596 | mid-rebuild -- unlike the steady-state append path |
| 2597 | (_append_oidx_lines), which is cheap specifically because it skips |
| 2598 | this.""" |
| 2599 | oidx_path.parent.mkdir(parents=True, exist_ok=True) |
| 2600 | temp_path = oidx_path.with_name(f".{oidx_path.name}.{os.getpid()}.{threading.get_ident()}.tmp") |
| 2601 | try: |
| 2602 | with temp_path.open("w", newline="") as handle: |
| 2603 | handle.write(f"{OIDX_HEADER_TAG}\t{data_ino}\n") |
| 2604 | handle.writelines( |
| 2605 | f"{row_start}\t{row_end}\t{op}\t{record_id}\n" |
| 2606 | for row_start, row_end, op, record_id in entries |
| 2607 | ) |
| 2608 | temp_path.replace(oidx_path) |
| 2609 | finally: |
| 2610 | if temp_path.exists(): |
| 2611 | temp_path.unlink() |
| 2612 | |
| 2613 | |
| 2614 | def _append_oidx_lines( |
| 2615 | oidx_path: Path, data_ino: int, entries: list[tuple[int, int, str, str]] |
| 2616 | ) -> None: |
| 2617 | """Append idx lines to an existing sidecar whose header ino the |
| 2618 | caller has already confirmed matches `data_ino`. Self-heals a torn |
| 2619 | idx tail first (same helper the data file uses -- the format needs no |
| 2620 | CSV awareness to repair: an idx line can never contain a literal tab |
| 2621 | or newline inside a field, since ids are restricted to _RECORD_ID_RE |
| 2622 | and offsets/op are plain digits/literals, so "does the file end with |
| 2623 | \\n" is exactly as reliable a completeness signal here as it is for |
| 2624 | records.tsv).""" |
| 2625 | if not entries: |
| 2626 | return |
| 2627 | _repair_torn_tail_lines(oidx_path) |
| 2628 | with oidx_path.open("a", newline="") as handle: |
| 2629 | handle.writelines( |
| 2630 | f"{row_start}\t{row_end}\t{op}\t{record_id}\n" |
| 2631 | for row_start, row_end, op, record_id in entries |
| 2632 | ) |
| 2633 | handle.flush() |
| 2634 | |
| 2635 | |
| 2636 | def _read_oidx_header(oidx_path: Path) -> int | None: |
| 2637 | """Return the data_ino recorded in the sidecar's header line, or None |
| 2638 | when the file is missing, empty, or its header doesn't parse (all |
| 2639 | treated identically by the caller: rebuild).""" |
| 2640 | try: |
| 2641 | with oidx_path.open("r", newline="") as handle: |
| 2642 | first_line = handle.readline() |
| 2643 | except (OSError, UnicodeDecodeError, ValueError): |
| 2644 | # A sidecar corrupted with non-UTF-8 bytes must read as "header |
| 2645 | # does not parse" -> rebuild, per _load_oidx's own contract |
| 2646 | # ("100% best-effort and NEVER raises"). Catching only OSError |
| 2647 | # here let a garbage sidecar crash every read of the collection |
| 2648 | # instead of self-healing -- the atomicity characterization's |
| 2649 | # sidecar finding. |
| 2650 | return None |
| 2651 | parts = first_line.rstrip("\n").split("\t") |
| 2652 | if len(parts) != 2 or parts[0] != OIDX_HEADER_TAG: |
| 2653 | return None |
| 2654 | try: |
| 2655 | return int(parts[1]) |
| 2656 | except ValueError: |
| 2657 | return None |
| 2658 | |
| 2659 | |
| 2660 | def _load_oidx_body( |
| 2661 | oidx_path: Path, |
| 2662 | ) -> tuple[dict[str, int], int | None, int] | None: |
| 2663 | """Parse a sidecar's data lines (everything after the header) into |
| 2664 | ({id: offset}, covered_bytes, indexed_row_count). Returns None when the |
| 2665 | file can't be trusted at all (missing, or a data line doesn't have |
| 2666 | exactly the 4 expected columns / non-integer offsets -- genuine |
| 2667 | corruption, not a torn write, so this doesn't try to salvage a prefix: |
| 2668 | the caller rebuilds instead of guessing). |
| 2669 | |
| 2670 | covered_bytes is None when there are no data lines yet (a header-only |
| 2671 | sidecar, e.g. right after this collection's very first indexed |
| 2672 | write): the caller supplies the real baseline in that case |
| 2673 | (_records_header_length), since a brand-new sidecar covers exactly |
| 2674 | the data file's own header bytes, not zero. |
| 2675 | |
| 2676 | indexed_row_count is the number of body lines parsed -- one per |
| 2677 | PHYSICAL row of records.tsv this sidecar has indexed (create, update, |
| 2678 | and delete rows all count; only the blank-id rows |
| 2679 | _scan_append_rows_for_offsets never indexes are excluded, per that |
| 2680 | function's docstring). Exists for object_records.append_collection_ |
| 2681 | stats' cheap physical-row-count path (a "pure line-count read of the |
| 2682 | sidecar", avoiding a full fold of records.tsv) -- no other caller |
| 2683 | needs it, so it costs this function nothing beyond incrementing a |
| 2684 | counter already being iterated for the fold below. |
| 2685 | |
| 2686 | A torn last line (the file doesn't end with "\\n") is dropped, same |
| 2687 | rule as everywhere else: an idx line, like a data row, is only |
| 2688 | trustworthy once its own newline lands. |
| 2689 | """ |
| 2690 | try: |
| 2691 | text = oidx_path.read_text() |
| 2692 | except OSError: |
| 2693 | return None |
| 2694 | lines = text.split("\n") |
| 2695 | if not lines or not lines[0].startswith(f"{OIDX_HEADER_TAG}\t"): |
| 2696 | return None |
| 2697 | |
| 2698 | body_lines = lines[1:] |
| 2699 | if body_lines and body_lines[-1] == "": |
| 2700 | body_lines = body_lines[:-1] # file ends with \n -- drop the trailing empty split |
| 2701 | elif body_lines: |
| 2702 | body_lines = body_lines[:-1] # torn last line -- drop it, unindexed |
| 2703 | |
| 2704 | result: dict[str, int] = {} |
| 2705 | covered_bytes: int | None = None |
| 2706 | indexed_row_count = 0 |
| 2707 | for line in body_lines: |
| 2708 | if not line: |
| 2709 | continue |
| 2710 | parts = line.split("\t") |
| 2711 | if len(parts) != 4: |
| 2712 | return None |
| 2713 | try: |
| 2714 | row_start = int(parts[0]) |
| 2715 | row_end = int(parts[1]) |
| 2716 | except ValueError: |
| 2717 | return None |
| 2718 | op, record_id = parts[2], parts[3] |
| 2719 | if op == OP_DELETE: |
| 2720 | result.pop(record_id, None) |
| 2721 | else: |
| 2722 | result[record_id] = row_start |
| 2723 | covered_bytes = row_end |
| 2724 | indexed_row_count += 1 |
| 2725 | |
| 2726 | return result, covered_bytes, indexed_row_count |
| 2727 | |
| 2728 | |
| 2729 | def _oidx_get_row( |
| 2730 | path: Path, physical_fields: list[str], offset: int |
| 2731 | ) -> dict[str, str] | None: |
| 2732 | """Seek to `offset` in records.tsv and parse exactly the one physical |
| 2733 | row starting there, returning the LOGICAL record (id + fields, `_op` |
| 2734 | stripped) -- or None if the row can't be read/parsed cleanly, which |
| 2735 | the caller treats as "sidecar inconclusive, fall back to a full |
| 2736 | fold" rather than as "record missing".""" |
| 2737 | # Read the full LOGICAL row via csv, not a raw readline(). A row whose |
| 2738 | # value contains an embedded newline (a quoted field spanning several |
| 2739 | # physical lines -- e.g. multi-line text, or pretty-printed JSON) must be |
| 2740 | # consumed to its real row terminator; a readline() would stop at the |
| 2741 | # first '\n' byte and silently return a truncated (often unparseable) |
| 2742 | # value. csv.reader over the file from `offset` handles the quoted-field |
| 2743 | # continuation exactly as the full-fold path (_parse_append_body) does, |
| 2744 | # keeping the by-id read consistent with it. Any read/parse trouble -> |
| 2745 | # None, so the caller falls back to a full fold rather than trusting a |
| 2746 | # partial row. |
| 2747 | try: |
| 2748 | handle = path.open("rb") |
| 2749 | except OSError: |
| 2750 | return None |
| 2751 | try: |
| 2752 | handle.seek(offset) |
| 2753 | text = io.TextIOWrapper(handle, encoding="utf-8", newline="") |
| 2754 | row = next(csv.reader(text, delimiter="\t")) |
| 2755 | except (OSError, UnicodeDecodeError, StopIteration, csv.Error): |
| 2756 | return None |
| 2757 | finally: |
| 2758 | handle.close() |
| 2759 | field_count = len(physical_fields) |
| 2760 | if not row or len(row) > field_count: |
| 2761 | return None |
| 2762 | record = { |
| 2763 | physical_fields[i]: (row[i] if i < len(row) else "") |
| 2764 | for i in range(field_count) |
| 2765 | } |
| 2766 | return {key: value for key, value in record.items() if key != OP_FIELD} |
| 2767 | |
| 2768 | |
| 2769 | def _load_oidx(path: Path) -> tuple[dict[str, int], bool]: |
| 2770 | """Return ({id: row_start_offset}, coherent) for `path`'s CURRENT |
| 2771 | append-mode content, using and maintaining the on-disk sidecar plus |
| 2772 | the small in-memory _OIDX_CACHE LRU. |
| 2773 | |
| 2774 | `coherent` is True exactly when the returned dict accounts for every |
| 2775 | byte of the file as observed by this call's own stat -- i.e. it is |
| 2776 | safe to treat "id not in the dict" as proof the id doesn't currently |
| 2777 | exist. False means only a best-effort snapshot could be produced |
| 2778 | (normally: a torn tail at EOF from a write that crashed before |
| 2779 | self-healing) and callers must not treat an absence -- or, to be |
| 2780 | conservative, even a presence -- as authoritative; see |
| 2781 | _fast_record_lookup, the only caller, which falls all the way back to |
| 2782 | a full fold whenever this is False. |
| 2783 | |
| 2784 | 100% best-effort and NEVER raises: any inconsistency (missing sidecar, |
| 2785 | corrupt sidecar, inode mismatch, a data file that shrank under a |
| 2786 | stable inode -- e.g. a torn-tail repair truncation) triggers a |
| 2787 | rebuild (one sequential scan of records.tsv) rather than propagating |
| 2788 | an error, per docs/append-only-storage-design.md: the sidecar is |
| 2789 | "never required for correctness." |
| 2790 | """ |
| 2791 | try: |
| 2792 | st = path.stat() |
| 2793 | except OSError: |
| 2794 | return {}, False |
| 2795 | data_ino = st.st_ino |
| 2796 | data_size = st.st_size |
| 2797 | cache_key = str(path.resolve(strict=False)) |
| 2798 | |
| 2799 | cached = _OIDX_CACHE.get(cache_key) |
| 2800 | if cached is not None: |
| 2801 | cached_ino, cached_covered, cached_dict = cached |
| 2802 | if cached_ino == data_ino and cached_covered <= data_size: |
| 2803 | if cached_covered == data_size: |
| 2804 | _OIDX_CACHE.move_to_end(cache_key) |
| 2805 | return cached_dict, True |
| 2806 | physical_fields = _physical_header(path) |
| 2807 | if physical_fields and physical_fields[0] == OP_FIELD: |
| 2808 | entries, new_covered, _row_count = _scan_append_rows_for_offsets( |
| 2809 | path, physical_fields, cached_covered |
| 2810 | ) |
| 2811 | new_dict = _fold_oidx_entries(entries, base=cached_dict) |
| 2812 | _append_oidx_lines(_oidx_path(path), data_ino, entries) |
| 2813 | _store_oidx_cache(cache_key, data_ino, new_covered, new_dict) |
| 2814 | return new_dict, new_covered == data_size |
| 2815 | # ino mismatch, or the file shrank under a stable inode: this |
| 2816 | # in-memory snapshot can't be trusted as a base -- drop it and |
| 2817 | # fall through to the on-disk / rebuild path below. |
| 2818 | _OIDX_CACHE.pop(cache_key, None) |
| 2819 | |
| 2820 | physical_fields = _physical_header(path) |
| 2821 | if not physical_fields or physical_fields[0] != OP_FIELD: |
| 2822 | return {}, False |
| 2823 | |
| 2824 | oidx_path = _oidx_path(path) |
| 2825 | header_ino = _read_oidx_header(oidx_path) |
| 2826 | if header_ino == data_ino: |
| 2827 | loaded = _load_oidx_body(oidx_path) |
| 2828 | if loaded is not None: |
| 2829 | body_dict, body_covered, _body_row_count = loaded |
| 2830 | covered = body_covered if body_covered is not None else _records_header_length(path) |
| 2831 | if covered <= data_size: |
| 2832 | if covered < data_size: |
| 2833 | entries, covered, _row_count = _scan_append_rows_for_offsets( |
| 2834 | path, physical_fields, covered |
| 2835 | ) |
| 2836 | body_dict = _fold_oidx_entries(entries, base=body_dict) |
| 2837 | _append_oidx_lines(oidx_path, data_ino, entries) |
| 2838 | _store_oidx_cache(cache_key, data_ino, covered, body_dict) |
| 2839 | return body_dict, covered == data_size |
| 2840 | # covered > data_size: the data file shrank under a stable |
| 2841 | # ino -- fall through to a full rebuild rather than trust a |
| 2842 | # sidecar that claims to know about bytes no longer there. |
| 2843 | |
| 2844 | # REBUILD: a full sequential scan of records.tsv from its own header, |
| 2845 | # functionally equivalent to the classic full-fold parse this branch |
| 2846 | # replaces for an over-threshold collection -- so, like that parse |
| 2847 | # (_parse_records_file -> _maybe_flag_auto_compact), it's also the |
| 2848 | # right place to (re-)evaluate auto-compaction. Incremental catch-up |
| 2849 | # above deliberately does NOT re-flag: this is a documented, narrower |
| 2850 | # trigger window than the pre-sidecar behavior (every full fold used |
| 2851 | # to re-check on every cold read) -- a collection gets an accurate |
| 2852 | # check whenever its sidecar needs rebuilding (first use past the |
| 2853 | # cache threshold, or any time compaction/a mode switch/corruption |
| 2854 | # invalidates it) but not on every subsequent catch-up scan. Flagging |
| 2855 | # on every catch-up too would need a persisted running physical-row |
| 2856 | # count in the sidecar itself; not worth the format complexity for a |
| 2857 | # threshold heuristic that self-corrects the next time a rebuild does |
| 2858 | # happen. |
| 2859 | header_bytes = _records_header_length(path) |
| 2860 | entries, covered, row_count = _scan_append_rows_for_offsets(path, physical_fields, header_bytes) |
| 2861 | fresh_dict = _fold_oidx_entries(entries) |
| 2862 | _write_oidx_file(oidx_path, data_ino, entries) |
| 2863 | _store_oidx_cache(cache_key, data_ino, covered, fresh_dict) |
| 2864 | if covered == data_size: |
| 2865 | _maybe_flag_auto_compact(path, physical_row_count=row_count, live_row_count=len(fresh_dict)) |
| 2866 | return fresh_dict, covered == data_size |
| 2867 | |
| 2868 | |
| 2869 | def _discard_oidx(path: Path) -> None: |
| 2870 | """Drop any sidecar tracking (in-memory and on-disk) for `path` -- |
| 2871 | called after a full rewrite of an append-physical file (compaction, a |
| 2872 | mode-transition rewrite in either direction, or the new-field |
| 2873 | fallback rewrite): the rewrite gives the file a new inode via |
| 2874 | temp+rename, which would make any existing sidecar's header ino stale |
| 2875 | even without this (the next _load_oidx would detect the mismatch and |
| 2876 | rebuild on its own) -- this just does it eagerly so a compacted or |
| 2877 | mode-switched collection's directory doesn't carry a permanently |
| 2878 | orphaned `.records.oidx` around |
| 2879 | (docs/append-only-storage-design.md Sidecars: "delete or rebuild |
| 2880 | sidecar after the rewrite"). Never called for a collection with no |
| 2881 | append-mode history -- see the SCOPE note above this section -- so a |
| 2882 | purely classic collection's write path never reaches here. Best- |
| 2883 | effort: an unlink failure is not a correctness problem, so this never |
| 2884 | raises. |
| 2885 | """ |
| 2886 | cache_key = str(path.resolve(strict=False)) |
| 2887 | _OIDX_CACHE.pop(cache_key, None) |
| 2888 | try: |
| 2889 | _oidx_path(path).unlink() |
| 2890 | except OSError: |
| 2891 | pass |
| 2892 | |
| 2893 | |
| 2894 | def _update_oidx_after_append( |
| 2895 | path: Path, |
| 2896 | rows: list[dict[str, str]], |
| 2897 | offsets: list[tuple[int, int]], |
| 2898 | ) -> None: |
| 2899 | """Keep the sidecar in sync with a fast append, in O(1): appends |
| 2900 | matching idx line(s) to the on-disk sidecar and updates the in-memory |
| 2901 | _OIDX_CACHE entry -- but ONLY when that entry is already warm for |
| 2902 | this path. When it isn't, this does nothing at all: a write must |
| 2903 | never pay to build or maintain a sidecar nobody is currently reading |
| 2904 | through (see _load_oidx, which lazily builds/catches-up on the next |
| 2905 | point op that actually needs it -- "Sidecar builds lazily on first |
| 2906 | over-threshold op"). |
| 2907 | |
| 2908 | `offsets` are exactly what _append_records_rows just returned for |
| 2909 | `rows` (same order, same length) -- no re-derivation (stat or scan) |
| 2910 | needed to keep this O(1) regardless of collection size. |
| 2911 | |
| 2912 | Mutates `cached_dict` IN PLACE rather than copying it first, UNLIKE |
| 2913 | _refresh_records_cache_after_append's equivalent step for |
| 2914 | _RECORDS_CACHE (which always copies, even on its "fast" path -- see |
| 2915 | that function's docstring). That copy is safe there because |
| 2916 | _RECORDS_CACHE's records/id_index are handed across the public API |
| 2917 | (get/list return values, directly or via one more copy) and so must |
| 2918 | never be mutated out from under an earlier caller -- and it's AFFORD- |
| 2919 | ABLE there because _RECORDS_CACHE only ever holds collections at or |
| 2920 | under DBBASIC_RECORDS_CACHE_MAX_ROWS. Neither holds for this dict: it |
| 2921 | is never exposed outside this module (every public entry point that |
| 2922 | might have sourced data from it -- get/create/update/delete via |
| 2923 | _fast_record_lookup -- only ever calls dict.get() on it and returns a |
| 2924 | freshly-parsed ROW, never the dict itself), and it exists specifically |
| 2925 | FOR collections too large for a copy to be the cheap operation it is |
| 2926 | for _RECORDS_CACHE (a 1M-entry dict copy alone was measured to add |
| 2927 | ~35-40ms to every single write here -- see the sidecar benchmark |
| 2928 | report -- reintroducing real per-write cost in exactly the size |
| 2929 | regime this feature exists to make O(1) again). A concurrent |
| 2930 | lock-free reader's dict.get() racing this mutation is benign under |
| 2931 | the GIL (each op is atomic; a race can only return a value from |
| 2932 | just-before or just-after the write, never corrupt memory or raise) |
| 2933 | -- the same class of eventual-consistency already documented and |
| 2934 | accepted for _RECORDS_CACHE's own CONCURRENCY note above. |
| 2935 | """ |
| 2936 | cache_key = str(path.resolve(strict=False)) |
| 2937 | cached = _OIDX_CACHE.get(cache_key) |
| 2938 | if cached is None: |
| 2939 | return |
| 2940 | cached_ino, cached_covered, cached_dict = cached |
| 2941 | if cached_covered != offsets[0][0]: |
| 2942 | # The cached view doesn't start exactly where this append starts |
| 2943 | # (should not happen -- writes are serialized under the caller's |
| 2944 | # file lock and this cache is only ever advanced by this same |
| 2945 | # process's own writes/reads) -- rather than risk recording wrong |
| 2946 | # offsets, drop it; the next _load_oidx call resyncs or rebuilds. |
| 2947 | _OIDX_CACHE.pop(cache_key, None) |
| 2948 | return |
| 2949 | |
| 2950 | entries: list[tuple[int, int, str, str]] = [] |
| 2951 | for row, (start, end) in zip(rows, offsets, strict=True): |
| 2952 | op = row.get(OP_FIELD, OP_UPSERT) |
| 2953 | record_id = row.get("id", "") |
| 2954 | entries.append((start, end, op, record_id)) |
| 2955 | if op == OP_DELETE: |
| 2956 | cached_dict.pop(record_id, None) |
| 2957 | else: |
| 2958 | cached_dict[record_id] = start |
| 2959 | |
| 2960 | _append_oidx_lines(_oidx_path(path), cached_ino, entries) |
| 2961 | _store_oidx_cache(cache_key, cached_ino, offsets[-1][1], cached_dict) |
| 2962 | |
| 2963 | |
| 2964 | def _fast_record_lookup( |
| 2965 | collection: str, |
| 2966 | path: Path, |
| 2967 | record_id: str, |
| 2968 | *, |
| 2969 | need_row: bool, |
| 2970 | ) -> tuple[ |
| 2971 | list[str], |
| 2972 | bool, |
| 2973 | dict[str, str] | None, |
| 2974 | list[dict[str, str]] | None, |
| 2975 | dict[str, int] | None, |
| 2976 | ]: |
| 2977 | """Resolve (fields, exists, existing_row, records_ref, id_index_ref) |
| 2978 | for one record id -- the shared point-op resolver behind get/create/ |
| 2979 | update/delete, in increasing order of cost: |
| 2980 | |
| 2981 | 1. A warm _RECORDS_CACHE hit (_peek_records_cache) -- the ordinary |
| 2982 | fast path for any collection at or under |
| 2983 | DBBASIC_RECORDS_CACHE_MAX_ROWS once touched once. records_ref/ |
| 2984 | id_index_ref are the module cache's own objects (never mutate -- |
| 2985 | see the _RECORDS_CACHE ALIASING SAFETY note). |
| 2986 | 2. On a miss, if the file is append-physical: the id->offset |
| 2987 | sidecar (_load_oidx). Answers in O(1) without ever folding the |
| 2988 | file -- this is the whole point of the sidecar (docs/append- |
| 2989 | only-storage-design.md item 4): a collection too large for |
| 2990 | _RECORDS_CACHE no longer pays an O(n) fold on every write. |
| 2991 | Trusted only when _load_oidx reports `coherent` -- otherwise |
| 2992 | (a torn tail at EOF from an unfinished write) this falls all the |
| 2993 | way through to step 3 rather than risk a wrong answer. |
| 2994 | records_ref/id_index_ref are None in this case: the fast-append |
| 2995 | write path this enables never needs the full record list (see |
| 2996 | _persist_write's folded_records_fn, which forces its own full |
| 2997 | fold via _cache_entry lazily, only if a rare full-rewrite path |
| 2998 | is chosen instead of a fast append). |
| 2999 | 3. _cache_entry -- the original full fold, unchanged, used for |
| 3000 | classic-mode files, a genuinely cold/small collection's first |
| 3001 | touch, and any sidecar-inconclusive case above. |
| 3002 | |
| 3003 | fields is always the LOGICAL field list (no `_op`). `existing_row`, |
| 3004 | when requested via need_row and the id exists, is always a fresh, |
| 3005 | independent copy safe to hand across the public API or mutate. |
| 3006 | """ |
| 3007 | cached = _peek_records_cache(path) |
| 3008 | if cached is not None: |
| 3009 | fields, records_ref, id_index_ref = cached |
| 3010 | index = id_index_ref.get(record_id) |
| 3011 | existing_row = dict(records_ref[index]) if (need_row and index is not None) else None |
| 3012 | return fields, index is not None, existing_row, records_ref, id_index_ref |
| 3013 | |
| 3014 | physical_header = _physical_header(path) |
| 3015 | if physical_header and physical_header[0] == OP_FIELD: |
| 3016 | id_offsets, coherent = _load_oidx(path) |
| 3017 | if coherent: |
| 3018 | offset = id_offsets.get(record_id) |
| 3019 | if offset is None: |
| 3020 | return physical_header[1:], False, None, None, None |
| 3021 | if not need_row: |
| 3022 | return physical_header[1:], True, None, None, None |
| 3023 | row = _oidx_get_row(path, physical_header, offset) |
| 3024 | if row is not None: |
| 3025 | return physical_header[1:], True, row, None, None |
| 3026 | # Row unreadable at its recorded offset despite a coherent |
| 3027 | # sidecar -- shouldn't happen; be conservative and fall |
| 3028 | # through to the authoritative full fold below. |
| 3029 | |
| 3030 | fields, records_ref, id_index_ref = _cache_entry(collection, path) |
| 3031 | index = id_index_ref.get(record_id) |
| 3032 | existing_row = dict(records_ref[index]) if (need_row and index is not None) else None |
| 3033 | return fields, index is not None, existing_row, records_ref, id_index_ref |
| 3034 | |
| 3035 | |
| 3036 | def _collection_storage_mode( |
| 3037 | collection: str, |
| 3038 | *, |
| 3039 | base_dir: Path | str, |
| 3040 | roots: Iterable[Path] | None, |
| 3041 | ) -> str: |
| 3042 | """Return a collection's storage mode from its schema's `storage` key. |
| 3043 | |
| 3044 | Defaults to classic when the schema has no `storage` key, is derived |
| 3045 | (no manual schema file), or doesn't exist yet -- see |
| 3046 | object_schemas.normalize_schema, which validates the key at schema- |
| 3047 | write time, so any value reaching here is already known-good, but this |
| 3048 | stays defensive rather than trusting that unconditionally. |
| 3049 | """ |
| 3050 | try: |
| 3051 | schema = object_schemas.get_schema(collection, base_dir=base_dir, roots=roots) |
| 3052 | except object_schemas.SchemaNotFoundError: |
| 3053 | return object_schemas.STORAGE_CLASSIC |
| 3054 | mode = schema.get("storage", object_schemas.STORAGE_CLASSIC) |
| 3055 | return mode if mode in object_schemas.VALID_STORAGE_MODES else object_schemas.STORAGE_CLASSIC |
| 3056 | |
| 3057 | |
| 3058 | def _persist_write( |
| 3059 | collection: str, |
| 3060 | path: Path, |
| 3061 | *, |
| 3062 | base_dir: Path | str, |
| 3063 | roots: Iterable[Path] | None, |
| 3064 | prior_fields: list[str], |
| 3065 | merged_fields: list[str], |
| 3066 | folded_records_fn, |
| 3067 | delta_row: dict[str, str] | None, |
| 3068 | delta_op: str, |
| 3069 | delta_id: str, |
| 3070 | ) -> None: |
| 3071 | """Persist one create/update/delete, choosing the write strategy. |
| 3072 | |
| 3073 | `folded_records_fn` is a zero-argument callable that builds the FULL |
| 3074 | folded record set after this op (a fresh, safe-to-mutate copy -- see |
| 3075 | the callers in create/update/delete_collection_record). It is called |
| 3076 | LAZILY, only on a full-rewrite path, and never on the fast-append |
| 3077 | path: the whole point of a fast append is O(1) work, and building |
| 3078 | that full copy is an O(n) cost callers can otherwise skip entirely |
| 3079 | (see _refresh_records_cache_after_append, which updates the cache |
| 3080 | from its own prior entry plus this one delta instead). `delta_row`/ |
| 3081 | `delta_op`/`delta_id` describe just this one op's row, used only by |
| 3082 | the fast-append path. Chooses among (see |
| 3083 | docs/append-only-storage-design.md and the Decisions this module's |
| 3084 | implementation is bound by): |
| 3085 | |
| 3086 | - FAST APPEND: storage is "append", the file is already physically |
| 3087 | in append format, this write doesn't introduce a field the |
| 3088 | current header lacks, and no auto-compaction is pending for this |
| 3089 | collection -- append one row, then update whichever of |
| 3090 | _RECORDS_CACHE / the id->offset sidecar (docs/append-only- |
| 3091 | storage-design.md item 4) is already warm for this path, in O(1), |
| 3092 | via _refresh_records_cache_after_append / _update_oidx_after_ |
| 3093 | append. Neither is FORCED warm here: a collection too large for |
| 3094 | _RECORDS_CACHE (the case the sidecar exists for) is left |
| 3095 | uncached by design, and a sidecar nobody has read through yet is |
| 3096 | left unbuilt (see _update_oidx_after_append) -- either would mean |
| 3097 | paying the exact O(n) fold-per-write cost this branch exists to |
| 3098 | avoid, just moved from "on every write" to "on every write that |
| 3099 | happens to miss the cache," which is most of them for a |
| 3100 | write-hot, over-threshold collection. |
| 3101 | - TRANSITION-IN: storage is "append" but the file isn't (yet) in |
| 3102 | append format (including a brand new collection) -- one full |
| 3103 | rewrite that adds the `_op` header column, applying this op in |
| 3104 | the same pass. |
| 3105 | - NEW-FIELD FALLBACK / AUTO-COMPACT: storage is "append", the file |
| 3106 | is already append-format, but this write needs a header column |
| 3107 | the file doesn't have yet, or a prior read flagged this |
| 3108 | collection for compaction -- a full rewrite that folds current |
| 3109 | content, applies this op, and (in the new-field case) extends the |
| 3110 | header. Still `_op`-columned. |
| 3111 | - COMPACT-TO-CLASSIC: storage is "classic" but the file is still |
| 3112 | physically in append format (opt-out just happened) -- one full |
| 3113 | rewrite that folds current content, applies this op, and drops |
| 3114 | the `_op` column. |
| 3115 | - CLASSIC: storage is "classic" and the file already is (or never |
| 3116 | was anything but) classic format -- the original full-rewrite |
| 3117 | behavior, completely unchanged. |
| 3118 | """ |
| 3119 | desired_mode = _collection_storage_mode(collection, base_dir=base_dir, roots=roots) |
| 3120 | physical_header = _physical_header(path) |
| 3121 | file_is_append_physical = bool(physical_header) and physical_header[0] == OP_FIELD |
| 3122 | cache_key = str(path.resolve(strict=False)) |
| 3123 | new_field_introduced = merged_fields != prior_fields |
| 3124 | pending_compaction = cache_key in _PENDING_COMPACTION |
| 3125 | |
| 3126 | if ( |
| 3127 | desired_mode == object_schemas.STORAGE_APPEND |
| 3128 | and file_is_append_physical |
| 3129 | and not new_field_introduced |
| 3130 | and not pending_compaction |
| 3131 | ): |
| 3132 | physical_fields = [OP_FIELD, *merged_fields] |
| 3133 | row = dict(delta_row) if delta_row is not None else {} |
| 3134 | row["id"] = delta_id |
| 3135 | row[OP_FIELD] = delta_op |
| 3136 | offsets = _append_records_rows(path, physical_fields, [row]) |
| 3137 | projected_row = _project_record(row, merged_fields) |
| 3138 | _refresh_records_cache_after_append( |
| 3139 | collection, path, merged_fields, delta_op, delta_id, projected_row |
| 3140 | ) |
| 3141 | _update_oidx_after_append(path, [row], offsets) |
| 3142 | return |
| 3143 | |
| 3144 | if file_is_append_physical: |
| 3145 | _discard_oidx(path) |
| 3146 | # folded_records_fn() is called BEFORE the discard below (not after, |
| 3147 | # as it may look more natural) because, on the sidecar-sourced lookup |
| 3148 | # path, it can itself force a fresh full fold (_cache_entry -> |
| 3149 | # _parse_records_file), which re-evaluates and can re-set auto-compact |
| 3150 | # flagging for this same cache_key from the PRE-write content. This |
| 3151 | # write is about to make that stale regardless of what it flagged |
| 3152 | # (append: compacting; classic/transition: rewriting from scratch |
| 3153 | # either way), so the discard must be the last word. |
| 3154 | folded_records = folded_records_fn() |
| 3155 | _PENDING_COMPACTION.discard(cache_key) |
| 3156 | if desired_mode == object_schemas.STORAGE_APPEND: |
| 3157 | physical_fields = [OP_FIELD, *merged_fields] |
| 3158 | _write_collection_records( |
| 3159 | collection, path, physical_fields, folded_records, cache_fields=merged_fields |
| 3160 | ) |
| 3161 | else: |
| 3162 | _write_collection_records(collection, path, merged_fields, folded_records) |
| 3163 | |
| 3164 | |
| 3165 | @contextmanager |
| 3166 | def _records_file_lock(records_file: Path): |
| 3167 | """Use a best-effort advisory lock for collection record mutations.""" |
| 3168 | lock_path = records_file.with_name(f".{records_file.name}.lock") |
| 3169 | lock_path.parent.mkdir(parents=True, exist_ok=True) |
| 3170 | |
| 3171 | with lock_path.open("a") as lock_file: |
| 3172 | try: |
| 3173 | import fcntl |
| 3174 | except ImportError: |
| 3175 | fcntl = None |
| 3176 | |
| 3177 | if fcntl is not None: |
| 3178 | fcntl.flock(lock_file, fcntl.LOCK_EX) |
| 3179 | |
| 3180 | try: |
| 3181 | yield |
| 3182 | finally: |
| 3183 | if fcntl is not None: |
| 3184 | fcntl.flock(lock_file, fcntl.LOCK_UN) |
| 3185 | |
| 3186 | |
| 3187 | def _validate_header(collection: str, fields: list[str] | None) -> None: |
| 3188 | if not fields: |
| 3189 | raise ValueError(f"Collection records file is missing a header: {collection}") |
| 3190 | |
| 3191 | clean = [field.strip() for field in fields] |
| 3192 | if any(not field for field in clean): |
| 3193 | raise ValueError(f"Collection records file has an empty field name: {collection}") |
| 3194 | if clean != fields: |
| 3195 | raise ValueError(f"Collection records file has whitespace in field names: {collection}") |
| 3196 | if len(set(clean)) != len(clean): |
| 3197 | raise ValueError(f"Collection records file has duplicate field names: {collection}") |
| 3198 | if "id" not in clean: |
| 3199 | raise ValueError(f"Collection records file must include an id column: {collection}") |
| 3200 | |
| 3201 | |
| 3202 | def _normalize_record_payload(payload: dict[str, Any], *, require_id: bool) -> dict[str, str]: |
| 3203 | if not isinstance(payload, dict): |
| 3204 | raise InvalidRecordPayloadError("Record payload must be an object") |
| 3205 | generate_id = require_id and ( |
| 3206 | "id" not in payload |
| 3207 | or payload.get("id") is None |
| 3208 | or payload.get("id") == "" |
| 3209 | ) |
| 3210 | |
| 3211 | clean: dict[str, str] = {} |
| 3212 | for key, value in payload.items(): |
| 3213 | if not isinstance(key, str) or not key: |
| 3214 | raise InvalidRecordPayloadError("Record field names must be non-empty strings") |
| 3215 | if key.strip() != key: |
| 3216 | raise InvalidRecordPayloadError(f"Record field name has whitespace: {key!r}") |
| 3217 | if "\t" in key or "\n" in key or "\r" in key: |
| 3218 | raise InvalidRecordPayloadError(f"Record field name is not TSV-safe: {key!r}") |
| 3219 | if key == OP_FIELD: |
| 3220 | raise InvalidRecordPayloadError( |
| 3221 | f"Record field name is reserved for internal storage use: {key!r}" |
| 3222 | ) |
| 3223 | if key == EXTRA_FIELD: |
| 3224 | clean[key] = _normalize_extra_value(value) |
| 3225 | continue |
| 3226 | if value is None: |
| 3227 | clean[key] = "" |
| 3228 | elif isinstance(value, (str, int, float, bool)): |
| 3229 | clean[key] = str(value) |
| 3230 | else: |
| 3231 | raise InvalidRecordPayloadError( |
| 3232 | f"Record field value must be scalar or null: {key}" |
| 3233 | ) |
| 3234 | |
| 3235 | if generate_id: |
| 3236 | clean["id"] = object_ids.new_uuid4() |
| 3237 | elif require_id and "id" not in clean: |
| 3238 | raise InvalidRecordPayloadError("Record payload must include an id") |
| 3239 | |
| 3240 | return clean |
| 3241 | |
| 3242 | |
| 3243 | def _normalize_extra_value(value: Any) -> str: |
| 3244 | """Return a TSV-safe JSON-object string for the ``extra`` field. |
| 3245 | |
| 3246 | Unlike other record fields (scalar or null only), ``extra`` may be |
| 3247 | submitted as a dict (serialized here) or as a JSON string that must |
| 3248 | already parse to an object. Anything else is rejected. |
| 3249 | """ |
| 3250 | if isinstance(value, dict): |
| 3251 | return json.dumps(value, sort_keys=True) |
| 3252 | if isinstance(value, str): |
| 3253 | try: |
| 3254 | parsed = json.loads(value) |
| 3255 | except (TypeError, ValueError) as exc: |
| 3256 | raise InvalidRecordPayloadError( |
| 3257 | "Record field 'extra' must be a JSON object" |
| 3258 | ) from exc |
| 3259 | if not isinstance(parsed, dict): |
| 3260 | raise InvalidRecordPayloadError("Record field 'extra' must be a JSON object") |
| 3261 | return value |
| 3262 | raise InvalidRecordPayloadError( |
| 3263 | "Record field 'extra' must be an object or a JSON object string" |
| 3264 | ) |
| 3265 | |
| 3266 | |
| 3267 | def _apply_auto_created_at( |
| 3268 | collection: str, |
| 3269 | record: dict[str, str], |
| 3270 | submitted_fields: frozenset[str], |
| 3271 | *, |
| 3272 | base_dir: Path | str, |
| 3273 | roots: Iterable[Path] | None, |
| 3274 | ) -> dict[str, str]: |
| 3275 | """Server-set ``created_at`` on create when the schema declares it. |
| 3276 | |
| 3277 | A schema field named ``created_at`` (date/datetime) is filled with the |
| 3278 | current UTC time on create, unless the client supplied it. Because the |
| 3279 | server fills it (not the client), it can be ``read_only`` — clients can |
| 3280 | neither omit it nor spoof it — which is what lets generated lists show |
| 3281 | a trustworthy relative timestamp. |
| 3282 | """ |
| 3283 | if "created_at" in submitted_fields: |
| 3284 | return record |
| 3285 | for field in _schema_fields(collection, base_dir=base_dir, roots=roots): |
| 3286 | if field.get("name") != "created_at": |
| 3287 | continue |
| 3288 | field_type = str(field.get("type") or "").lower() |
| 3289 | now = datetime.now(timezone.utc) |
| 3290 | if field_type == "date": |
| 3291 | value = now.date().isoformat() |
| 3292 | elif field_type in {"datetime", "timestamp"}: |
| 3293 | value = now.isoformat().replace("+00:00", "Z") |
| 3294 | else: |
| 3295 | return record |
| 3296 | clean = dict(record) |
| 3297 | clean["created_at"] = value |
| 3298 | return clean |
| 3299 | return record |
| 3300 | |
| 3301 | |
| 3302 | def _apply_schema_defaults( |
| 3303 | collection: str, |
| 3304 | record: dict[str, str], |
| 3305 | *, |
| 3306 | base_dir: Path | str, |
| 3307 | roots: Iterable[Path] | None, |
| 3308 | ) -> dict[str, str]: |
| 3309 | fields = _schema_fields(collection, base_dir=base_dir, roots=roots) |
| 3310 | if not fields: |
| 3311 | return record |
| 3312 | |
| 3313 | clean = dict(record) |
| 3314 | for field in fields: |
| 3315 | name = field["name"] |
| 3316 | if name in clean and clean[name] != "": |
| 3317 | continue |
| 3318 | if _is_computed_or_read_only(field): |
| 3319 | continue |
| 3320 | if "default" not in field: |
| 3321 | continue |
| 3322 | clean[name] = _schema_scalar_to_string(field["default"], field_name=name) |
| 3323 | return clean |
| 3324 | |
| 3325 | |
| 3326 | # --------------------------------------------------------------------------- |
| 3327 | # Formula & rollup fields (plan/formula-rollup-spec.md): derived values, |
| 3328 | # MATERIALIZED at this storage layer so every writer (HTTP, daemon, seeds, |
| 3329 | # imports) keeps them consistent and every surface reads plain stored values. |
| 3330 | # Formulas recompute on writes of the record itself; rollups recompute on |
| 3331 | # writes/deletes in the source collection (single-hop, thread-local guarded). |
| 3332 | # A broken formula stores "" and never fails the write -- derived values are |
| 3333 | # non-authoritative, the opposite posture of pre-write hooks. |
| 3334 | # --------------------------------------------------------------------------- |
| 3335 | |
| 3336 | _ROLLUP_LOCAL = threading.local() |
| 3337 | |
| 3338 | |
| 3339 | def _apply_computed_fields( |
| 3340 | collection: str, |
| 3341 | record: dict[str, str], |
| 3342 | *, |
| 3343 | base_dir: Path | str, |
| 3344 | roots: Iterable[Path] | None, |
| 3345 | ) -> dict[str, str]: |
| 3346 | """Recompute this record's own derived values: rollups first (so a parent |
| 3347 | created/edited after its children exist is immediately correct), then |
| 3348 | formulas (which may reference the fresh rollup values).""" |
| 3349 | fields = _schema_fields(collection, base_dir=base_dir, roots=roots) |
| 3350 | if not fields: |
| 3351 | return record |
| 3352 | rollups = object_computed.rollup_fields(fields) |
| 3353 | formulas = object_computed.formula_fields(fields) |
| 3354 | if not rollups and not formulas: |
| 3355 | return record |
| 3356 | out = dict(record) |
| 3357 | for field in rollups: |
| 3358 | spec = field["rollup"] |
| 3359 | try: |
| 3360 | children = [ |
| 3361 | row |
| 3362 | for row in read_collection_records( |
| 3363 | spec["collection"], base_dir=base_dir, roots=roots |
| 3364 | ) |
| 3365 | if row.get(spec["fk_field"]) == out.get("id") |
| 3366 | ] |
| 3367 | except (object_collections.CollectionNotFoundError, OSError, ValueError): |
| 3368 | continue |
| 3369 | out[field["name"]] = object_computed.compute_rollup(spec, children) |
| 3370 | return object_computed.apply_formulas(fields, out) |
| 3371 | |
| 3372 | |
| 3373 | def _rollup_parents_for_source( |
| 3374 | source_collection: str, |
| 3375 | *, |
| 3376 | base_dir: Path | str, |
| 3377 | roots: Iterable[Path] | None, |
| 3378 | ) -> list[tuple[str, list[dict[str, Any]]]]: |
| 3379 | """(parent_collection, rollup_fields) pairs whose rollups aggregate the |
| 3380 | given source collection. Declared schemas only (the schemas dir); a |
| 3381 | collection cannot roll up itself (v1 loop guard).""" |
| 3382 | parents: list[tuple[str, list[dict[str, Any]]]] = [] |
| 3383 | schema_dir = Path(base_dir) / "schemas" |
| 3384 | try: |
| 3385 | names = sorted(path.stem for path in schema_dir.glob("*.json")) |
| 3386 | except OSError: |
| 3387 | return parents |
| 3388 | for name in names: |
| 3389 | if name == source_collection: |
| 3390 | continue |
| 3391 | fields = _schema_fields(name, base_dir=base_dir, roots=roots) |
| 3392 | specs = [ |
| 3393 | field |
| 3394 | for field in object_computed.rollup_fields(fields) |
| 3395 | if field["rollup"]["collection"] == source_collection |
| 3396 | ] |
| 3397 | if specs: |
| 3398 | parents.append((name, specs)) |
| 3399 | return parents |
| 3400 | |
| 3401 | |
| 3402 | def _recompute_rollups_for_source( |
| 3403 | source_collection: str, |
| 3404 | affected_rows: list[dict[str, str] | None], |
| 3405 | *, |
| 3406 | base_dir: Path | str, |
| 3407 | roots: Iterable[Path] | None, |
| 3408 | ) -> None: |
| 3409 | """After a write/delete in source_collection, bring affected parents' |
| 3410 | rollup fields current. Single-hop: the parent update this performs never |
| 3411 | triggers further rollup scans (thread-local guard). Best-effort by design: |
| 3412 | a failed parent refresh must never fail the child's own write.""" |
| 3413 | if getattr(_ROLLUP_LOCAL, "active", False): |
| 3414 | return |
| 3415 | parents = _rollup_parents_for_source(source_collection, base_dir=base_dir, roots=roots) |
| 3416 | if not parents: |
| 3417 | return |
| 3418 | _ROLLUP_LOCAL.active = True |
| 3419 | try: |
| 3420 | for parent_collection, specs in parents: |
| 3421 | fk_values: set[str] = set() |
| 3422 | for field in specs: |
| 3423 | fk_field = field["rollup"]["fk_field"] |
| 3424 | for row in affected_rows: |
| 3425 | if row and row.get(fk_field): |
| 3426 | fk_values.add(str(row[fk_field])) |
| 3427 | if not fk_values: |
| 3428 | continue |
| 3429 | try: |
| 3430 | source_rows = read_collection_records( |
| 3431 | source_collection, base_dir=base_dir, roots=roots |
| 3432 | ) |
| 3433 | except (object_collections.CollectionNotFoundError, OSError, ValueError): |
| 3434 | continue |
| 3435 | for parent_id in sorted(fk_values): |
| 3436 | try: |
| 3437 | parent = get_collection_record( |
| 3438 | parent_collection, parent_id, base_dir=base_dir, roots=roots |
| 3439 | ) |
| 3440 | except (RecordNotFoundError, object_collections.CollectionNotFoundError, |
| 3441 | InvalidRecordIdError, OSError, ValueError): |
| 3442 | continue |
| 3443 | changes: dict[str, str] = {} |
| 3444 | for field in specs: |
| 3445 | spec = field["rollup"] |
| 3446 | children = [ |
| 3447 | row for row in source_rows if row.get(spec["fk_field"]) == parent_id |
| 3448 | ] |
| 3449 | changes[field["name"]] = object_computed.compute_rollup(spec, children) |
| 3450 | if all(str(parent.get(k, "")) == v for k, v in changes.items()): |
| 3451 | continue |
| 3452 | try: |
| 3453 | update_collection_record( |
| 3454 | parent_collection, |
| 3455 | parent_id, |
| 3456 | changes, |
| 3457 | base_dir=base_dir, |
| 3458 | roots=roots, |
| 3459 | actor="system:rollup", |
| 3460 | allow_computed_submission=True, |
| 3461 | ) |
| 3462 | except Exception: |
| 3463 | continue |
| 3464 | finally: |
| 3465 | _ROLLUP_LOCAL.active = False |
| 3466 | |
| 3467 | |
| 3468 | def recompute_computed_fields( |
| 3469 | collection: str, |
| 3470 | *, |
| 3471 | base_dir: Path | str = DEFAULT_DATA_DIR, |
| 3472 | roots: Iterable[Path] | None = None, |
| 3473 | ) -> int: |
| 3474 | """Backfill/refresh every derived value in a collection; returns how many |
| 3475 | records changed. For definition changes and first adoption -- called |
| 3476 | explicitly (install migration, admin action, cron), never automatically in |
| 3477 | a runtime loop (docs: no runtime object rewriting; same spirit for data).""" |
| 3478 | fields = _schema_fields(collection, base_dir=base_dir, roots=roots) |
| 3479 | computed_names = { |
| 3480 | field["name"] |
| 3481 | for field in object_computed.rollup_fields(fields) |
| 3482 | + object_computed.formula_fields(fields) |
| 3483 | } |
| 3484 | if not computed_names: |
| 3485 | return 0 |
| 3486 | changed = 0 |
| 3487 | for row in read_collection_records(collection, base_dir=base_dir, roots=roots): |
| 3488 | desired = _apply_computed_fields(collection, dict(row), base_dir=base_dir, roots=roots) |
| 3489 | delta = { |
| 3490 | name: desired.get(name, "") |
| 3491 | for name in computed_names |
| 3492 | if str(row.get(name, "")) != str(desired.get(name, "")) |
| 3493 | } |
| 3494 | if not delta: |
| 3495 | continue |
| 3496 | update_collection_record( |
| 3497 | collection, |
| 3498 | row["id"], |
| 3499 | delta, |
| 3500 | base_dir=base_dir, |
| 3501 | roots=roots, |
| 3502 | actor="system:recompute", |
| 3503 | allow_computed_submission=True, |
| 3504 | ) |
| 3505 | changed += 1 |
| 3506 | return changed |
| 3507 | |
| 3508 | |
| 3509 | def _validate_field_transitions( |
| 3510 | collection: str, |
| 3511 | existing: dict[str, str], |
| 3512 | updated: dict[str, str], |
| 3513 | *, |
| 3514 | base_dir: Path | str, |
| 3515 | roots: Iterable[Path] | None, |
| 3516 | subject: object_permissions.PermissionSubject | None = None, |
| 3517 | ) -> None: |
| 3518 | """Enforce declared value transitions on update. |
| 3519 | |
| 3520 | A field may declare which values each current value can move to. Each |
| 3521 | list entry is either a plain string, allowed for any caller who may |
| 3522 | update the record, or a guarded object: |
| 3523 | |
| 3524 | {"name": "status", "type": "enum", "enum": [...], |
| 3525 | "transitions": { |
| 3526 | "open": ["assigned", {"to": "cancelled", "when": {"owner_id": "$user_id"}}], |
| 3527 | ... |
| 3528 | }} |
| 3529 | |
| 3530 | A guarded move is allowed only once every ``when`` clause matches the |
| 3531 | record's CURRENT stored values (``existing``, before this update) |
| 3532 | against the resolved subject variable or literal -- the same closed |
| 3533 | set and matching rules row filters use (see |
| 3534 | object_permissions.record_matches_filter). |
| 3535 | |
| 3536 | This is deliberately data plus one check — not a state machine |
| 3537 | framework: no hooks, no side effects, no transition callbacks. A |
| 3538 | current value missing from the map cannot change; an empty existing |
| 3539 | value may move anywhere. |
| 3540 | |
| 3541 | Guards are enforced only when ``subject`` is supplied. The HTTP update |
| 3542 | path resolves and plumbs the request subject once permissions have |
| 3543 | already run (see update_collection_record); direct library callers |
| 3544 | with no subject -- daemon, CLI, tests -- are trusted callers: a |
| 3545 | guarded move is still checked for validity (its "to" is in the list) |
| 3546 | but the "when" clause is not enforced. |
| 3547 | """ |
| 3548 | fields = _schema_fields(collection, base_dir=base_dir, roots=roots) |
| 3549 | for field in fields: |
| 3550 | transitions = field.get("transitions") |
| 3551 | if not isinstance(transitions, dict): |
| 3552 | continue |
| 3553 | name = field["name"] |
| 3554 | old_value = existing.get(name, "") |
| 3555 | new_value = updated.get(name, "") |
| 3556 | if old_value == new_value or _is_empty(old_value): |
| 3557 | continue |
| 3558 | |
| 3559 | entries = transitions.get(old_value) |
| 3560 | entries = entries if isinstance(entries, list) else [] |
| 3561 | |
| 3562 | allowed_values: list[str] = [] |
| 3563 | move_is_valid = False |
| 3564 | guard_failed = False |
| 3565 | for entry in entries: |
| 3566 | if isinstance(entry, str): |
| 3567 | to_value, when = entry, None |
| 3568 | elif isinstance(entry, dict): |
| 3569 | to_value, when = str(entry.get("to", "")), entry.get("when") or {} |
| 3570 | else: |
| 3571 | continue |
| 3572 | |
| 3573 | if to_value and to_value not in allowed_values: |
| 3574 | allowed_values.append(to_value) |
| 3575 | if to_value != new_value: |
| 3576 | continue |
| 3577 | |
| 3578 | if when is None or subject is None: |
| 3579 | move_is_valid = True |
| 3580 | elif object_permissions.record_matches_filter(existing, when, subject): |
| 3581 | move_is_valid = True |
| 3582 | else: |
| 3583 | guard_failed = True |
| 3584 | |
| 3585 | if move_is_valid: |
| 3586 | continue |
| 3587 | |
| 3588 | options = ", ".join(allowed_values) if allowed_values else "none" |
| 3589 | if guard_failed: |
| 3590 | raise TransitionNotAllowedError( |
| 3591 | f"Record field '{name}' cannot move from '{old_value}' to " |
| 3592 | f"'{new_value}' for this subject (allowed: {options})" |
| 3593 | ) |
| 3594 | raise InvalidRecordPayloadError( |
| 3595 | f"Record field '{name}' cannot move from '{old_value}' to " |
| 3596 | f"'{new_value}' (allowed: {options})" |
| 3597 | ) |
| 3598 | |
| 3599 | |
| 3600 | def _canonicalize_schema_values( |
| 3601 | collection: str, |
| 3602 | record: dict[str, str], |
| 3603 | *, |
| 3604 | base_dir: Path | str, |
| 3605 | roots: Iterable[Path] | None, |
| 3606 | ) -> dict[str, str]: |
| 3607 | """Store schema-typed values in one canonical form. |
| 3608 | |
| 3609 | Boolean fields accept several spellings on input ("True", "1", "yes") |
| 3610 | but must be stored as "true"/"false" so permission row filters like |
| 3611 | {"is_public": "true"} match by string comparison. Runs after |
| 3612 | validation, so every value here is known to parse. |
| 3613 | """ |
| 3614 | fields = _schema_fields(collection, base_dir=base_dir, roots=roots) |
| 3615 | if not fields: |
| 3616 | return record |
| 3617 | |
| 3618 | clean = dict(record) |
| 3619 | for field in fields: |
| 3620 | name = field["name"] |
| 3621 | value = clean.get(name) |
| 3622 | if _is_empty(value): |
| 3623 | continue |
| 3624 | field_type = str(field.get("type") or "text").lower() |
| 3625 | if field_type in _BOOLEAN_TYPES: |
| 3626 | clean[name] = "true" if _parse_boolean(value, field_name=name) else "false" |
| 3627 | return clean |
| 3628 | |
| 3629 | |
| 3630 | def _validate_record_against_schema( |
| 3631 | collection: str, |
| 3632 | record: dict[str, str], |
| 3633 | *, |
| 3634 | submitted_fields: frozenset[str], |
| 3635 | base_dir: Path | str, |
| 3636 | roots: Iterable[Path] | None, |
| 3637 | allow_read_only_submission: bool = False, |
| 3638 | allow_computed_submission: bool = False, |
| 3639 | ) -> None: |
| 3640 | fields = _schema_fields(collection, base_dir=base_dir, roots=roots) |
| 3641 | if not fields: |
| 3642 | return |
| 3643 | |
| 3644 | for field in fields: |
| 3645 | name = field["name"] |
| 3646 | value = record.get(name, "") |
| 3647 | |
| 3648 | if name in submitted_fields and _is_computed_or_read_only(field): |
| 3649 | is_computed = _is_computed_field(field) |
| 3650 | permitted = ( |
| 3651 | (allow_computed_submission and is_computed) |
| 3652 | or (allow_read_only_submission and not is_computed) |
| 3653 | ) |
| 3654 | if not permitted: |
| 3655 | raise InvalidRecordPayloadError( |
| 3656 | f"Record field '{name}' is computed or read-only and cannot be written" |
| 3657 | ) |
| 3658 | |
| 3659 | if _field_is_required(field) and not _is_computed_or_read_only(field) and _is_empty(value): |
| 3660 | raise InvalidRecordPayloadError(f"Record field '{name}' is required") |
| 3661 | |
| 3662 | if _is_empty(value): |
| 3663 | continue |
| 3664 | |
| 3665 | _validate_field_type(field, value) |
| 3666 | _validate_field_enum(field, value) |
| 3667 | _validate_field_rules(field, value) |
| 3668 | _validate_field_relation(field, value, base_dir=base_dir) |
| 3669 | |
| 3670 | |
| 3671 | def _check_field_storable(record: dict[str, Any]) -> None: |
| 3672 | """Reject a write whose any field value cannot be safely stored in the |
| 3673 | canonical plain-text TSV substrate, surfacing every case as a clean |
| 3674 | InvalidRecordPayloadError (a 4xx) rather than letting a raw stdlib |
| 3675 | _csv.Error or UnicodeEncodeError escape the public API. Three unstorable |
| 3676 | classes: |
| 3677 | |
| 3678 | - **Too large** (> MAX_TSV_FIELD_BYTES): without this, an oversize cell |
| 3679 | silently empties an append-mode collection on read. |
| 3680 | - **NUL (0x00)**: Python's csv writer literally cannot represent a NUL |
| 3681 | under QUOTE_ALL/MINIMAL ("need to escape, but no escapechar set"), and a |
| 3682 | NUL corrupts the text tooling the plain-text-durability guarantee rests |
| 3683 | on -- it is not text. Legitimate JSON escapes NUL to \\u0000, so real |
| 3684 | structured data never carries a raw one. |
| 3685 | - **Unencodable UTF-8** (a lone surrogate, e.g. from bad input or |
| 3686 | json.dumps(ensure_ascii=False) over broken data): can't be written at |
| 3687 | all. |
| 3688 | |
| 3689 | Enforced on create/update (the entry points for new/changed data); |
| 3690 | internal rewrites (compaction, transition) don't pass here -- they only |
| 3691 | re-serialize data that already passed this gate. Iterates the record's own |
| 3692 | values, so it guards schemaless collections and the packed `extra` blob |
| 3693 | too. |
| 3694 | """ |
| 3695 | for name, value in record.items(): |
| 3696 | if value is None: |
| 3697 | continue |
| 3698 | text = str(value) |
| 3699 | if "\x00" in text: |
| 3700 | raise InvalidRecordPayloadError( |
| 3701 | f"Record field '{name}' contains a NUL byte, which cannot be " |
| 3702 | f"stored in the plain-text substrate (JSON encodes NUL as " |
| 3703 | f"\\u0000)" |
| 3704 | ) |
| 3705 | try: |
| 3706 | size = len(text.encode("utf-8")) |
| 3707 | except UnicodeEncodeError as exc: |
| 3708 | raise InvalidRecordPayloadError( |
| 3709 | f"Record field '{name}' is not valid UTF-8 (unencodable " |
| 3710 | f"surrogate) and cannot be stored: {exc}" |
| 3711 | ) from exc |
| 3712 | if size > MAX_TSV_FIELD_BYTES: |
| 3713 | raise InvalidRecordPayloadError( |
| 3714 | f"Record field '{name}' is {size} bytes, exceeding the " |
| 3715 | f"{MAX_TSV_FIELD_BYTES}-byte per-field maximum" |
| 3716 | ) |
| 3717 | |
| 3718 | |
| 3719 | # A bare carriage return -- "\r" NOT immediately followed by "\n". csv's |
| 3720 | # QUOTE_MINIMAL does not quote a field for a lone CR (it quotes only for the |
| 3721 | # delimiter, the quotechar, and "\n"), but csv.reader treats a bare CR in an |
| 3722 | # UNQUOTED field as a row terminator -- silently splitting one row into two |
| 3723 | # (and, in append mode, hiding every later record from the fold path). "\r\n" |
| 3724 | # and lone "\n" are already handled by QUOTE_MINIMAL (both contain "\n"). |
| 3725 | _BARE_CR_RE = re.compile(r"\r(?!\n)") |
| 3726 | |
| 3727 | |
| 3728 | def _rows_need_full_quoting(rows: Iterable[dict[str, Any]]) -> bool: |
| 3729 | """True if any field value in `rows` contains a bare CR (see _BARE_CR_RE). |
| 3730 | |
| 3731 | When it does, that write uses csv.QUOTE_ALL so every field is quoted and |
| 3732 | the CR round-trips intact; the overwhelming common case (no bare CR) stays |
| 3733 | QUOTE_MINIMAL and byte-identical to before. This keeps the compact |
| 3734 | plain-text format for normal data while making a lone CR lossless instead |
| 3735 | of silently corrupting -- no format change for anyone who never stores a |
| 3736 | bare CR. Readers parse quoted and unquoted rows alike, so a file may mix |
| 3737 | both. Note: values destined for the packed `extra` JSON blob have their CR |
| 3738 | escaped by json.dumps, so scanning raw values here only ever OVER-detects |
| 3739 | (a harmless extra-quoted write), never misses a CR that reaches a column. |
| 3740 | """ |
| 3741 | for row in rows: |
| 3742 | for value in row.values(): |
| 3743 | if value is None: |
| 3744 | continue |
| 3745 | text = str(value) |
| 3746 | if "\r" in text and _BARE_CR_RE.search(text): |
| 3747 | return True |
| 3748 | return False |
| 3749 | |
| 3750 | |
| 3751 | def _schema_fields( |
| 3752 | collection: str, |
| 3753 | *, |
| 3754 | base_dir: Path | str, |
| 3755 | roots: Iterable[Path] | None, |
| 3756 | ) -> list[dict[str, Any]]: |
| 3757 | try: |
| 3758 | schema = object_schemas.get_schema(collection, base_dir=base_dir, roots=roots) |
| 3759 | except object_schemas.SchemaNotFoundError: |
| 3760 | return [] |
| 3761 | fields = schema.get("fields", []) |
| 3762 | if not isinstance(fields, list): |
| 3763 | return [] |
| 3764 | return [field for field in fields if isinstance(field, dict) and "name" in field] |
| 3765 | |
| 3766 | |
| 3767 | def _field_is_required(field: dict[str, Any]) -> bool: |
| 3768 | validation = field.get("validation") if isinstance(field.get("validation"), dict) else {} |
| 3769 | return bool(field.get("required") or validation.get("required") or validation.get("not_null")) |
| 3770 | |
| 3771 | |
| 3772 | def _is_computed_or_read_only(field: dict[str, Any]) -> bool: |
| 3773 | field_type = str(field.get("type", "")).lower() |
| 3774 | return bool( |
| 3775 | field_type == "computed" |
| 3776 | or field.get("computed") |
| 3777 | or field.get("read_only") |
| 3778 | or field.get("readonly") |
| 3779 | or field.get("readOnly") |
| 3780 | ) |
| 3781 | |
| 3782 | |
| 3783 | def _is_computed_field(field: dict[str, Any]) -> bool: |
| 3784 | """Return True for a server-derived formula field, as opposed to a |
| 3785 | field that is merely ``read_only`` (client can't set it, but nothing |
| 3786 | computes its value -- see ``preserve_read_only`` on |
| 3787 | ``create_collection_record``).""" |
| 3788 | field_type = str(field.get("type", "")).lower() |
| 3789 | return bool(field_type == "computed" or field.get("computed")) |
| 3790 | |
| 3791 | |
| 3792 | def _is_empty(value: str | None) -> bool: |
| 3793 | return value is None or value == "" |
| 3794 | |
| 3795 | |
| 3796 | def _schema_scalar_to_string(value: Any, *, field_name: str) -> str: |
| 3797 | if value is None: |
| 3798 | return "" |
| 3799 | if isinstance(value, (str, int, float, bool)): |
| 3800 | return str(value) |
| 3801 | raise InvalidRecordPayloadError(f"Schema default for '{field_name}' must be scalar or null") |
| 3802 | |
| 3803 | |
| 3804 | def _validate_field_type(field: dict[str, Any], value: str) -> None: |
| 3805 | name = field["name"] |
| 3806 | field_type = str(field.get("type") or "text").lower() |
| 3807 | if field_type in _INTEGER_TYPES: |
| 3808 | _parse_integer(value, field_name=name) |
| 3809 | elif field_type in _FLOAT_TYPES: |
| 3810 | _parse_float(value, field_name=name) |
| 3811 | elif field_type in _BOOLEAN_TYPES: |
| 3812 | _parse_boolean(value, field_name=name) |
| 3813 | elif field_type == "date": |
| 3814 | _parse_date(value, field_name=name) |
| 3815 | elif field_type in {"datetime", "timestamp"}: |
| 3816 | _parse_datetime(value, field_name=name) |
| 3817 | elif field_type == "enum": |
| 3818 | _validate_field_enum(field, value, required=True) |
| 3819 | |
| 3820 | |
| 3821 | def _validate_field_enum(field: dict[str, Any], value: str, *, required: bool = False) -> None: |
| 3822 | enum_payload = field.get("enum") |
| 3823 | validation = field.get("validation") if isinstance(field.get("validation"), dict) else {} |
| 3824 | if enum_payload is None: |
| 3825 | enum_payload = validation.get("choices", validation.get("in")) |
| 3826 | values = _enum_values(enum_payload) |
| 3827 | if not values: |
| 3828 | if required: |
| 3829 | raise InvalidRecordPayloadError(f"Record field '{field['name']}' enum has no values") |
| 3830 | return |
| 3831 | if value not in values: |
| 3832 | allowed = ", ".join(values) |
| 3833 | raise InvalidRecordPayloadError( |
| 3834 | f"Record field '{field['name']}' must be one of: {allowed}" |
| 3835 | ) |
| 3836 | |
| 3837 | |
| 3838 | def _enum_values(enum_payload: Any) -> list[str]: |
| 3839 | if enum_payload is None: |
| 3840 | return [] |
| 3841 | if isinstance(enum_payload, dict): |
| 3842 | enum_payload = enum_payload.get("values") |
| 3843 | if not isinstance(enum_payload, list): |
| 3844 | return [] |
| 3845 | values = [] |
| 3846 | for item in enum_payload: |
| 3847 | if isinstance(item, (str, int, float, bool)): |
| 3848 | values.append(str(item)) |
| 3849 | return values |
| 3850 | |
| 3851 | |
| 3852 | def _validate_field_relation( |
| 3853 | field: dict[str, Any], |
| 3854 | value: str, |
| 3855 | *, |
| 3856 | base_dir: Path | str, |
| 3857 | ) -> None: |
| 3858 | """Require relation values to be existing record ids in the target collection. |
| 3859 | |
| 3860 | A relation is a validated pointer plus a display hint — deliberately not |
| 3861 | an association framework: no joins, no lazy loading, no cascades. |
| 3862 | """ |
| 3863 | relation = field.get("relation") |
| 3864 | if relation is None: |
| 3865 | return |
| 3866 | |
| 3867 | name = field["name"] |
| 3868 | if isinstance(relation, str): |
| 3869 | target = relation |
| 3870 | elif isinstance(relation, dict): |
| 3871 | target = relation.get("collection") |
| 3872 | else: |
| 3873 | raise InvalidRecordPayloadError( |
| 3874 | f"Record field '{name}' relation must be a collection name or object" |
| 3875 | ) |
| 3876 | |
| 3877 | if not isinstance(target, str) or not object_collections.validate_collection_name(target): |
| 3878 | raise InvalidRecordPayloadError( |
| 3879 | f"Record field '{name}' relation has an invalid collection" |
| 3880 | ) |
| 3881 | |
| 3882 | try: |
| 3883 | get_collection_record(target, value, base_dir=base_dir) |
| 3884 | except (object_collections.CollectionNotFoundError, RecordNotFoundError) as exc: |
| 3885 | raise InvalidRecordPayloadError( |
| 3886 | f"Record field '{name}' references a missing record: {target}/{value}" |
| 3887 | ) from exc |
| 3888 | except InvalidRecordIdError as exc: |
| 3889 | raise InvalidRecordPayloadError( |
| 3890 | f"Record field '{name}' relation value is not a valid record id" |
| 3891 | ) from exc |
| 3892 | |
| 3893 | |
| 3894 | def _validate_field_rules(field: dict[str, Any], value: str) -> None: |
| 3895 | validation = field.get("validation") |
| 3896 | if not isinstance(validation, dict): |
| 3897 | return |
| 3898 | |
| 3899 | name = field["name"] |
| 3900 | if "min_length" in validation and len(value) < _rule_int(validation["min_length"], name, "min_length"): |
| 3901 | raise InvalidRecordPayloadError(f"Record field '{name}' is shorter than min_length") |
| 3902 | if "max_length" in validation and len(value) > _rule_int(validation["max_length"], name, "max_length"): |
| 3903 | raise InvalidRecordPayloadError(f"Record field '{name}' is longer than max_length") |
| 3904 | |
| 3905 | pattern = validation.get("pattern", validation.get("regex")) |
| 3906 | if pattern is not None: |
| 3907 | if not isinstance(pattern, str): |
| 3908 | raise InvalidRecordPayloadError(f"Record field '{name}' validation pattern must be a string") |
| 3909 | try: |
| 3910 | matches = re.fullmatch(pattern, value) |
| 3911 | except re.error as exc: |
| 3912 | raise InvalidRecordPayloadError( |
| 3913 | f"Record field '{name}' validation pattern is invalid" |
| 3914 | ) from exc |
| 3915 | if not matches: |
| 3916 | raise InvalidRecordPayloadError(f"Record field '{name}' does not match pattern") |
| 3917 | |
| 3918 | if "min" in validation: |
| 3919 | if _parse_float(value, field_name=name) < _rule_float(validation["min"], name, "min"): |
| 3920 | raise InvalidRecordPayloadError(f"Record field '{name}' is below min") |
| 3921 | if "max" in validation: |
| 3922 | if _parse_float(value, field_name=name) > _rule_float(validation["max"], name, "max"): |
| 3923 | raise InvalidRecordPayloadError(f"Record field '{name}' is above max") |
| 3924 | |
| 3925 | |
| 3926 | def _parse_integer(value: str, *, field_name: str) -> int: |
| 3927 | try: |
| 3928 | if value.strip() != value or not re.fullmatch(r"[+-]?\d+", value): |
| 3929 | raise ValueError |
| 3930 | return int(value) |
| 3931 | except ValueError as exc: |
| 3932 | raise InvalidRecordPayloadError(f"Record field '{field_name}' must be an integer") from exc |
| 3933 | |
| 3934 | |
| 3935 | def _parse_float(value: str, *, field_name: str) -> float: |
| 3936 | try: |
| 3937 | number = float(value) |
| 3938 | except ValueError as exc: |
| 3939 | raise InvalidRecordPayloadError(f"Record field '{field_name}' must be a number") from exc |
| 3940 | if not math.isfinite(number): |
| 3941 | raise InvalidRecordPayloadError(f"Record field '{field_name}' must be a finite number") |
| 3942 | return number |
| 3943 | |
| 3944 | |
| 3945 | def _parse_boolean(value: str, *, field_name: str) -> bool: |
| 3946 | text = value.lower() |
| 3947 | if text in _TRUE_VALUES: |
| 3948 | return True |
| 3949 | if text in _FALSE_VALUES: |
| 3950 | return False |
| 3951 | raise InvalidRecordPayloadError(f"Record field '{field_name}' must be a boolean") |
| 3952 | |
| 3953 | |
| 3954 | def _parse_date(value: str, *, field_name: str) -> date: |
| 3955 | try: |
| 3956 | return date.fromisoformat(value) |
| 3957 | except ValueError as exc: |
| 3958 | raise InvalidRecordPayloadError(f"Record field '{field_name}' must be a date") from exc |
| 3959 | |
| 3960 | |
| 3961 | def _parse_datetime(value: str, *, field_name: str) -> datetime: |
| 3962 | try: |
| 3963 | return datetime.fromisoformat(value.replace("Z", "+00:00")) |
| 3964 | except ValueError as exc: |
| 3965 | raise InvalidRecordPayloadError(f"Record field '{field_name}' must be a datetime") from exc |
| 3966 | |
| 3967 | |
| 3968 | def _rule_int(value: Any, field_name: str, rule_name: str) -> int: |
| 3969 | try: |
| 3970 | return int(value) |
| 3971 | except (TypeError, ValueError) as exc: |
| 3972 | raise InvalidRecordPayloadError( |
| 3973 | f"Record field '{field_name}' validation {rule_name} must be an integer" |
| 3974 | ) from exc |
| 3975 | |
| 3976 | |
| 3977 | def _rule_float(value: Any, field_name: str, rule_name: str) -> float: |
| 3978 | try: |
| 3979 | number = float(value) |
| 3980 | except (TypeError, ValueError) as exc: |
| 3981 | raise InvalidRecordPayloadError( |
| 3982 | f"Record field '{field_name}' validation {rule_name} must be a number" |
| 3983 | ) from exc |
| 3984 | if not math.isfinite(number): |
| 3985 | raise InvalidRecordPayloadError( |
| 3986 | f"Record field '{field_name}' validation {rule_name} must be finite" |
| 3987 | ) |
| 3988 | return number |
| 3989 | |
| 3990 | |
| 3991 | def _extra_field_names( |
| 3992 | collection: str, |
| 3993 | *, |
| 3994 | base_dir: Path | str, |
| 3995 | roots: Iterable[Path] | None, |
| 3996 | ) -> set[str]: |
| 3997 | """Return the names of schema fields whose value lives inside ``extra``.""" |
| 3998 | fields = _schema_fields(collection, base_dir=base_dir, roots=roots) |
| 3999 | return {field["name"] for field in fields if field.get("store") == EXTRA_FIELD} |
| 4000 | |
| 4001 | |
| 4002 | def _parse_extra_blob(value: str | None) -> dict[str, Any]: |
| 4003 | """Parse an ``extra`` TSV cell into a dict, tolerating missing/invalid data.""" |
| 4004 | if not value: |
| 4005 | return {} |
| 4006 | try: |
| 4007 | parsed = json.loads(value) |
| 4008 | except (TypeError, ValueError): |
| 4009 | return {} |
| 4010 | return parsed if isinstance(parsed, dict) else {} |
| 4011 | |
| 4012 | |
| 4013 | def _route_extra( |
| 4014 | record: dict[str, str], |
| 4015 | *, |
| 4016 | existing_blob: dict[str, Any], |
| 4017 | extra_names: set[str], |
| 4018 | ) -> dict[str, str]: |
| 4019 | """Fold ``extra`` and any ``store: extra`` view fields into one JSON blob. |
| 4020 | |
| 4021 | Pure: builds on ``existing_blob`` (so a partial update never wipes |
| 4022 | untouched blob keys), shallow-merges any client-submitted ``extra`` |
| 4023 | object into it, then pulls each declared view field out of the |
| 4024 | top-level record and into the blob. The record keeps a single |
| 4025 | ``extra`` TSV column holding the serialized blob (or no column at all |
| 4026 | when the blob is empty), never the view field's own column. |
| 4027 | """ |
| 4028 | blob = dict(existing_blob) |
| 4029 | if EXTRA_FIELD in record: |
| 4030 | blob.update(_parse_extra_blob(record.pop(EXTRA_FIELD))) |
| 4031 | for name in extra_names: |
| 4032 | if name in record: |
| 4033 | blob[name] = record.pop(name) |
| 4034 | |
| 4035 | if blob: |
| 4036 | serialized = json.dumps(blob, sort_keys=True) |
| 4037 | if "\t" in serialized or "\n" in serialized or "\r" in serialized: |
| 4038 | raise InvalidRecordPayloadError("Record field 'extra' is not TSV-safe") |
| 4039 | record[EXTRA_FIELD] = serialized |
| 4040 | else: |
| 4041 | record.pop(EXTRA_FIELD, None) |
| 4042 | return record |
| 4043 | |
| 4044 | |
| 4045 | def _surface_extra(record: dict[str, str], *, extra_names: set[str]) -> dict[str, str]: |
| 4046 | """Expose each declared ``store: extra`` view field at the top level. |
| 4047 | |
| 4048 | Pure: ``record["extra"]`` is left as-is (still the raw JSON string) so |
| 4049 | undeclared blob data stays reachable. A no-op when there are no view |
| 4050 | fields and no ``extra`` column. |
| 4051 | """ |
| 4052 | if not extra_names: |
| 4053 | return record |
| 4054 | blob = _parse_extra_blob(record.get(EXTRA_FIELD)) |
| 4055 | for name in extra_names: |
| 4056 | record[name] = str(blob.get(name, "")) |
| 4057 | return record |
| 4058 | |
| 4059 | |
| 4060 | def _merge_fields(existing_fields: list[str], record: dict[str, str]) -> list[str]: |
| 4061 | fields = list(existing_fields) if existing_fields else ["id"] |
| 4062 | if "id" not in fields: |
| 4063 | fields.insert(0, "id") |
| 4064 | for field in record: |
| 4065 | if field not in fields: |
| 4066 | fields.append(field) |
| 4067 | return fields |
| 4068 | |
| 4069 | |
| 4070 | def _project_record(record: dict[str, str], fields: list[str]) -> dict[str, str]: |
| 4071 | return {field: record.get(field, "") for field in fields} |
| 4072 | |
| 4073 | |
| 4074 | def _validate_page(*, limit: int, offset: int) -> None: |
| 4075 | if limit < 1: |
| 4076 | raise ValueError("Record limit must be at least 1") |
| 4077 | if limit > MAX_RECORD_LIMIT: |
| 4078 | raise ValueError(f"Record limit must be at most {MAX_RECORD_LIMIT}") |
| 4079 | if offset < 0: |
| 4080 | raise ValueError("Record offset must be at least 0") |