askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_durability_torn_write_characterization.py · 39.5 KB · 833 lines
| 1 | """CHARACTERIZATION tests: crash-recovery / torn-write durability of the TSV |
| 2 | substrate (object_records.py). This is the crown-jewel property of a |
| 3 | "database": does a write interrupted at ANY byte offset (process killed, |
| 4 | disk full, container OOM-killed mid-flush) leave the store in a valid state |
| 5 | with no data loss and no corruption? |
| 6 | |
| 7 | Pure characterization -- this file does not modify any production module. |
| 8 | Where behavior fails a data-safety property, the assertion is written to |
| 9 | state the property a crash-safe substrate SHOULD have, and is either (a) |
| 10 | left to fail loudly (a hard assertion, when the failure would be a NEW |
| 11 | finding not already known/deferred), or (b) wrapped in `pytest.mark.xfail |
| 12 | (strict=True)` when it reproduces the already-documented, deliberately |
| 13 | deferred substrate bug #2 (see object_records._repair_torn_tail's |
| 14 | "KNOWN LIMITATION" docstring paragraph, and |
| 15 | test_append_mode_torn_tail_mid_multiline_row_is_silently_resurrected_and_ |
| 16 | cascades_FINDING in test_embedded_json_lines_characterization.py, which |
| 17 | first identified it). Do not "fix" a failing assertion here by weakening it |
| 18 | to match broken behavior, and do not fix object_records.py from this file. |
| 19 | |
| 20 | CONTEXT (see object_records.py for the authoritative version of this): |
| 21 | - CLASSIC mode always rewrites the whole file atomically: temp file + |
| 22 | `Path.replace` (_write_collection_records). A crash strictly before the |
| 23 | replace leaves the OLD file completely untouched (atomic rename is all- |
| 24 | or-nothing at the filesystem level) plus, possibly, an orphaned temp |
| 25 | file next to it that no read path ever looks at. |
| 26 | - APPEND mode appends rows in place (open("a")) and maintains an |
| 27 | id->offset sidecar (`.records.oidx`) alongside records.tsv. Before each |
| 28 | append, `_repair_torn_tail` truncates any unterminated fragment left by |
| 29 | a prior crash (_drop_torn_tail's rule: "does the file end with a raw |
| 30 | newline byte" is treated as "is the tail committed"). Both compaction |
| 31 | and any full rewrite of an append-format file reuse the same atomic |
| 32 | temp+replace path as classic mode. |
| 33 | - SUBSTRATE BUG #2 (documented, deferred, xfailed elsewhere -- NOT fixed |
| 34 | here): the "ends with \\n" check behind both _drop_torn_tail and |
| 35 | _repair_torn_tail is QUOTE-BLIND. It only makes sense for a row whose |
| 36 | value never itself contains a raw newline byte. A crash that lands |
| 37 | right after a newline that is INSIDE an open quoted field (e.g. a |
| 38 | pretty-printed embedded-JSON cell) leaves the file ending in "\\n" even |
| 39 | though the row -- and the write -- never actually completed. Both |
| 40 | checks then wrongly conclude "not torn": the fragment is resurrected as |
| 41 | a garbled "committed" record, and (because its quote was never closed |
| 42 | on disk) every row appended afterward is silently absorbed AS TEXT into |
| 43 | that still-open field until the collection is compacted. |
| 44 | |
| 45 | This file's job is to map that surface exhaustively (every byte offset of a |
| 46 | representative multi-line row, not just the one case already known) and to |
| 47 | probe the durability of every OTHER torn-write surface this substrate has: |
| 48 | single-line append rows, the oidx sidecar file, a torn header, an |
| 49 | interrupted compaction, and classic mode's atomic-rename guarantee. |
| 50 | """ |
| 51 | |
| 52 | from __future__ import annotations |
| 53 | |
| 54 | import json |
| 55 | import os |
| 56 | import threading |
| 57 | from pathlib import Path |
| 58 | |
| 59 | import pytest |
| 60 | |
| 61 | import object_records |
| 62 | |
| 63 | pytestmark = pytest.mark.conformance |
| 64 | |
| 65 | |
| 66 | ID_FIELD = {"name": "id"} |
| 67 | VALUE_FIELD = {"name": "value"} |
| 68 | LINES_FIELD = {"name": "lines", "type": "textarea"} |
| 69 | |
| 70 | |
| 71 | # ============================================================================= |
| 72 | # setup helpers (mirror tests/test_object_records.py and |
| 73 | # tests/test_embedded_json_lines_characterization.py's conventions exactly) |
| 74 | # ============================================================================= |
| 75 | |
| 76 | |
| 77 | def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 78 | path = data_dir / "schemas" / f"{collection}.json" |
| 79 | path.parent.mkdir(parents=True, exist_ok=True) |
| 80 | path.write_text(json.dumps({"fields": fields})) |
| 81 | return path |
| 82 | |
| 83 | |
| 84 | def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 85 | path = data_dir / "schemas" / f"{collection}.json" |
| 86 | path.parent.mkdir(parents=True, exist_ok=True) |
| 87 | path.write_text(json.dumps({"fields": fields, "storage": "append"})) |
| 88 | return path |
| 89 | |
| 90 | |
| 91 | def _clear_caches() -> None: |
| 92 | """Force the NEXT read to go past the warm in-process caches -- see |
| 93 | tests/test_embedded_json_lines_characterization.py's identical helper |
| 94 | for why this matters for exercising the id->offset sidecar path.""" |
| 95 | object_records._RECORDS_CACHE.clear() |
| 96 | object_records._OIDX_CACHE.clear() |
| 97 | |
| 98 | |
| 99 | def _records_path(data_dir: Path, collection: str) -> Path: |
| 100 | return data_dir / "collections" / collection / "records.tsv" |
| 101 | |
| 102 | |
| 103 | def _stray_tmp_path(real_path: Path) -> Path: |
| 104 | """A sibling temp-file name matching the EXACT pattern |
| 105 | _write_collection_records uses (`.{name}.{pid}.{tid}.tmp`), so this |
| 106 | reproduces -- byte-for-byte, filename-wise -- the file a real crash |
| 107 | between that function's temp-write and its `Path.replace` call would |
| 108 | leave behind.""" |
| 109 | return real_path.with_name(f".{real_path.name}.{os.getpid()}.{threading.get_ident()}.tmp") |
| 110 | |
| 111 | |
| 112 | def _captured_next_row_bytes( |
| 113 | data_dir: Path, collection: str, payload: dict |
| 114 | ) -> tuple[bytes, bytes]: |
| 115 | """Perform a REAL create_collection_record and return (bytes_before, |
| 116 | row_bytes): row_bytes is the EXACT physical bytes production appended |
| 117 | for this create, captured by diffing records.tsv before/after, and the |
| 118 | file is then restored to bytes_before -- i.e. this "un-commits" the |
| 119 | create, leaving a pristine last-known-good file plus a byte-exact |
| 120 | template of what a real write to it would look like, for callers to |
| 121 | layer synthetic torn prefixes onto. Using the module's own writer to |
| 122 | produce these bytes (rather than hand-building csv text) guarantees the |
| 123 | dialect/quoting exactly matches what a genuine crash would have been |
| 124 | interrupted mid-way through. |
| 125 | """ |
| 126 | path = _records_path(data_dir, collection) |
| 127 | before = path.read_bytes() |
| 128 | object_records.create_collection_record(collection, payload, base_dir=data_dir, roots=[]) |
| 129 | after = path.read_bytes() |
| 130 | assert after.startswith(before), "sanity: a fast append must only ever add bytes at EOF" |
| 131 | row_bytes = after[len(before):] |
| 132 | path.write_bytes(before) |
| 133 | oidx_path = object_records._oidx_path(path) |
| 134 | if oidx_path.exists(): |
| 135 | oidx_path.unlink() |
| 136 | _clear_caches() |
| 137 | return before, row_bytes |
| 138 | |
| 139 | |
| 140 | def _fold_and_by_id_agree(collection: str, data_dir: Path, ids: list[str]) -> bool: |
| 141 | """Property (c) from the task: the fold-all read (read_collection_records) |
| 142 | and the by-id read (get_collection_record, which for an append-mode |
| 143 | collection with a cold cache goes through the id->offset sidecar) must |
| 144 | return the identical value for every id the fold path claims is live, |
| 145 | and get_collection_record must raise RecordNotFoundError for every id |
| 146 | the fold path does NOT claim is live.""" |
| 147 | _clear_caches() |
| 148 | folded = {r["id"]: r for r in object_records.read_collection_records( |
| 149 | collection, base_dir=data_dir, roots=[] |
| 150 | )} |
| 151 | for record_id in ids: |
| 152 | _clear_caches() |
| 153 | if record_id in folded: |
| 154 | try: |
| 155 | by_id = object_records.get_collection_record( |
| 156 | collection, record_id, base_dir=data_dir, roots=[] |
| 157 | ) |
| 158 | except object_records.RecordNotFoundError: |
| 159 | return False |
| 160 | if by_id != folded[record_id]: |
| 161 | return False |
| 162 | else: |
| 163 | try: |
| 164 | object_records.get_collection_record( |
| 165 | collection, record_id, base_dir=data_dir, roots=[] |
| 166 | ) |
| 167 | except object_records.RecordNotFoundError: |
| 168 | continue |
| 169 | return False |
| 170 | return True |
| 171 | |
| 172 | |
| 173 | # ============================================================================= |
| 174 | # 1. APPEND MODE -- single-physical-line row, EVERY byte offset |
| 175 | # ============================================================================= |
| 176 | |
| 177 | |
| 178 | def test_append_single_line_row_torn_at_every_byte_offset_self_heals(tmp_path): |
| 179 | """Baseline surface, swept exhaustively rather than spot-checked: a row |
| 180 | with no embedded newline of its own has no way to "look complete" via |
| 181 | the endswith(b'\\n') check unless the FULL row (including its own |
| 182 | terminating \\n) is present. Every strict prefix is therefore expected, |
| 183 | at EVERY byte offset, to (a) lose no previously-committed record, (b) |
| 184 | never resurrect a phantom/partial record, (c) have the fold and by-id |
| 185 | read paths agree, and (d) self-heal cleanly on the next write. This is |
| 186 | the property the whole append-mode design is supposed to guarantee; |
| 187 | confirming it holds at every offset (not just one hand-picked cut) is |
| 188 | the point of this test. |
| 189 | """ |
| 190 | data_dir = tmp_path / "data" |
| 191 | write_append_schema(data_dir, "logs", [ID_FIELD, VALUE_FIELD]) |
| 192 | object_records.create_collection_record( |
| 193 | "logs", {"id": "A", "value": "one"}, base_dir=data_dir, roots=[] |
| 194 | ) |
| 195 | object_records.create_collection_record( |
| 196 | "logs", {"id": "B", "value": "two"}, base_dir=data_dir, roots=[] |
| 197 | ) |
| 198 | path = _records_path(data_dir, "logs") |
| 199 | |
| 200 | before, row_bytes = _captured_next_row_bytes( |
| 201 | data_dir, "logs", {"id": "C", "value": "three-single-physical-line"} |
| 202 | ) |
| 203 | assert row_bytes.count(b"\n") == 1, "sanity: genuinely a single physical line" |
| 204 | assert row_bytes.endswith(b"\n") |
| 205 | |
| 206 | for offset in range(0, len(row_bytes)): # every strict prefix: 0 .. len-1 |
| 207 | fragment = row_bytes[:offset] |
| 208 | path.write_bytes(before + fragment) |
| 209 | oidx_path = object_records._oidx_path(path) |
| 210 | if oidx_path.exists(): |
| 211 | oidx_path.unlink() |
| 212 | _clear_caches() |
| 213 | |
| 214 | listing = object_records.list_collection_records("logs", base_dir=data_dir, roots=[])[ |
| 215 | "records" |
| 216 | ] |
| 217 | assert listing == [{"id": "A", "value": "one"}, {"id": "B", "value": "two"}], ( |
| 218 | f"offset={offset}/{len(row_bytes)}: torn fragment must not lose A/B or " |
| 219 | f"resurrect a phantom C; got {listing!r}" |
| 220 | ) |
| 221 | assert _fold_and_by_id_agree("logs", data_dir, ["A", "B", "C"]), ( |
| 222 | f"offset={offset}: fold-all and by-id reads disagree" |
| 223 | ) |
| 224 | |
| 225 | # Self-heal: the next normal write must land as its own clean record, |
| 226 | # and the torn fragment must never resurface. |
| 227 | object_records.create_collection_record( |
| 228 | "logs", {"id": "D", "value": "four"}, base_dir=data_dir, roots=[] |
| 229 | ) |
| 230 | _clear_caches() |
| 231 | listing_after = object_records.list_collection_records( |
| 232 | "logs", base_dir=data_dir, roots=[] |
| 233 | )["records"] |
| 234 | assert listing_after == [ |
| 235 | {"id": "A", "value": "one"}, |
| 236 | {"id": "B", "value": "two"}, |
| 237 | {"id": "D", "value": "four"}, |
| 238 | ], f"offset={offset}: self-heal did not land D cleanly; got {listing_after!r}" |
| 239 | |
| 240 | |
| 241 | # ============================================================================= |
| 242 | # 2. APPEND MODE -- multi-physical-line row (embedded newline), EVERY byte |
| 243 | # offset -- maps substrate bug #2's exact trigger surface |
| 244 | # ============================================================================= |
| 245 | |
| 246 | |
| 247 | def _hostile_multiline_blob() -> str: |
| 248 | payload = [ |
| 249 | {"sku": "SKU-1", "note": 'café "q" \\z 日本', "amount_cents": 1099}, |
| 250 | {"sku": "SKU-2", "note": "plain ascii", "amount_cents": -50}, |
| 251 | ] |
| 252 | return json.dumps(payload, indent=2) # pretty-printed: real raw "\n" bytes in the cell |
| 253 | |
| 254 | |
| 255 | def _setup_multiline_scenario(tmp_path) -> tuple[Path, Path, bytes, bytes, list[int]]: |
| 256 | """Shared setup for both multi-line torn-tail tests below. Returns |
| 257 | (data_dir, path, before, row_bytes, predicted_bug_offsets). |
| 258 | |
| 259 | predicted_bug_offsets is every offset whose fragment contains AT LEAST |
| 260 | ONE raw "\\n" byte of its own (i.e. every offset from right after the |
| 261 | row's FIRST embedded newline through its second-to-last byte) -- |
| 262 | empirically confirmed (see this file's own exploratory run) to be the |
| 263 | REAL trigger condition, which is measurably BROADER than "the fragment |
| 264 | itself ends with \\n": _drop_torn_tail's rfind(b'\\n') doesn't require |
| 265 | the crash to land exactly after a newline -- it back-scans from |
| 266 | WHEREVER the crash landed to the nearest earlier "\\n" and keeps |
| 267 | everything up to and including it. Once the fragment contains any |
| 268 | embedded newline at all, that back-scan finds it (rather than falling |
| 269 | all the way back to the end of the PRIOR row) and truncates there, |
| 270 | leaving a still-open-quote row ending in "\\n" -- at which point |
| 271 | csv.reader, given EOF while inside quotes, tolerantly treats it as the |
| 272 | field's own terminator instead of raising. So the true trigger surface |
| 273 | is not "crash lands right after a newline" but "crash lands ANYWHERE |
| 274 | once the row's value has emitted its first raw newline byte" -- for a |
| 275 | typical pretty-printed JSON cell, that is within the first handful of |
| 276 | bytes of the value, meaning nearly the ENTIRE remainder of the row's |
| 277 | write window is vulnerable, not just the instants immediately |
| 278 | following each newline. |
| 279 | """ |
| 280 | data_dir = tmp_path / "data" |
| 281 | write_append_schema(data_dir, "invoices", [ID_FIELD, LINES_FIELD]) |
| 282 | object_records.create_collection_record( |
| 283 | "invoices", {"id": "A", "lines": json.dumps([{"sku": "A-1"}])}, base_dir=data_dir, roots=[] |
| 284 | ) |
| 285 | object_records.create_collection_record( |
| 286 | "invoices", {"id": "B", "lines": json.dumps([{"sku": "B-1"}])}, base_dir=data_dir, roots=[] |
| 287 | ) |
| 288 | path = _records_path(data_dir, "invoices") |
| 289 | |
| 290 | before, row_bytes = _captured_next_row_bytes( |
| 291 | data_dir, "invoices", {"id": "C", "lines": _hostile_multiline_blob()} |
| 292 | ) |
| 293 | assert row_bytes.count(b"\n") > 3, "sanity: genuinely spans several physical lines" |
| 294 | assert row_bytes.endswith(b"\n") |
| 295 | |
| 296 | predicted_bug_offsets = [ |
| 297 | offset for offset in range(1, len(row_bytes)) |
| 298 | if b"\n" in row_bytes[:offset] |
| 299 | ] |
| 300 | assert predicted_bug_offsets, "sanity: the blob must contain internal newlines to test" |
| 301 | return data_dir, path, before, row_bytes, predicted_bug_offsets |
| 302 | |
| 303 | |
| 304 | def test_append_multiline_row_torn_at_every_byte_offset_maps_bug_2_surface(tmp_path): |
| 305 | """THE comprehensive sweep. For every byte offset of a real, captured, |
| 306 | multi-physical-line row: |
| 307 | |
| 308 | - offsets whose fragment contains NO raw "\\n" byte of its own (i.e. |
| 309 | the crash landed before the value ever emitted its first embedded |
| 310 | newline) are the substrate's unconditional promise -- the whole |
| 311 | fragment plus everything before it, once _drop_torn_tail's |
| 312 | rfind(b'\\n') runs, collapses back to exactly the last committed |
| 313 | row (B's), so these must ALWAYS recover cleanly (no data loss, no |
| 314 | phantom, reads agree, self-heals). Any failure here would be a NEW |
| 315 | finding, not bug #2, and is a hard assertion. |
| 316 | - offsets whose fragment DOES contain an embedded "\\n" are exactly |
| 317 | bug #2's real trigger surface (see _setup_multiline_scenario's |
| 318 | docstring for why this is the correct predictor -- broader than |
| 319 | "ends with \\n"). These are recorded into a matrix and reported, |
| 320 | but NOT hard-asserted here (see the dedicated xfail(strict=True) |
| 321 | test right below this one, which asserts the bug actually |
| 322 | reproduces at each of them; keeping that assertion out of THIS |
| 323 | test lets the full sweep run to completion and print a complete |
| 324 | matrix even though most of its rows are "expected bad"). |
| 325 | """ |
| 326 | data_dir, path, before, row_bytes, predicted_bug_offsets = _setup_multiline_scenario(tmp_path) |
| 327 | |
| 328 | unexpected: list[tuple[int, list[str], list[str]]] = [] |
| 329 | bug_confirmed: list[int] = [] |
| 330 | bug_did_not_reproduce: list[int] = [] |
| 331 | clean_count = 0 |
| 332 | |
| 333 | for offset in range(1, len(row_bytes)): # skip 0 (trivial no-op) and full length (not torn) |
| 334 | fragment = row_bytes[:offset] |
| 335 | has_internal_nl = b"\n" in fragment |
| 336 | path.write_bytes(before + fragment) |
| 337 | oidx_path = object_records._oidx_path(path) |
| 338 | if oidx_path.exists(): |
| 339 | oidx_path.unlink() |
| 340 | _clear_caches() |
| 341 | |
| 342 | listing = object_records.list_collection_records("invoices", base_dir=data_dir, roots=[])[ |
| 343 | "records" |
| 344 | ] |
| 345 | ids = [r["id"] for r in listing] |
| 346 | reads_agree = _fold_and_by_id_agree("invoices", data_dir, ["A", "B", "C"]) |
| 347 | clean_before_write = (ids == ["A", "B"]) and reads_agree |
| 348 | |
| 349 | object_records.create_collection_record( |
| 350 | "invoices", {"id": "D", "lines": json.dumps([{"sku": "D-1"}])}, |
| 351 | base_dir=data_dir, roots=[], |
| 352 | ) |
| 353 | _clear_caches() |
| 354 | listing_after = object_records.list_collection_records( |
| 355 | "invoices", base_dir=data_dir, roots=[] |
| 356 | )["records"] |
| 357 | ids_after = [r["id"] for r in listing_after] |
| 358 | d_landed_clean = ids_after == ["A", "B", "D"] |
| 359 | |
| 360 | fully_clean = clean_before_write and d_landed_clean |
| 361 | |
| 362 | if fully_clean: |
| 363 | clean_count += 1 |
| 364 | if has_internal_nl: |
| 365 | bug_did_not_reproduce.append(offset) |
| 366 | elif has_internal_nl: |
| 367 | bug_confirmed.append(offset) |
| 368 | else: |
| 369 | unexpected.append((offset, ids, ids_after)) |
| 370 | |
| 371 | print(f"\n[torn-tail matrix] multiline row: {len(row_bytes)} bytes total, " |
| 372 | f"{len(predicted_bug_offsets)} / {len(row_bytes) - 1} offsets predicted vulnerable " |
| 373 | f"(fragment contains an embedded newline)") |
| 374 | print(f"[torn-tail matrix] clean-recovery offsets: {clean_count} / {len(row_bytes) - 1}") |
| 375 | print(f"[torn-tail matrix] bug #2 CONFIRMED (resurrection/cascade) at {len(bug_confirmed)} " |
| 376 | f"offsets: {bug_confirmed}") |
| 377 | if bug_did_not_reproduce: |
| 378 | print(f"[torn-tail matrix] predicted-vulnerable offsets that recovered cleanly anyway: " |
| 379 | f"{bug_did_not_reproduce}") |
| 380 | |
| 381 | assert not unexpected, ( |
| 382 | "NEW FINDING (outside bug #2's known quote-blind surface): a torn " |
| 383 | "fragment with NO embedded newline of its own -- one the documented " |
| 384 | "torn-tail rule unconditionally promises to collapse back to the last " |
| 385 | f"committed row -- failed to recover cleanly. offset/ids/ids_after: {unexpected!r}" |
| 386 | ) |
| 387 | # Every offset that reproduced the bug is, exactly, one whose fragment |
| 388 | # contains an embedded "\n" -- confirming the (broader-than-originally- |
| 389 | # documented) trigger condition precisely. |
| 390 | assert set(bug_confirmed) <= set(predicted_bug_offsets) |
| 391 | |
| 392 | |
| 393 | def test_append_multiline_row_torn_at_every_offset_recovers_bug_2_regression(tmp_path): |
| 394 | """Bug #2's write side, FIXED -- this is the regression guard the old |
| 395 | strict xfail said it would become. |
| 396 | |
| 397 | History, because the shape of this test is the evidence: the xfail it |
| 398 | replaces asserted the bug REPRODUCED at every offset -- a phantom C on |
| 399 | a bare read (effect 1) and the next write D swallowed into C's open |
| 400 | quote (effect 2). The read-side fix (2026-07-20) killed effect 1, so |
| 401 | the xfail could never flip to XPASS however the write side fared: it |
| 402 | kept "failing" on an assertion about a bug that was already half gone. |
| 403 | A strict xfail that bundles two effects can only flip when BOTH flip. |
| 404 | |
| 405 | Now, at EVERY offset from right after the row's first embedded newline |
| 406 | through its last byte (the full trigger surface the characterization |
| 407 | mapped -- see _setup_multiline_scenario): |
| 408 | |
| 409 | 1. a bare read shows exactly [A, B] -- no phantom C (read-side fix); |
| 410 | 2. the next write's repair truncates the torn fragment QUOTE-AWARELY, |
| 411 | so D lands as its own row instead of vanishing into C's unclosed |
| 412 | quote (write-side fix: _repair_torn_tail via committed_prefix_len); |
| 413 | 3. a fresh parse afterwards shows [A, B, D] with D's payload intact. |
| 414 | |
| 415 | The caches are cleared each iteration, so the repair runs COLD -- no |
| 416 | anchor, full scan -- which is the crash case, and the case the |
| 417 | reverted first fix got wrong by delegating to a warm cache. |
| 418 | """ |
| 419 | data_dir, path, before, row_bytes, predicted_bug_offsets = _setup_multiline_scenario(tmp_path) |
| 420 | |
| 421 | for offset in predicted_bug_offsets: |
| 422 | fragment = row_bytes[:offset] |
| 423 | assert b"\n" in fragment |
| 424 | path.write_bytes(before + fragment) |
| 425 | oidx_path = object_records._oidx_path(path) |
| 426 | if oidx_path.exists(): |
| 427 | oidx_path.unlink() |
| 428 | _clear_caches() |
| 429 | |
| 430 | listing = object_records.list_collection_records( |
| 431 | "invoices", base_dir=data_dir, roots=[] |
| 432 | )["records"] |
| 433 | ids = [r["id"] for r in listing] |
| 434 | assert ids == ["A", "B"], ( |
| 435 | f"offset={offset}: a bare read must drop the torn C, got {ids!r}" |
| 436 | ) |
| 437 | |
| 438 | object_records.create_collection_record( |
| 439 | "invoices", {"id": "D", "lines": json.dumps([{"sku": "D-1"}])}, |
| 440 | base_dir=data_dir, roots=[], |
| 441 | ) |
| 442 | _clear_caches() |
| 443 | listing_after = object_records.list_collection_records( |
| 444 | "invoices", base_dir=data_dir, roots=[] |
| 445 | )["records"] |
| 446 | by_id = {r["id"]: r for r in listing_after} |
| 447 | assert sorted(by_id) == ["A", "B", "D"], ( |
| 448 | f"offset={offset}: D must land as its own row after repair; " |
| 449 | f"got {sorted(by_id)!r}" |
| 450 | ) |
| 451 | assert by_id["D"]["lines"] == json.dumps([{"sku": "D-1"}]), ( |
| 452 | f"offset={offset}: D's payload must survive intact" |
| 453 | ) |
| 454 | |
| 455 | |
| 456 | # ============================================================================= |
| 457 | # 3. APPEND MODE -- torn oidx SIDECAR (`.records.oidx`), every byte offset |
| 458 | # ============================================================================= |
| 459 | |
| 460 | |
| 461 | def test_append_oidx_sidecar_torn_at_every_byte_offset_self_heals(tmp_path, monkeypatch): |
| 462 | """The sidecar's own line format (`{start}\\t{end}\\t{op}\\t{id}\\n`) can |
| 463 | never itself contain a raw newline mid-line -- ids are restricted to |
| 464 | _RECORD_ID_RE and the other three columns are plain digits/op-literals |
| 465 | -- so, unlike records.tsv, there is no quote-blindness surface here at |
| 466 | all: "ends with \\n" is a SOUND completeness check for every physical |
| 467 | line of this file, not just an approximation. The sidecar is also, by |
| 468 | its own design ("never required for correctness" -- see _load_oidx's |
| 469 | docstring), never the sole source of truth: any torn/missing tail is |
| 470 | expected to be caught up from records.tsv itself. This sweeps every |
| 471 | byte offset of one real captured idx line and confirms that promise: |
| 472 | the collection's data is untouched (records.tsv is never even written |
| 473 | to it by this scenario) and every id resolves correctly regardless of |
| 474 | how badly mangled the sidecar's own tail is. |
| 475 | """ |
| 476 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 477 | data_dir = tmp_path / "data" |
| 478 | write_append_schema(data_dir, "widgets", [ID_FIELD, VALUE_FIELD]) |
| 479 | path = _records_path(data_dir, "widgets") |
| 480 | oidx_path = object_records._oidx_path(path) |
| 481 | |
| 482 | object_records.create_collection_record( |
| 483 | "widgets", {"id": "w1", "value": "Alpha"}, base_dir=data_dir, roots=[] |
| 484 | ) |
| 485 | object_records.create_collection_record( |
| 486 | "widgets", {"id": "w2", "value": "Beta"}, base_dir=data_dir, roots=[] |
| 487 | ) |
| 488 | # Force the sidecar to exist and be warm/caught-up through w2. |
| 489 | object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[]) |
| 490 | assert oidx_path.exists() |
| 491 | oidx_before = oidx_path.read_bytes() |
| 492 | records_before = path.read_bytes() |
| 493 | |
| 494 | object_records.create_collection_record( |
| 495 | "widgets", {"id": "w3", "value": "Gamma"}, base_dir=data_dir, roots=[] |
| 496 | ) |
| 497 | oidx_after = oidx_path.read_bytes() |
| 498 | records_after = path.read_bytes() |
| 499 | assert oidx_after.startswith(oidx_before) |
| 500 | assert records_after.startswith(records_before) |
| 501 | oidx_row_bytes = oidx_after[len(oidx_before):] |
| 502 | assert oidx_row_bytes.count(b"\n") == 1 |
| 503 | |
| 504 | # records.tsv legitimately, fully committed w3's row -- only the SIDECAR |
| 505 | # is torn in this scenario (a crash between the data append and the |
| 506 | # sidecar catching up to it). |
| 507 | for offset in range(0, len(oidx_row_bytes) + 1): # every prefix, 0 .. full line |
| 508 | oidx_path.write_bytes(oidx_before + oidx_row_bytes[:offset]) |
| 509 | path.write_bytes(records_after) # data file: always fully committed |
| 510 | object_records._OIDX_CACHE.clear() |
| 511 | object_records._RECORDS_CACHE.clear() |
| 512 | |
| 513 | id_offsets, coherent = object_records._load_oidx(path) |
| 514 | assert coherent, f"oidx offset={offset}: catch-up scan must reach full coherence" |
| 515 | assert set(id_offsets) == {"w1", "w2", "w3"}, ( |
| 516 | f"oidx offset={offset}: catch-up must recover all three ids; got {set(id_offsets)}" |
| 517 | ) |
| 518 | |
| 519 | object_records._OIDX_CACHE.clear() |
| 520 | object_records._RECORDS_CACHE.clear() |
| 521 | for record_id, expected_value in (("w1", "Alpha"), ("w2", "Beta"), ("w3", "Gamma")): |
| 522 | got = object_records.get_collection_record( |
| 523 | "widgets", record_id, base_dir=data_dir, roots=[] |
| 524 | ) |
| 525 | assert got == {"id": record_id, "value": expected_value}, ( |
| 526 | f"oidx offset={offset}: {record_id} did not resolve correctly; got {got!r}" |
| 527 | ) |
| 528 | |
| 529 | |
| 530 | # ============================================================================= |
| 531 | # 4. APPEND MODE -- torn HEADER (interrupted very first write) |
| 532 | # ============================================================================= |
| 533 | |
| 534 | |
| 535 | def test_append_torn_header_every_byte_offset_characterization(tmp_path): |
| 536 | """The header line is written ONLY as part of a full-rewrite (temp file + |
| 537 | atomic rename) -- the transition-in write for a collection's very first |
| 538 | record, per _persist_write's TRANSITION-IN branch -- and every full |
| 539 | rewrite in this module goes through _write_collection_records's |
| 540 | temp+replace. Because of that, a torn header is NOT actually reachable |
| 541 | via any real crash in this module's own write path: readers can only |
| 542 | ever observe the OLD file (rename never happened) or the NEW complete |
| 543 | file (rename happened), never a half-written header -- see section 6/7 |
| 544 | below for that atomicity property tested directly. |
| 545 | |
| 546 | This test instead treats the header as a hostile/hand-corrupted file |
| 547 | (e.g. a foreign process, a disk-level bit-rot, a manual edit, a copy |
| 548 | caught mid-write) and characterizes whatever the module actually does -- |
| 549 | a SYNTHETIC scenario, documented as such, not a mapped crash-reachable |
| 550 | surface. The result is NOT the simple "always self-heals" this test |
| 551 | originally assumed. Sweeping every byte offset of the 13-byte header |
| 552 | "_op\\tid\\tvalue\\n" reveals several distinct outcomes depending on |
| 553 | exactly where the cut lands: |
| 554 | |
| 555 | - offset 0 (a genuinely empty existing file): both read and write |
| 556 | raise an uncaught `ValueError: ... is missing a header` -- |
| 557 | _cache_entry's "file doesn't exist -> empty collection" special |
| 558 | case is NOT extended to "file exists but is empty", so an empty |
| 559 | records.tsv is treated as corrupt rather than as an empty |
| 560 | collection. No self-heal at all. |
| 561 | - offsets that cut inside "_op" or leave a header missing an "id" |
| 562 | column (1, 2, 3, 5) or that produce an empty trailing field name |
| 563 | (4, 7): both read and write raise -- again no self-heal, but at |
| 564 | least loud/consistent about it on every call. |
| 565 | - offset 6 ("_op\\tid", missing only "\\tvalue"): read succeeds |
| 566 | (empty collection) and write CORRECTLY self-heals via the |
| 567 | new-field-fallback full rewrite, producing a clean 3-column file. |
| 568 | - offsets 8-11 (a partial "value" column name -- "v", "va", "val", |
| 569 | "valu"): read succeeds (empty collection), but write "self-heals" |
| 570 | into a PERMANENTLY WRONG SCHEMA: it treats the partial name as a |
| 571 | genuine, distinct existing column and adds the real "value" column |
| 572 | alongside it, leaving every future row with a silently-injected |
| 573 | extra empty field (e.g. {"id": "w1", "v": "", "value": "Alpha"}). |
| 574 | Not a crash -- worse, in a sense: a quiet, permanent schema |
| 575 | pollution that round-trips forever afterward. |
| 576 | - offset 12 (the full, correct 3-column header content, missing ONLY |
| 577 | its own trailing "\\n"): read alone succeeds (empty collection). |
| 578 | CREATE used to be the worst outcome in the sweep -- the old repair, |
| 579 | finding no newline anywhere, hit its truncate-whole-file fallback, |
| 580 | destroyed the intact header, appended a bare headerless row, and |
| 581 | REPORTED SUCCESS, leaving every subsequent read raising an uncaught |
| 582 | ValueError. FIXED: the quote-aware repair still (correctly) treats |
| 583 | a never-terminated header as uncommitted, but _append_records_rows |
| 584 | now restores the header from physical_fields whenever it finds the |
| 585 | file empty, so the append lands in a well-formed file and the next |
| 586 | read returns the row. |
| 587 | |
| 588 | This test was a strict xfail while that offset-12 silent corruption |
| 589 | existed; its terminal assertion IS the fixed-world invariant (a |
| 590 | reported success must never leave the collection unreadable), so the |
| 591 | fix simply removed the decorator. The full per-offset matrix is still |
| 592 | printed for the record. |
| 593 | """ |
| 594 | header_bytes = b"_op\tid\tvalue\n" |
| 595 | matrix: list[tuple[int, str, str, str]] = [] |
| 596 | |
| 597 | for offset in range(0, len(header_bytes)): # every strict prefix of the header line itself |
| 598 | data_dir = tmp_path / f"case_{offset}" |
| 599 | write_append_schema(data_dir, "widgets", [ID_FIELD, VALUE_FIELD]) |
| 600 | path = _records_path(data_dir, "widgets") |
| 601 | path.parent.mkdir(parents=True, exist_ok=True) |
| 602 | path.write_bytes(header_bytes[:offset]) |
| 603 | _clear_caches() |
| 604 | |
| 605 | list_outcome = "ok" |
| 606 | try: |
| 607 | listing = object_records.list_collection_records( |
| 608 | "widgets", base_dir=data_dir, roots=[] |
| 609 | )["records"] |
| 610 | list_outcome = "ok-empty" if listing == [] else f"ok-nonempty:{listing!r}" |
| 611 | except Exception as exc: # noqa: BLE001 -- characterizing whatever happens |
| 612 | list_outcome = f"{type(exc).__name__}: {exc}" |
| 613 | |
| 614 | _clear_caches() |
| 615 | create_outcome = "ok" |
| 616 | create_raised = False |
| 617 | try: |
| 618 | object_records.create_collection_record( |
| 619 | "widgets", {"id": "w1", "value": "Alpha"}, base_dir=data_dir, roots=[] |
| 620 | ) |
| 621 | except Exception as exc: # noqa: BLE001 |
| 622 | create_outcome = f"{type(exc).__name__}: {exc}" |
| 623 | create_raised = True |
| 624 | |
| 625 | _clear_caches() |
| 626 | post_create_read_outcome = "n/a (create raised)" |
| 627 | if not create_raised: |
| 628 | try: |
| 629 | listing_after = object_records.list_collection_records( |
| 630 | "widgets", base_dir=data_dir, roots=[] |
| 631 | )["records"] |
| 632 | post_create_read_outcome = f"ok:{listing_after!r}" |
| 633 | except Exception as exc: # noqa: BLE001 |
| 634 | post_create_read_outcome = f"{type(exc).__name__}: {exc}" |
| 635 | |
| 636 | matrix.append((offset, list_outcome, create_outcome, post_create_read_outcome)) |
| 637 | |
| 638 | print("\n[torn-header matrix] offset -> (list, create, read-immediately-after-create):") |
| 639 | for row in matrix: |
| 640 | print(f" {row}") |
| 641 | |
| 642 | # The one invariant that should hold regardless of how a corrupt/torn |
| 643 | # header is otherwise handled: create_collection_record must never |
| 644 | # report SUCCESS (no exception) while silently leaving the collection |
| 645 | # unreadable on the very next read -- that is strictly worse than a loud |
| 646 | # failure, since the caller believes their write succeeded. |
| 647 | silent_corruptions = [ |
| 648 | (offset, post_read) |
| 649 | for offset, _list, create, post_read in matrix |
| 650 | if create == "ok" and not post_read.startswith("ok") |
| 651 | ] |
| 652 | assert not silent_corruptions, ( |
| 653 | "FINDING: create_collection_record reported success (raised nothing) " |
| 654 | "while silently corrupting records.tsv such that the VERY NEXT read " |
| 655 | "fails. A caller sees their write as having succeeded with no error, " |
| 656 | "yet the whole collection becomes unreadable immediately afterward. " |
| 657 | f"(offset, read-immediately-after-create outcome): {silent_corruptions!r}" |
| 658 | ) |
| 659 | |
| 660 | |
| 661 | # ============================================================================= |
| 662 | # 5. APPEND MODE -- interrupted COMPACTION |
| 663 | # ============================================================================= |
| 664 | |
| 665 | |
| 666 | def test_append_interrupted_compaction_leaves_pre_compaction_data_intact(tmp_path): |
| 667 | """compact_collection rewrites via the exact same atomic temp+replace |
| 668 | path as any other full rewrite (_write_collection_records). A crash |
| 669 | strictly before the final `Path.replace` therefore can only ever leave |
| 670 | the PRE-compaction file (complete, with its dead rows still physically |
| 671 | present but harmlessly so -- they're already excluded by folding) |
| 672 | untouched, plus an orphaned temp file no read path looks at. This |
| 673 | demonstrates that guarantee directly: a stray, correctly-named temp file |
| 674 | holding a plausible (but never-renamed) compacted body sits alongside |
| 675 | the real file, and every read/write must behave exactly as if it were |
| 676 | never there -- then a REAL compaction afterward must still succeed |
| 677 | despite the leftover stray file. |
| 678 | """ |
| 679 | data_dir = tmp_path / "data" |
| 680 | write_append_schema(data_dir, "events", [ID_FIELD, VALUE_FIELD]) |
| 681 | for i in range(5): |
| 682 | object_records.create_collection_record( |
| 683 | "events", {"id": f"e{i}", "value": f"v{i}"}, base_dir=data_dir, roots=[] |
| 684 | ) |
| 685 | # Create dead rows (superseded physical rows) so compaction has real work. |
| 686 | object_records.update_collection_record( |
| 687 | "events", "e0", {"value": "v0-updated"}, base_dir=data_dir, roots=[] |
| 688 | ) |
| 689 | object_records.delete_collection_record("events", "e1", base_dir=data_dir, roots=[]) |
| 690 | |
| 691 | path = _records_path(data_dir, "events") |
| 692 | pre_compaction_bytes = path.read_bytes() |
| 693 | |
| 694 | stray = _stray_tmp_path(path) |
| 695 | stray.write_bytes(b"_op\tid\tvalue\n\te0\tv0-would-be-compacted\n") # plausible, never renamed |
| 696 | try: |
| 697 | _clear_caches() |
| 698 | listing = object_records.list_collection_records("events", base_dir=data_dir, roots=[])[ |
| 699 | "records" |
| 700 | ] |
| 701 | assert listing == [ |
| 702 | {"id": "e0", "value": "v0-updated"}, |
| 703 | {"id": "e2", "value": "v2"}, |
| 704 | {"id": "e3", "value": "v3"}, |
| 705 | {"id": "e4", "value": "v4"}, |
| 706 | ], f"a stray uncommitted temp file must never be read; got {listing!r}" |
| 707 | assert path.read_bytes() == pre_compaction_bytes, ( |
| 708 | "a bare read must never touch/rewrite records.tsv" |
| 709 | ) |
| 710 | |
| 711 | # Write activity must also be unaffected by the stray file's presence. |
| 712 | object_records.create_collection_record( |
| 713 | "events", {"id": "e5", "value": "v5"}, base_dir=data_dir, roots=[] |
| 714 | ) |
| 715 | _clear_caches() |
| 716 | assert stray.exists(), "the real write must not disturb an unrelated stray temp file" |
| 717 | |
| 718 | # A REAL compaction, run with the stray file still present, must |
| 719 | # succeed and produce correct live content. |
| 720 | result = object_records.compact_collection("events", base_dir=data_dir, roots=[]) |
| 721 | assert result["rows_after"] < result["rows_before"] |
| 722 | _clear_caches() |
| 723 | listing_after = object_records.list_collection_records( |
| 724 | "events", base_dir=data_dir, roots=[] |
| 725 | )["records"] |
| 726 | assert listing_after == [ |
| 727 | {"id": "e0", "value": "v0-updated"}, |
| 728 | {"id": "e2", "value": "v2"}, |
| 729 | {"id": "e3", "value": "v3"}, |
| 730 | {"id": "e4", "value": "v4"}, |
| 731 | {"id": "e5", "value": "v5"}, |
| 732 | ] |
| 733 | finally: |
| 734 | if stray.exists(): |
| 735 | stray.unlink() |
| 736 | |
| 737 | |
| 738 | # ============================================================================= |
| 739 | # 6/7. CLASSIC MODE -- atomic-rename durability (crash between temp-write and |
| 740 | # rename), and the same guarantee re-confirmed for an APPEND-mode |
| 741 | # transition-in / new-field full rewrite |
| 742 | # ============================================================================= |
| 743 | |
| 744 | |
| 745 | def test_classic_mode_crash_between_temp_write_and_rename_serves_last_good_version(tmp_path): |
| 746 | """Classic mode never appends in place -- every write is a full rewrite |
| 747 | via temp file + atomic `Path.replace`. Simulate a crash strictly before |
| 748 | that replace: leave a real, correctly-named, fully-written temp file |
| 749 | (containing what WOULD have become the next version) sitting next to |
| 750 | the real file, which still holds the last version that actually |
| 751 | finished. Every read must serve exactly that last good version, never a |
| 752 | mix, and the temp file must never be picked up as real data. |
| 753 | """ |
| 754 | data_dir = tmp_path / "data" |
| 755 | write_schema(data_dir, "widgets", [ID_FIELD, VALUE_FIELD]) |
| 756 | object_records.create_collection_record( |
| 757 | "widgets", {"id": "w1", "value": "Alpha"}, base_dir=data_dir, roots=[] |
| 758 | ) |
| 759 | path = _records_path(data_dir, "widgets") |
| 760 | last_good_bytes = path.read_bytes() |
| 761 | |
| 762 | # Simulate: an update to w1 got as far as fully writing its temp file, |
| 763 | # then the process died before `temp_path.replace(path)` ran. |
| 764 | stray = _stray_tmp_path(path) |
| 765 | stray.write_text("id\tvalue\nw1\tAlpha-WOULD-HAVE-BEEN-UPDATED\n") |
| 766 | try: |
| 767 | object_records._RECORDS_CACHE.clear() |
| 768 | listing = object_records.list_collection_records("widgets", base_dir=data_dir, roots=[])[ |
| 769 | "records" |
| 770 | ] |
| 771 | assert listing == [{"id": "w1", "value": "Alpha"}], ( |
| 772 | f"must serve only the last COMMITTED version; got {listing!r}" |
| 773 | ) |
| 774 | assert path.read_bytes() == last_good_bytes |
| 775 | |
| 776 | object_records._RECORDS_CACHE.clear() |
| 777 | fetched = object_records.get_collection_record( |
| 778 | "widgets", "w1", base_dir=data_dir, roots=[] |
| 779 | ) |
| 780 | assert fetched == {"id": "w1", "value": "Alpha"} |
| 781 | |
| 782 | # A real write, with the stray temp file still present, must succeed |
| 783 | # normally and not be confused by it. |
| 784 | object_records.update_collection_record( |
| 785 | "widgets", "w1", {"value": "Alpha2"}, base_dir=data_dir, roots=[] |
| 786 | ) |
| 787 | object_records._RECORDS_CACHE.clear() |
| 788 | listing_after = object_records.list_collection_records( |
| 789 | "widgets", base_dir=data_dir, roots=[] |
| 790 | )["records"] |
| 791 | assert listing_after == [{"id": "w1", "value": "Alpha2"}] |
| 792 | finally: |
| 793 | if stray.exists(): |
| 794 | stray.unlink() |
| 795 | |
| 796 | |
| 797 | def test_append_mode_transition_in_crash_between_temp_write_and_rename(tmp_path): |
| 798 | """The append-mode analogue of the classic-mode test above: a brand-new |
| 799 | append-mode collection's very first write is a TRANSITION-IN full |
| 800 | rewrite (adds the `_op` header column), which is the same |
| 801 | _write_collection_records temp+replace path. Before that first write |
| 802 | ever completes, records.tsv does not exist at all -- a crash leaves |
| 803 | only an orphaned temp file, and the collection must still read back as |
| 804 | genuinely empty (not error, not phantom data) and a subsequent real |
| 805 | write must still succeed and transition in normally. |
| 806 | """ |
| 807 | data_dir = tmp_path / "data" |
| 808 | write_append_schema(data_dir, "widgets", [ID_FIELD, VALUE_FIELD]) |
| 809 | path = _records_path(data_dir, "widgets") |
| 810 | assert not path.exists() |
| 811 | |
| 812 | stray = _stray_tmp_path(path) |
| 813 | stray.parent.mkdir(parents=True, exist_ok=True) |
| 814 | stray.write_text("_op\tid\tvalue\n\tw1\tWould-have-been-committed\n") |
| 815 | try: |
| 816 | listing = object_records.list_collection_records("widgets", base_dir=data_dir, roots=[])[ |
| 817 | "records" |
| 818 | ] |
| 819 | assert listing == [], f"a never-renamed temp file must not surface as data; got {listing!r}" |
| 820 | assert not path.exists(), "a bare read must not materialize records.tsv from the stray file" |
| 821 | |
| 822 | object_records.create_collection_record( |
| 823 | "widgets", {"id": "w1", "value": "Alpha"}, base_dir=data_dir, roots=[] |
| 824 | ) |
| 825 | object_records._RECORDS_CACHE.clear() |
| 826 | listing_after = object_records.list_collection_records( |
| 827 | "widgets", base_dir=data_dir, roots=[] |
| 828 | )["records"] |
| 829 | assert listing_after == [{"id": "w1", "value": "Alpha"}] |
| 830 | assert path.exists() |
| 831 | finally: |
| 832 | if stray.exists(): |
| 833 | stray.unlink() |