askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_durability_atomicity_characterization.py · 30.9 KB · 655 lines
| 1 | """CHARACTERIZATION tests: does object_records.py give a DATABASE-grade |
| 2 | ATOMICITY guarantee for a single write, and is a concurrent reader ever |
| 3 | able to observe a torn/half-written intermediate state? |
| 4 | |
| 5 | A database must be able to answer: "a write either fully lands or not at |
| 6 | all, and readers never see a torn intermediate." This file probes exactly |
| 7 | that question against both storage modes: |
| 8 | |
| 9 | - CLASSIC mode: every write (_write_collection_records) writes a whole |
| 10 | new file to a temp path IN THE SAME DIRECTORY, then calls |
| 11 | `temp_path.replace(path)` -- a POSIX atomic rename. A reader that |
| 12 | already has the file open keeps its old (complete) view; a reader |
| 13 | that opens fresh after the rename gets the new (complete) view. |
| 14 | Never a mix. |
| 15 | - APPEND mode: a write (_append_records_rows) appends new physical |
| 16 | row(s) to the SAME inode, then (_persist_write's fast-append branch) |
| 17 | updates the id->offset sidecar (`.records.oidx`) as a SEPARATE, |
| 18 | later step. The append itself is a single buffered `handle.write()` + |
| 19 | one `flush()` at the end of a "a" open -- the row's bytes are either |
| 20 | fully written or (torn-tail case, covered by |
| 21 | tests/test_object_records.py's own torn-tail suite) not, but the |
| 22 | sidecar update is a genuinely distinct write that can be skipped, |
| 23 | fail, or (in a real crash) simply never happen. |
| 24 | |
| 25 | This is pure characterization of EXISTING behavior. It does not modify |
| 26 | any production module. No test here is written to a production-code |
| 27 | signature that doesn't exist yet, and no failing assertion is "fixed" by |
| 28 | weakening it -- a violation found here is left failing, wrapped in |
| 29 | `@pytest.mark.xfail(strict=True, ...)`, and documented, exactly like |
| 30 | tests/test_embedded_json_lines_characterization.py's torn-tail FINDING. |
| 31 | One such violation was found here (a non-UTF8-corrupted sidecar crashes a |
| 32 | read instead of self-healing); every other guarantee probed below HOLDS. |
| 33 | See each test's docstring and the summary at the bottom of this file. |
| 34 | """ |
| 35 | |
| 36 | from __future__ import annotations |
| 37 | |
| 38 | import json |
| 39 | import threading |
| 40 | from pathlib import Path |
| 41 | |
| 42 | import pytest |
| 43 | |
| 44 | import object_records |
| 45 | |
| 46 | pytestmark = pytest.mark.conformance |
| 47 | |
| 48 | |
| 49 | ID_FIELD = {"name": "id"} |
| 50 | VALUE_FIELD = {"name": "value"} |
| 51 | |
| 52 | |
| 53 | # --- setup helpers (mirror tests/test_object_records.py and |
| 54 | # tests/test_embedded_json_lines_characterization.py's conventions) -------- |
| 55 | |
| 56 | |
| 57 | def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 58 | path = data_dir / "schemas" / f"{collection}.json" |
| 59 | path.parent.mkdir(parents=True, exist_ok=True) |
| 60 | path.write_text(json.dumps({"fields": fields})) |
| 61 | return path |
| 62 | |
| 63 | |
| 64 | def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 65 | path = data_dir / "schemas" / f"{collection}.json" |
| 66 | path.parent.mkdir(parents=True, exist_ok=True) |
| 67 | path.write_text(json.dumps({"fields": fields, "storage": "append"})) |
| 68 | return path |
| 69 | |
| 70 | |
| 71 | def _clear_caches() -> None: |
| 72 | """Force the NEXT read past the warm in-process caches -- see the |
| 73 | identical helper in test_embedded_json_lines_characterization.py.""" |
| 74 | object_records._RECORDS_CACHE.clear() |
| 75 | object_records._OIDX_CACHE.clear() |
| 76 | |
| 77 | |
| 78 | # ============================================================================= |
| 79 | # 1. ATOMIC RENAME (classic mode) -- held-fd technique |
| 80 | # ============================================================================= |
| 81 | |
| 82 | |
| 83 | def test_classic_write_held_reader_fd_sees_whole_old_or_whole_new_never_torn(tmp_path): |
| 84 | """GUARANTEE: HOLDS. Mirrors |
| 85 | tests/test_object_records.py::test_classic_write_keeps_an_open_readers_fd_on_a_complete_version |
| 86 | -- the "Q2 property" -- generalized here to explicitly state the |
| 87 | atomicity finding this file exists to pin down: a reader fd opened |
| 88 | BEFORE a classic-mode write (create/update/delete, all routed through |
| 89 | _write_collection_records's temp-file + Path.replace) reads the |
| 90 | COMPLETE pre-write file byte-for-byte, even after the write has fully |
| 91 | completed; a fresh read after the write sees the COMPLETE post-write |
| 92 | file. There is no interleaving in which a reader observes a partial |
| 93 | mix of old and new bytes, because the atomic rename swaps a directory |
| 94 | entry to a new inode instantaneously -- the held fd's inode is |
| 95 | unaffected and was always a complete, valid file.""" |
| 96 | data_dir = tmp_path / "data" |
| 97 | write_schema(data_dir, "ctr", [ID_FIELD, VALUE_FIELD]) |
| 98 | path = object_records.collection_records_file("ctr", base_dir=data_dir) |
| 99 | object_records.create_collection_record("ctr", {"id": "c1", "value": "0"}, base_dir=data_dir, roots=[]) |
| 100 | pre_write_bytes = path.read_bytes() |
| 101 | |
| 102 | with path.open("rb") as held: |
| 103 | # Sanity: the held fd can already read the pre-write content. |
| 104 | assert held.read() == pre_write_bytes |
| 105 | held.seek(0) |
| 106 | |
| 107 | object_records.update_collection_record( |
| 108 | "ctr", "c1", {"value": "1"}, base_dir=data_dir, roots=[] |
| 109 | ) |
| 110 | |
| 111 | # The write is fully complete on disk by the time update_collection_record |
| 112 | # returns (it is a synchronous call) -- yet the held fd, unaffected by |
| 113 | # the rename, still reads the WHOLE old version, not a mix. |
| 114 | assert held.read() == pre_write_bytes |
| 115 | |
| 116 | # A fresh reader (opening the new directory entry) sees the WHOLE new |
| 117 | # version. |
| 118 | assert path.read_bytes() == b"id\tvalue\nc1\t1\n" |
| 119 | |
| 120 | |
| 121 | # ============================================================================= |
| 122 | # 2. ATOMIC RENAME (classic mode) -- concurrent thread, raw-byte level |
| 123 | # ============================================================================= |
| 124 | |
| 125 | |
| 126 | def test_classic_atomic_rename_reader_thread_raw_bytes_always_one_complete_snapshot(tmp_path): |
| 127 | """GUARANTEE: HOLDS. Purest form of the atomic-rename probe: a reader |
| 128 | thread repeatedly reads the RAW BYTES of records.tsv (bypassing |
| 129 | object_records' own CSV parsing entirely -- so this can't be |
| 130 | accidentally masked by a forgiving parser) while a writer thread |
| 131 | concurrently performs N updates, each one a full temp-file + rename |
| 132 | replace. Every single raw read observed by the reader must be |
| 133 | EXACTLY one of the N+1 known-good complete snapshots -- never a |
| 134 | truncated, duplicated, or spliced mixture of two versions.""" |
| 135 | data_dir = tmp_path / "data" |
| 136 | write_schema(data_dir, "ctr", [ID_FIELD, VALUE_FIELD]) |
| 137 | object_records.create_collection_record("ctr", {"id": "c1", "value": "0"}, base_dir=data_dir, roots=[]) |
| 138 | path = object_records.collection_records_file("ctr", base_dir=data_dir) |
| 139 | |
| 140 | n = 300 |
| 141 | known_good_snapshots = {f"id\tvalue\nc1\t{i}\n" for i in range(n)} |
| 142 | |
| 143 | stop = threading.Event() |
| 144 | anomalies: list[str] = [] |
| 145 | reads_done = [0] |
| 146 | |
| 147 | def reader() -> None: |
| 148 | while not stop.is_set(): |
| 149 | try: |
| 150 | raw = path.read_text() |
| 151 | except FileNotFoundError: |
| 152 | # Should never happen (rename always leaves a directory |
| 153 | # entry) -- if it does, that IS a torn-intermediate finding. |
| 154 | anomalies.append("<FileNotFoundError: no directory entry momentarily>") |
| 155 | continue |
| 156 | reads_done[0] += 1 |
| 157 | if raw not in known_good_snapshots: |
| 158 | anomalies.append(raw) |
| 159 | |
| 160 | reader_thread = threading.Thread(target=reader) |
| 161 | reader_thread.start() |
| 162 | try: |
| 163 | for i in range(n): |
| 164 | object_records.update_collection_record( |
| 165 | "ctr", "c1", {"value": str(i)}, base_dir=data_dir, roots=[] |
| 166 | ) |
| 167 | finally: |
| 168 | stop.set() |
| 169 | reader_thread.join(timeout=15) |
| 170 | |
| 171 | assert not reader_thread.is_alive(), "reader thread failed to stop" |
| 172 | assert reads_done[0] > 0, "reader thread never got a chance to run concurrently" |
| 173 | assert anomalies == [], ( |
| 174 | f"raw bytes read were NOT a complete known-good snapshot in " |
| 175 | f"{len(anomalies)} case(s) out of {reads_done[0]} reads; first " |
| 176 | f"offending read: {anomalies[0]!r}" |
| 177 | ) |
| 178 | |
| 179 | |
| 180 | # ============================================================================= |
| 181 | # 3. os.replace SAME-FILESYSTEM CONFIRMATION |
| 182 | # ============================================================================= |
| 183 | |
| 184 | |
| 185 | def test_classic_temp_file_is_created_in_the_same_directory_as_the_target(tmp_path, monkeypatch): |
| 186 | """GUARANTEE: HOLDS. `os.replace`/`Path.replace` is only atomic when |
| 187 | source and destination are on the SAME filesystem; across filesystems |
| 188 | the OS (or Python) must fall back to copy+delete, which is NOT atomic |
| 189 | and reintroduces exactly the torn-intermediate risk this whole file |
| 190 | is probing for. This test spies on Path.replace (restored automatically |
| 191 | by monkeypatch at teardown) to confirm, for a REAL write through the |
| 192 | public API, that the temp file _write_collection_records creates |
| 193 | lives in the exact same directory as -- and therefore is |
| 194 | guaranteed to share a device/filesystem with -- the target |
| 195 | records.tsv it replaces. (Source-level confirmation: object_records.py's |
| 196 | _write_collection_records builds `temp_path` via |
| 197 | `path.with_name(...)`, which by construction keeps the same parent |
| 198 | directory; this test verifies that holds for a real end-to-end call, |
| 199 | not just by reading the source.)""" |
| 200 | data_dir = tmp_path / "data" |
| 201 | write_schema(data_dir, "ctr", [ID_FIELD, VALUE_FIELD]) |
| 202 | path = object_records.collection_records_file("ctr", base_dir=data_dir) |
| 203 | |
| 204 | captured: list[tuple[Path, Path, int, int]] = [] |
| 205 | original_replace = Path.replace |
| 206 | |
| 207 | def spy_replace(self: Path, target): |
| 208 | target_path = Path(target) |
| 209 | # Stat BEFORE the rename consumes the temp path's directory entry. |
| 210 | dev_temp = self.stat().st_dev |
| 211 | dev_target_dir = target_path.parent.stat().st_dev |
| 212 | captured.append((self, target_path, dev_temp, dev_target_dir)) |
| 213 | return original_replace(self, target) |
| 214 | |
| 215 | monkeypatch.setattr(Path, "replace", spy_replace) |
| 216 | |
| 217 | object_records.create_collection_record("ctr", {"id": "c1", "value": "0"}, base_dir=data_dir, roots=[]) |
| 218 | |
| 219 | assert captured, "expected _write_collection_records to call Path.replace at least once" |
| 220 | temp_path, target_path, dev_temp, dev_target_dir = captured[-1] |
| 221 | assert target_path == path |
| 222 | assert temp_path.parent == target_path.parent, ( |
| 223 | "temp file and target must live in the same directory for the " |
| 224 | "replace to be a pure (atomic) rename rather than a cross-device " |
| 225 | "copy+delete" |
| 226 | ) |
| 227 | assert dev_temp == dev_target_dir, ( |
| 228 | "temp file and target directory are not on the same filesystem " |
| 229 | "(st_dev differs) -- os.replace would not be atomic here" |
| 230 | ) |
| 231 | |
| 232 | |
| 233 | # ============================================================================= |
| 234 | # 4. APPEND MODE ORDERING -- record bytes land BEFORE the sidecar update, |
| 235 | # and are recoverable from the TSV alone if the sidecar update never |
| 236 | # happens (the sidecar is genuinely disposable / "never required for |
| 237 | # correctness") |
| 238 | # ============================================================================= |
| 239 | |
| 240 | |
| 241 | def test_append_mode_record_bytes_survive_and_are_recoverable_when_sidecar_update_fails( |
| 242 | tmp_path, monkeypatch |
| 243 | ): |
| 244 | """GUARANTEE: HOLDS -- and this is the central append-mode finding. |
| 245 | _persist_write's fast-append branch performs, in order: (1) |
| 246 | `_append_records_rows` -- appends the new physical row and flushes it |
| 247 | to records.tsv, returning its exact byte span; (2) updates the |
| 248 | in-process records cache; (3) `_update_oidx_after_append` -- updates |
| 249 | the id->offset sidecar. Step 3 is a SEPARATE write to a SEPARATE file, |
| 250 | with no transaction wrapping it and step 1 together. |
| 251 | |
| 252 | This test simulates a crash landing exactly between step 1 and step 3 |
| 253 | by monkeypatching `_update_oidx_after_append` to raise. The row's |
| 254 | bytes are, by construction, already fully written and flushed to |
| 255 | records.tsv by the time that patched function is even called -- so |
| 256 | the record must still be present on disk, and (once the sidecar |
| 257 | fault is removed and caches are cleared, simulating the process |
| 258 | restarting) still fully readable via the public API, sourced from |
| 259 | the TSV alone (the in-memory sidecar cache is dropped and the |
| 260 | on-disk sidecar is stale/never written for this row, forcing a fresh |
| 261 | _load_oidx rebuild -- see _load_oidx's "100% best-effort and NEVER |
| 262 | raises" contract, docs/append-only-storage-design.md's "sidecar is |
| 263 | never required for correctness").""" |
| 264 | data_dir = tmp_path / "data" |
| 265 | write_append_schema(data_dir, "logs", [ID_FIELD, VALUE_FIELD]) |
| 266 | object_records.create_collection_record("logs", {"id": "l1", "value": "one"}, base_dir=data_dir, roots=[]) |
| 267 | path = object_records.collection_records_file("logs", base_dir=data_dir) |
| 268 | |
| 269 | def boom(*args, **kwargs): |
| 270 | raise RuntimeError("simulated crash between record append and sidecar update") |
| 271 | |
| 272 | monkeypatch.setattr(object_records, "_update_oidx_after_append", boom) |
| 273 | |
| 274 | with pytest.raises(RuntimeError): |
| 275 | object_records.create_collection_record( |
| 276 | "logs", {"id": "l2", "value": "two"}, base_dir=data_dir, roots=[] |
| 277 | ) |
| 278 | |
| 279 | # The record row's bytes are ALREADY durably on disk despite the |
| 280 | # "crash" in the step that was supposed to run after it. |
| 281 | raw = path.read_text() |
| 282 | assert "l2" in raw and "two" in raw, ( |
| 283 | "the append (step 1) must have already landed on disk before the " |
| 284 | f"simulated sidecar-update failure (step 3); got file content: {raw!r}" |
| 285 | ) |
| 286 | |
| 287 | # Restore the real sidecar-update function (simulating the process |
| 288 | # restarting clean) and drop every in-memory cache (simulating a cold |
| 289 | # process start) -- the sidecar on disk was never told about l2 at |
| 290 | # all. The record must still be readable, sourced purely from the |
| 291 | # canonical TSV via a fresh sidecar rebuild. |
| 292 | monkeypatch.undo() |
| 293 | _clear_caches() |
| 294 | |
| 295 | listing = object_records.list_collection_records("logs", base_dir=data_dir, roots=[])["records"] |
| 296 | assert listing == [ |
| 297 | {"id": "l1", "value": "one"}, |
| 298 | {"id": "l2", "value": "two"}, |
| 299 | ] |
| 300 | |
| 301 | _clear_caches() |
| 302 | by_id = object_records.get_collection_record("logs", "l2", base_dir=data_dir, roots=[]) |
| 303 | assert by_id == {"id": "l2", "value": "two"} |
| 304 | |
| 305 | |
| 306 | def test_append_mode_sidecar_fully_deleted_record_still_recoverable_from_tsv_alone( |
| 307 | tmp_path, monkeypatch |
| 308 | ): |
| 309 | """GUARANTEE: HOLDS. A stronger, more literal version of "the sidecar |
| 310 | is disposable": build up a normal append-mode collection, force the |
| 311 | on-disk `.records.oidx` sidecar to actually exist (a cold by-id read |
| 312 | lazily builds/writes it -- see _load_oidx), then DELETE it outright |
| 313 | (simulating a crash that landed after the record append but with the |
| 314 | sidecar file never having been written/flushed at all, or simply lost). |
| 315 | A subsequent read must still resolve every record correctly, purely by |
| 316 | falling back to a fresh sequential scan of records.tsv (the sidecar's |
| 317 | own "100% best-effort and NEVER raises" rebuild path) -- and it must |
| 318 | reconstitute a fresh sidecar file on disk as a side effect, since the |
| 319 | sidecar is a derived accelerator, not a source of truth.""" |
| 320 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 321 | data_dir = tmp_path / "data" |
| 322 | write_append_schema(data_dir, "logs", [ID_FIELD, VALUE_FIELD]) |
| 323 | for i in range(5): |
| 324 | object_records.create_collection_record( |
| 325 | "logs", {"id": f"l{i}", "value": str(i)}, base_dir=data_dir, roots=[] |
| 326 | ) |
| 327 | path = object_records.collection_records_file("logs", base_dir=data_dir) |
| 328 | |
| 329 | # Force the on-disk sidecar to exist: a cold by-id read (cache |
| 330 | # disabled via the env var above) must go through _load_oidx, which |
| 331 | # writes it out. |
| 332 | _clear_caches() |
| 333 | warm = object_records.get_collection_record("logs", "l2", base_dir=data_dir, roots=[]) |
| 334 | assert warm == {"id": "l2", "value": "2"} |
| 335 | sidecar_path = path.with_name(object_records.OIDX_FILE) |
| 336 | assert sidecar_path.exists(), "expected the sidecar to have been built by the by-id read above" |
| 337 | |
| 338 | # Simulate total loss of the sidecar (crash before it was ever |
| 339 | # written/flushed, or the file simply lost) -- delete it, and drop |
| 340 | # every in-memory trace of it too (simulating a cold process). |
| 341 | sidecar_path.unlink() |
| 342 | _clear_caches() |
| 343 | assert not sidecar_path.exists() |
| 344 | |
| 345 | # Every record must still resolve correctly, by-id and via a full list, |
| 346 | # sourced entirely from records.tsv. |
| 347 | for i in range(5): |
| 348 | rec = object_records.get_collection_record("logs", f"l{i}", base_dir=data_dir, roots=[]) |
| 349 | assert rec == {"id": f"l{i}", "value": str(i)} |
| 350 | |
| 351 | _clear_caches() |
| 352 | listing = object_records.list_collection_records("logs", base_dir=data_dir, roots=[])["records"] |
| 353 | assert listing == [{"id": f"l{i}", "value": str(i)} for i in range(5)] |
| 354 | |
| 355 | # And the sidecar was disposable in BOTH directions: it self-heals |
| 356 | # (a fresh rebuild is written back to disk once something reads |
| 357 | # through it again). |
| 358 | assert sidecar_path.exists(), "expected _load_oidx's rebuild path to have re-created the sidecar" |
| 359 | |
| 360 | |
| 361 | def test_append_mode_sidecar_corrupted_valid_utf8_garbage_still_recoverable( |
| 362 | tmp_path, monkeypatch |
| 363 | ): |
| 364 | """GUARANTEE: HOLDS. Baseline for the FINDING test right below this |
| 365 | one: the sidecar file present and non-empty, but WRONG -- overwritten |
| 366 | with plain-ASCII (therefore valid-UTF8) garbage that doesn't match the |
| 367 | "oidx1\\t<ino>" header format at all -- simulating a corrupt-but-still- |
| 368 | decodable sidecar write. `_read_oidx_header`'s tag/column-count/int |
| 369 | checks correctly treat this as an unparseable header and `_load_oidx` |
| 370 | falls back to a full, correct rebuild from the canonical records.tsv.""" |
| 371 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 372 | data_dir = tmp_path / "data" |
| 373 | write_append_schema(data_dir, "logs", [ID_FIELD, VALUE_FIELD]) |
| 374 | for i in range(5): |
| 375 | object_records.create_collection_record( |
| 376 | "logs", {"id": f"l{i}", "value": str(i)}, base_dir=data_dir, roots=[] |
| 377 | ) |
| 378 | path = object_records.collection_records_file("logs", base_dir=data_dir) |
| 379 | |
| 380 | _clear_caches() |
| 381 | object_records.get_collection_record("logs", "l2", base_dir=data_dir, roots=[]) |
| 382 | sidecar_path = path.with_name(object_records.OIDX_FILE) |
| 383 | assert sidecar_path.exists() |
| 384 | |
| 385 | # Corrupt it: valid-UTF8 (plain ASCII) garbage, no valid |
| 386 | # "oidx1\t<ino>" header shape at all. |
| 387 | sidecar_path.write_text("this-is-not-a-real-sidecar-header-line\nmore junk\n") |
| 388 | _clear_caches() |
| 389 | |
| 390 | for i in range(5): |
| 391 | rec = object_records.get_collection_record("logs", f"l{i}", base_dir=data_dir, roots=[]) |
| 392 | assert rec == {"id": f"l{i}", "value": str(i)}, ( |
| 393 | "a corrupt-but-decodable sidecar must never cause a wrong/missing " |
| 394 | "read result -- it must be detected as incoherent and the read " |
| 395 | "must fall back to the canonical records.tsv" |
| 396 | ) |
| 397 | |
| 398 | |
| 399 | def test_append_mode_sidecar_corrupted_non_utf8_garbage_self_heals( |
| 400 | tmp_path, monkeypatch |
| 401 | ): |
| 402 | """A sidecar corrupted with invalid-UTF-8 bytes self-heals like every |
| 403 | other corrupt-sidecar shape. FIXED, formerly a strict-xfail FINDING: |
| 404 | _read_oidx_header opened the file in text mode with only `except |
| 405 | OSError`, so garbage that was not valid UTF-8 raised an uncaught |
| 406 | UnicodeDecodeError on every read instead of reading as "header does |
| 407 | not parse -> rebuild" -- breaking _load_oidx's own contract ("100% |
| 408 | best-effort and NEVER raises"). The hardening catches decode errors in |
| 409 | the same clause, and this test flipped from asserting the crash to |
| 410 | asserting the recovery the contract always promised.""" |
| 411 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 412 | data_dir = tmp_path / "data" |
| 413 | write_append_schema(data_dir, "logs", [ID_FIELD, VALUE_FIELD]) |
| 414 | for i in range(5): |
| 415 | object_records.create_collection_record( |
| 416 | "logs", {"id": f"l{i}", "value": str(i)}, base_dir=data_dir, roots=[] |
| 417 | ) |
| 418 | path = object_records.collection_records_file("logs", base_dir=data_dir) |
| 419 | |
| 420 | _clear_caches() |
| 421 | object_records.get_collection_record("logs", "l2", base_dir=data_dir, roots=[]) |
| 422 | sidecar_path = path.with_name(object_records.OIDX_FILE) |
| 423 | assert sidecar_path.exists() |
| 424 | |
| 425 | # Corrupt it: invalid-UTF-8 bytes, no valid "oidx1\t<ino>" header at all. |
| 426 | sidecar_path.write_bytes(b"\x00\x01garbage-not-an-oidx-file\xff\xfe") |
| 427 | _clear_caches() |
| 428 | |
| 429 | for i in range(5): |
| 430 | rec = object_records.get_collection_record("logs", f"l{i}", base_dir=data_dir, roots=[]) |
| 431 | assert rec == {"id": f"l{i}", "value": str(i)}, ( |
| 432 | "a corrupt sidecar must never cause a wrong/missing read result -- " |
| 433 | "it must be detected as incoherent and the read must fall back to " |
| 434 | "the canonical records.tsv" |
| 435 | ) |
| 436 | |
| 437 | |
| 438 | # ============================================================================= |
| 439 | # 5. CROSS-ARTIFACT (NOT single-record) ATOMICITY BOUNDARY -- create's |
| 440 | # record write vs. its record-change emission |
| 441 | # ============================================================================= |
| 442 | |
| 443 | |
| 444 | def test_record_write_and_its_record_change_emission_are_two_separate_commits_not_one( |
| 445 | tmp_path, monkeypatch |
| 446 | ): |
| 447 | """GUARANTEE: honestly documented, NOT a single-transaction atomicity |
| 448 | claim -- this is the boundary the report calls out explicitly. |
| 449 | create_collection_record performs two independent durable writes: |
| 450 | (a) the record row itself, via _persist_write (atomic for classic, |
| 451 | append-then-sidecar for append -- both probed above), and (b) a |
| 452 | record-change audit entry, via object_record_changes.append_record_change, |
| 453 | which appends a JSON line to a COMPLETELY SEPARATE file |
| 454 | (record_changes/<collection>/changes.jsonl) under its OWN lock. These |
| 455 | are not wrapped in any shared transaction. |
| 456 | |
| 457 | Simulated here by monkeypatching append_record_change to raise AFTER |
| 458 | the record write has already happened (it is called after |
| 459 | _persist_write in create_collection_record's body): the record itself |
| 460 | is fully committed and durably readable even though its accompanying |
| 461 | change-log entry never landed. This is the honest, narrower claim: |
| 462 | a SINGLE record's write is atomic; a record plus a separate artifact |
| 463 | describing it is NOT jointly atomic -- each write is atomic |
| 464 | individually, but the pair can diverge if the process dies between |
| 465 | them.""" |
| 466 | data_dir = tmp_path / "data" |
| 467 | write_schema(data_dir, "contacts", [ID_FIELD, {"name": "name"}]) |
| 468 | |
| 469 | def boom(*args, **kwargs): |
| 470 | raise RuntimeError("simulated crash between record write and record-change emission") |
| 471 | |
| 472 | monkeypatch.setattr(object_records.object_record_changes, "append_record_change", boom) |
| 473 | |
| 474 | with pytest.raises(RuntimeError): |
| 475 | object_records.create_collection_record( |
| 476 | "contacts", {"id": "c1", "name": "Ada"}, base_dir=data_dir, roots=[] |
| 477 | ) |
| 478 | |
| 479 | # The record write landed and is durably readable, DESPITE the |
| 480 | # "crash" in the very next step of the same function call. |
| 481 | monkeypatch.undo() |
| 482 | _clear_caches() |
| 483 | rec = object_records.get_collection_record("contacts", "c1", base_dir=data_dir, roots=[]) |
| 484 | assert rec == {"id": "c1", "name": "Ada"}, ( |
| 485 | "the record write (step 1) must be unaffected by a failure in the " |
| 486 | "subsequent, separate record-change write (step 2)" |
| 487 | ) |
| 488 | |
| 489 | # ...but there is no corresponding record-change entry: the pair |
| 490 | # genuinely diverged. |
| 491 | changes_path = data_dir / "record_changes" / "contacts" / "changes.jsonl" |
| 492 | assert not changes_path.exists() or changes_path.read_text() == "", ( |
| 493 | "expected NO record-change entry for c1 (its emission was the thing " |
| 494 | "that 'crashed') -- if this fails, the two writes may have become " |
| 495 | "coupled and this characterization needs updating, not the " |
| 496 | "assertion weakened" |
| 497 | ) |
| 498 | |
| 499 | |
| 500 | # ============================================================================= |
| 501 | # 6. READER-DURING-WRITE, VIA THE PUBLIC API -- both storage modes |
| 502 | # ============================================================================= |
| 503 | |
| 504 | |
| 505 | def test_reader_during_write_classic_mode_never_observes_partial_or_corrupt_records(tmp_path): |
| 506 | """GUARANTEE: HOLDS. A reader thread calls read_collection_records in a |
| 507 | tight loop (going through the REAL parser, unlike test 2 above) while |
| 508 | a writer thread performs N updates concurrently, classic mode. Every |
| 509 | single read observed by the reader must be a well-formed, single-row, |
| 510 | two-field record with a numeric value -- proving the reader-holds-no- |
| 511 | lock design (see object_records.py's CONCURRENCY note) never yields a |
| 512 | torn/half-written row for this write pattern.""" |
| 513 | data_dir = tmp_path / "data" |
| 514 | write_schema(data_dir, "ctr", [ID_FIELD, VALUE_FIELD]) |
| 515 | object_records.create_collection_record("ctr", {"id": "c1", "value": "0"}, base_dir=data_dir, roots=[]) |
| 516 | |
| 517 | n = 300 |
| 518 | stop = threading.Event() |
| 519 | anomalies: list[str] = [] |
| 520 | reads_done = [0] |
| 521 | |
| 522 | def reader() -> None: |
| 523 | while not stop.is_set(): |
| 524 | try: |
| 525 | records = object_records.read_collection_records("ctr", base_dir=data_dir, roots=[]) |
| 526 | except Exception as exc: # noqa: BLE001 - characterizing, must not miss any failure mode |
| 527 | anomalies.append(f"exception during read: {exc!r}") |
| 528 | continue |
| 529 | reads_done[0] += 1 |
| 530 | if len(records) != 1: |
| 531 | anomalies.append(f"unexpected record count (torn write?): {records!r}") |
| 532 | continue |
| 533 | rec = records[0] |
| 534 | if set(rec.keys()) != {"id", "value"} or rec["id"] != "c1": |
| 535 | anomalies.append(f"torn/partial record shape: {rec!r}") |
| 536 | continue |
| 537 | try: |
| 538 | int(rec["value"]) |
| 539 | except ValueError: |
| 540 | anomalies.append(f"unparseable value (torn write?): {rec!r}") |
| 541 | |
| 542 | reader_thread = threading.Thread(target=reader) |
| 543 | reader_thread.start() |
| 544 | try: |
| 545 | for i in range(n): |
| 546 | object_records.update_collection_record( |
| 547 | "ctr", "c1", {"value": str(i)}, base_dir=data_dir, roots=[] |
| 548 | ) |
| 549 | finally: |
| 550 | stop.set() |
| 551 | reader_thread.join(timeout=15) |
| 552 | |
| 553 | assert not reader_thread.is_alive() |
| 554 | assert reads_done[0] > 0, "reader thread never got a chance to run concurrently" |
| 555 | assert anomalies == [], f"reader observed {len(anomalies)} anomalies; first: {anomalies[0]!r}" |
| 556 | |
| 557 | |
| 558 | def test_reader_during_write_append_mode_never_observes_partial_or_corrupt_records( |
| 559 | tmp_path, monkeypatch |
| 560 | ): |
| 561 | """GUARANTEE: HOLDS. Same probe as above, append mode -- and with |
| 562 | DBBASIC_RECORDS_CACHE_MAX_ROWS=0 so the reader is forced through the |
| 563 | id->offset sidecar / cold full-fold path on every single call (the |
| 564 | warm-cache path would trivially never see anything, since the writer |
| 565 | holds the module's own lock while updating the cache -- this is |
| 566 | specifically about the LOCK-FREE disk-backed read path racing a live |
| 567 | append). Every by-id read must return a well-formed, complete, |
| 568 | numeric-valued record -- never a truncated/garbled row from a write |
| 569 | caught mid-flight, and never a stale sidecar entry pointing at a |
| 570 | since-superseded offset.""" |
| 571 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 572 | data_dir = tmp_path / "data" |
| 573 | write_append_schema(data_dir, "ctr", [ID_FIELD, VALUE_FIELD]) |
| 574 | object_records.create_collection_record("ctr", {"id": "c1", "value": "0"}, base_dir=data_dir, roots=[]) |
| 575 | |
| 576 | n = 250 |
| 577 | stop = threading.Event() |
| 578 | anomalies: list[str] = [] |
| 579 | reads_done = [0] |
| 580 | |
| 581 | def reader() -> None: |
| 582 | while not stop.is_set(): |
| 583 | try: |
| 584 | rec = object_records.get_collection_record("ctr", "c1", base_dir=data_dir, roots=[]) |
| 585 | except Exception as exc: # noqa: BLE001 - characterizing, must not miss any failure mode |
| 586 | anomalies.append(f"exception during by-id read: {exc!r}") |
| 587 | continue |
| 588 | reads_done[0] += 1 |
| 589 | if rec is None: |
| 590 | anomalies.append("record disappeared mid-write") |
| 591 | continue |
| 592 | if set(rec.keys()) != {"id", "value"} or rec["id"] != "c1": |
| 593 | anomalies.append(f"torn/partial record shape: {rec!r}") |
| 594 | continue |
| 595 | try: |
| 596 | int(rec["value"]) |
| 597 | except ValueError: |
| 598 | anomalies.append(f"unparseable value (torn write?): {rec!r}") |
| 599 | |
| 600 | reader_thread = threading.Thread(target=reader) |
| 601 | reader_thread.start() |
| 602 | try: |
| 603 | for i in range(n): |
| 604 | object_records.update_collection_record( |
| 605 | "ctr", "c1", {"value": str(i)}, base_dir=data_dir, roots=[] |
| 606 | ) |
| 607 | finally: |
| 608 | stop.set() |
| 609 | reader_thread.join(timeout=15) |
| 610 | |
| 611 | assert not reader_thread.is_alive() |
| 612 | assert reads_done[0] > 0, "reader thread never got a chance to run concurrently" |
| 613 | assert anomalies == [], f"reader observed {len(anomalies)} anomalies; first: {anomalies[0]!r}" |
| 614 | |
| 615 | |
| 616 | # ============================================================================= |
| 617 | # SUMMARY (see also the accompanying report) |
| 618 | # |
| 619 | # - Single-record write, classic mode: ATOMIC (all-or-nothing). Backed |
| 620 | # by temp-file-in-same-directory + Path.replace (POSIX atomic |
| 621 | # rename). A concurrent reader NEVER observes a torn/partial file -- |
| 622 | # confirmed both at the raw-byte level (test 2) and through a held |
| 623 | # pre-write fd (test 1) -- and the temp file's same-directory |
| 624 | # placement is confirmed to share a filesystem (st_dev) with the |
| 625 | # target, so the replace is a true rename, never a cross-device |
| 626 | # copy+delete fallback (test 3). |
| 627 | # - Single-record write, append mode: the RECORD ROW ITSELF is atomic |
| 628 | # in the same sense -- it lands as one flushed write to the file |
| 629 | # before anything else happens. The id->offset SIDECAR update is a |
| 630 | # SEPARATE, later, best-effort step -- and is provably disposable: |
| 631 | # deleting it, or having its update fail outright (simulating a crash |
| 632 | # between the append and the sidecar write) never loses or corrupts a |
| 633 | # record -- every read falls back to a fresh, correct rebuild from the |
| 634 | # canonical records.tsv (tests 4a/4b). This is the concrete meaning of |
| 635 | # "sidecar is never required for correctness": it is a derived |
| 636 | # accelerator, not a source of truth. ONE EXCEPTION FOUND (test 4d, |
| 637 | # xfail/FINDING, not fixed here): a sidecar corrupted with bytes that |
| 638 | # are not valid UTF-8 crashes the read (UnicodeDecodeError) instead of |
| 639 | # being treated like every other corrupt-header shape and falling back |
| 640 | # to a rebuild -- _read_oidx_header's `except OSError` doesn't catch |
| 641 | # it. Valid-UTF8-but-wrong-format garbage IS handled correctly |
| 642 | # (test 4c, HOLDS) -- this is specifically a decode-error gap. |
| 643 | # - Reader-during-write: HOLDS in both modes. A lock-free reader thread |
| 644 | # racing a writer thread across hundreds of updates never observed a |
| 645 | # torn, partial, or unparseable record, at either the raw-byte level |
| 646 | # (classic) or the parsed-record level (both modes) (tests 2, 6a, 6b). |
| 647 | # - Cross-artifact atomicity: DOES NOT HOLD, and is not claimed to. |
| 648 | # create_collection_record's record write and its record-change audit |
| 649 | # emission are two independent commits to two independent files, not |
| 650 | # one transaction -- a failure between them leaves the record durably |
| 651 | # committed with no matching change-log entry (test 5). Each write is |
| 652 | # individually atomic; the PAIR is not jointly atomic. This is the |
| 653 | # honest boundary of what this engine's atomicity guarantee covers: |
| 654 | # single-record, single-file writes -- not multi-artifact operations. |
| 655 | # ============================================================================= |