askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_durability_plaintext_rebuild_characterization.py · 23.1 KB · 532 lines
| 1 | """CHARACTERIZATION tests proving (or falsifying) the GOVERNING PRINCIPLE the |
| 2 | plain-text-durability pitch rests on: |
| 3 | |
| 4 | The canonical ``records.tsv`` file is the SOLE source of truth. Every |
| 5 | accelerator built on top of it -- the id->offset sidecar |
| 6 | (``.records.oidx``), the in-process ``_RECORDS_CACHE`` / ``_OIDX_CACHE``, |
| 7 | and (for derived collections) folded/aggregated views -- is DERIVED and |
| 8 | REBUILDABLE from the TSV, never authoritative. Delete every accelerator |
| 9 | and you lose *speed*, never a single byte of data. This is the "delete |
| 10 | our indexes, your data is still all there in files you can read" proof: |
| 11 | the contract-winning demo (see plan/database-test-strategy.md, gitignored). |
| 12 | |
| 13 | This file is pure characterization of EXISTING behavior. It does not modify |
| 14 | any production module (object_records.py, object_stock.py, etc.) and does |
| 15 | not touch packages/. Where a probe reveals the principle does NOT hold for |
| 16 | some accelerator, the assertion is written to state the correctness property |
| 17 | the durability pitch REQUIRES, and is left to fail (or is reported via a |
| 18 | soft check with a printed finding) rather than "fixed" by weakening it. |
| 19 | |
| 20 | Layout under test (read from object_records.py's own module docstring/ |
| 21 | comments, not guessed): |
| 22 | - ``data/collections/{collection}/records.tsv`` -- the canonical file. |
| 23 | Header row 1 names every logical field (self-describing); for |
| 24 | append-storage collections the physical header additionally carries a |
| 25 | leading ``_op`` column (values ``""`` = upsert, ``"del"`` = tombstone). |
| 26 | - ``data/collections/{collection}/.records.oidx`` -- the id->byte-offset |
| 27 | sidecar (append mode only). A dotfile, invisible to every collection |
| 28 | glob and to backup. Format: header line ``"oidx1\\t<data_ino>"`` then |
| 29 | one ``"<row_start>\\t<row_end>\\t<op>\\t<id>"`` line per indexed physical |
| 30 | row. 100% disposable per object_records.py's own comment: "any reader |
| 31 | unable to make sense of it ... rebuilds it with one sequential scan of |
| 32 | records.tsv rather than raising." |
| 33 | - ``object_records._RECORDS_CACHE`` / ``._OIDX_CACHE`` -- in-process, |
| 34 | module-level dicts. Not on disk at all; trivially disposable by |
| 35 | definition, but exercised here anyway so the "cold reopen" path is |
| 36 | genuinely cold, matching how a fresh process would see the world. |
| 37 | |
| 38 | Four probes, each reported as PRINCIPLE HOLDS or PRINCIPLE VIOLATED: |
| 39 | 1. REBUILD-FROM-TSV-ALONE |
| 40 | 2. NO-HIDDEN-STATE |
| 41 | 3. READ-WITH-STANDARD-TOOLS |
| 42 | 4. ARCHIVAL round-trip (naive external reader, zero DBBASIC code) |
| 43 | """ |
| 44 | |
| 45 | from __future__ import annotations |
| 46 | |
| 47 | import csv |
| 48 | import json |
| 49 | from pathlib import Path |
| 50 | |
| 51 | import pytest |
| 52 | |
| 53 | import object_records |
| 54 | |
| 55 | pytestmark = pytest.mark.conformance |
| 56 | |
| 57 | |
| 58 | ID_FIELD = {"name": "id"} |
| 59 | AMOUNT_FIELD = {"name": "amount_cents"} |
| 60 | NOTE_FIELD = {"name": "note", "type": "textarea"} |
| 61 | CRLF_FIELD = {"name": "crlf_field", "type": "textarea"} |
| 62 | BARE_CR_FIELD = {"name": "bare_cr_field", "type": "textarea"} |
| 63 | COMMAS_FIELD = {"name": "commas_field"} |
| 64 | EMPTY_FIELD = {"name": "empty_field"} |
| 65 | |
| 66 | ALL_FIELDS = [ |
| 67 | ID_FIELD, |
| 68 | AMOUNT_FIELD, |
| 69 | NOTE_FIELD, |
| 70 | CRLF_FIELD, |
| 71 | BARE_CR_FIELD, |
| 72 | COMMAS_FIELD, |
| 73 | EMPTY_FIELD, |
| 74 | ] |
| 75 | LOGICAL_FIELD_NAMES = [f["name"] for f in ALL_FIELDS] # excludes "_op" |
| 76 | |
| 77 | |
| 78 | # --- setup helpers (mirror tests/test_embedded_json_lines_characterization.py) -- |
| 79 | |
| 80 | |
| 81 | def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 82 | path = data_dir / "schemas" / f"{collection}.json" |
| 83 | path.parent.mkdir(parents=True, exist_ok=True) |
| 84 | path.write_text(json.dumps({"fields": fields})) |
| 85 | return path |
| 86 | |
| 87 | |
| 88 | def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 89 | path = data_dir / "schemas" / f"{collection}.json" |
| 90 | path.parent.mkdir(parents=True, exist_ok=True) |
| 91 | path.write_text(json.dumps({"fields": fields, "storage": "append"})) |
| 92 | return path |
| 93 | |
| 94 | |
| 95 | def _clear_caches() -> None: |
| 96 | """Cold every in-process accelerator: a fresh _RECORDS_CACHE + |
| 97 | _OIDX_CACHE is what forces the NEXT read to actually go back to disk |
| 98 | (records.tsv, and -- for append mode -- rebuild/reconsult the |
| 99 | sidecar) instead of serving a warm hit. This is the in-memory half of |
| 100 | "delete every accelerator"; the on-disk half is unlinking |
| 101 | .records.oidx, done explicitly in each test below.""" |
| 102 | object_records._RECORDS_CACHE.clear() |
| 103 | object_records._OIDX_CACHE.clear() |
| 104 | |
| 105 | |
| 106 | def _exotic_payload(suffix: str, amount: int) -> dict: |
| 107 | """A record whose values are deliberately hostile to a naive delimiter |
| 108 | scan: literal TAB, NEWLINE, double-quote, backslash, Unicode, CRLF, a |
| 109 | BARE CR (no paired LF -- the specific case the task's own note flags: |
| 110 | "Bare-CR rows are now quoted, so standard csv parsing handles them"), |
| 111 | embedded commas (harmless under a TAB delimiter, but a nice archival |
| 112 | sanity check), and a genuinely empty field.""" |
| 113 | return { |
| 114 | "id": f"rec-{suffix}", |
| 115 | "amount_cents": str(amount), |
| 116 | "note": f"café \t x\n y \"q\" \\z 日本 #{suffix}", |
| 117 | "crlf_field": f"alpha-{suffix}\r\nbeta-{suffix}", |
| 118 | "bare_cr_field": f"before-{suffix}\rafter-{suffix}", |
| 119 | "commas_field": f"a,b,c,{suffix}", |
| 120 | "empty_field": "", |
| 121 | } |
| 122 | |
| 123 | |
| 124 | def _records_tsv_path(data_dir: Path, collection: str) -> Path: |
| 125 | return object_records.collection_records_file(collection, base_dir=data_dir) |
| 126 | |
| 127 | |
| 128 | def _oidx_path(data_dir: Path, collection: str) -> Path: |
| 129 | return object_records._oidx_path(_records_tsv_path(data_dir, collection)) |
| 130 | |
| 131 | |
| 132 | def _collection_dir_files(data_dir: Path, collection: str) -> list[Path]: |
| 133 | coll_dir = _records_tsv_path(data_dir, collection).parent |
| 134 | if not coll_dir.exists(): |
| 135 | return [] |
| 136 | return sorted(p for p in coll_dir.iterdir() if p.is_file()) |
| 137 | |
| 138 | |
| 139 | # ============================================================================= |
| 140 | # 1. REBUILD-FROM-TSV-ALONE |
| 141 | # ============================================================================= |
| 142 | |
| 143 | |
| 144 | def test_rebuild_from_tsv_alone_classic_mode(tmp_path): |
| 145 | """Classic-mode collection: no sidecar is ever built for classic mode |
| 146 | (only append-storage collections use .records.oidx) -- so this probe's |
| 147 | job here is narrower: prove that clearing the in-process caches alone |
| 148 | (the only "accelerator" classic mode has) still yields byte-identical |
| 149 | reads. Included primarily as the control case for the append-mode test |
| 150 | right below it.""" |
| 151 | data_dir = tmp_path / "data" |
| 152 | collection = "widgets_classic" |
| 153 | write_schema(data_dir, collection, ALL_FIELDS) |
| 154 | |
| 155 | records = [_exotic_payload(str(i), 1000 + i) for i in range(8)] |
| 156 | for rec in records: |
| 157 | object_records.create_collection_record(collection, rec, base_dir=data_dir, roots=[]) |
| 158 | |
| 159 | path = _records_tsv_path(data_dir, collection) |
| 160 | tsv_bytes_before = path.read_bytes() |
| 161 | |
| 162 | all_before = object_records.read_collection_records(collection, base_dir=data_dir, roots=[]) |
| 163 | by_id_before = { |
| 164 | rec["id"]: object_records.get_collection_record( |
| 165 | collection, rec["id"], base_dir=data_dir, roots=[] |
| 166 | ) |
| 167 | for rec in records[::2] |
| 168 | } |
| 169 | |
| 170 | # Delete every accelerator this mode has (no on-disk sidecar exists for |
| 171 | # classic mode, but assert that explicitly rather than assume it). |
| 172 | sidecar = _oidx_path(data_dir, collection) |
| 173 | assert not sidecar.exists(), "classic mode unexpectedly produced an oidx sidecar" |
| 174 | _clear_caches() |
| 175 | |
| 176 | tsv_bytes_after = path.read_bytes() |
| 177 | all_after = object_records.read_collection_records(collection, base_dir=data_dir, roots=[]) |
| 178 | by_id_after = { |
| 179 | rec["id"]: object_records.get_collection_record( |
| 180 | collection, rec["id"], base_dir=data_dir, roots=[] |
| 181 | ) |
| 182 | for rec in records[::2] |
| 183 | } |
| 184 | |
| 185 | assert tsv_bytes_after == tsv_bytes_before, "PRINCIPLE VIOLATED: records.tsv mutated by a mere read" |
| 186 | assert all_after == all_before, "PRINCIPLE VIOLATED: cold reread diverges from original in classic mode" |
| 187 | assert by_id_after == by_id_before, "PRINCIPLE VIOLATED: cold by-id reread diverges in classic mode" |
| 188 | print("\n[REBUILD-FROM-TSV-ALONE / classic] PRINCIPLE HOLDS: cold reread byte-identical.") |
| 189 | |
| 190 | |
| 191 | def test_rebuild_from_tsv_alone_append_mode_with_updates_and_deletes(tmp_path, monkeypatch): |
| 192 | """The real test of the sidecar's disposability: an append-mode |
| 193 | collection with creates, an UPDATE (superseding physical row) and a |
| 194 | DELETE (tombstone row) already on disk, plus exotic content throughout. |
| 195 | DBBASIC_RECORDS_CACHE_MAX_ROWS=0 forces every read through the |
| 196 | id->offset sidecar rather than the ordinary records cache (same trick |
| 197 | the embedded-JSON characterization file uses), so this genuinely |
| 198 | exercises the accelerator being proven disposable, not a shortcut |
| 199 | around it. |
| 200 | |
| 201 | Sequence: capture the full golden dataset (fold + several by-id reads), |
| 202 | THEN physically delete .records.oidx from disk and clear every |
| 203 | in-process cache WITHOUT touching records.tsv, THEN reread and compare. |
| 204 | """ |
| 205 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 206 | data_dir = tmp_path / "data" |
| 207 | collection = "widgets_append" |
| 208 | write_append_schema(data_dir, collection, ALL_FIELDS) |
| 209 | |
| 210 | records = [_exotic_payload(str(i), 2000 + i) for i in range(10)] |
| 211 | for rec in records: |
| 212 | object_records.create_collection_record(collection, rec, base_dir=data_dir, roots=[]) |
| 213 | |
| 214 | # An UPDATE (a second, superseding physical row for the same id) and a |
| 215 | # DELETE (a tombstone row) -- both must fold correctly from a from- |
| 216 | # scratch sidecar rebuild, not just from a freshly-built one. |
| 217 | object_records.update_collection_record( |
| 218 | collection, "rec-3", {"note": "UPDATED note, still hostile: \t\n\"\\ 日本"}, |
| 219 | base_dir=data_dir, roots=[], |
| 220 | ) |
| 221 | object_records.delete_collection_record(collection, "rec-7", base_dir=data_dir, roots=[]) |
| 222 | |
| 223 | path = _records_tsv_path(data_dir, collection) |
| 224 | sidecar = _oidx_path(data_dir, collection) |
| 225 | |
| 226 | _clear_caches() |
| 227 | tsv_bytes_before = path.read_bytes() |
| 228 | all_before = object_records.read_collection_records(collection, base_dir=data_dir, roots=[]) |
| 229 | surviving_ids = [r["id"] for r in all_before] |
| 230 | assert "rec-7" not in surviving_ids # delete really took |
| 231 | assert len(all_before) == 9 |
| 232 | |
| 233 | _clear_caches() |
| 234 | by_id_before = { |
| 235 | rid: object_records.get_collection_record(collection, rid, base_dir=data_dir, roots=[]) |
| 236 | for rid in surviving_ids |
| 237 | } |
| 238 | assert "UPDATED note" in by_id_before["rec-3"]["note"] |
| 239 | |
| 240 | # --- delete EVERY accelerator: the on-disk sidecar AND every in-process |
| 241 | # cache -- without touching records.tsv at all. --- |
| 242 | assert sidecar.exists(), "expected the sidecar to have been built by the reads above" |
| 243 | sidecar.unlink() |
| 244 | _clear_caches() |
| 245 | |
| 246 | tsv_bytes_after_delete = path.read_bytes() |
| 247 | assert tsv_bytes_after_delete == tsv_bytes_before, ( |
| 248 | "PRINCIPLE VIOLATED: deleting the sidecar + clearing caches touched records.tsv" |
| 249 | ) |
| 250 | assert not sidecar.exists() |
| 251 | |
| 252 | # Reopen/reread with everything cold: the sidecar must be silently, |
| 253 | # transparently rebuilt from records.tsv alone. |
| 254 | all_after = object_records.read_collection_records(collection, base_dir=data_dir, roots=[]) |
| 255 | assert all_after == all_before, "PRINCIPLE VIOLATED: fold-all diverges after sidecar deletion" |
| 256 | |
| 257 | _clear_caches() |
| 258 | by_id_after = { |
| 259 | rid: object_records.get_collection_record(collection, rid, base_dir=data_dir, roots=[]) |
| 260 | for rid in surviving_ids |
| 261 | } |
| 262 | assert by_id_after == by_id_before, ( |
| 263 | "PRINCIPLE VIOLATED: by-id reads diverge after sidecar deletion -- the sidecar was load-bearing" |
| 264 | ) |
| 265 | |
| 266 | # records.tsv itself must still be byte-for-byte what it was -- the |
| 267 | # rebuild is allowed to (and does) recreate .records.oidx as a |
| 268 | # *side effect* of a read, but must never touch the canonical file. |
| 269 | tsv_bytes_final = path.read_bytes() |
| 270 | assert tsv_bytes_final == tsv_bytes_before, ( |
| 271 | "PRINCIPLE VIOLATED: records.tsv changed as a side effect of rebuilding the sidecar" |
| 272 | ) |
| 273 | |
| 274 | print( |
| 275 | "\n[REBUILD-FROM-TSV-ALONE / append, with update+delete] PRINCIPLE HOLDS: " |
| 276 | "after unlinking .records.oidx and clearing _RECORDS_CACHE/_OIDX_CACHE, " |
| 277 | "read_collection_records and every by-id get_collection_record reproduced " |
| 278 | "the pre-deletion dataset exactly, and records.tsv's bytes never changed." |
| 279 | ) |
| 280 | if sidecar.exists(): |
| 281 | print(" (sidecar was transparently rebuilt on disk as a read side effect -- expected/fine.)") |
| 282 | |
| 283 | |
| 284 | # ============================================================================= |
| 285 | # 2. NO-HIDDEN-STATE |
| 286 | # ============================================================================= |
| 287 | |
| 288 | |
| 289 | def test_no_hidden_state_only_records_tsv_is_load_bearing(tmp_path, monkeypatch): |
| 290 | """Enumerate every file that exists for a collection, then delete |
| 291 | EVERYTHING except records.tsv (not just the sidecar -- whatever else |
| 292 | is sitting in the collection directory) and confirm the full dataset |
| 293 | is still completely intact. If any byte of truth turned out to live |
| 294 | only in a non-.tsv file, this is where it would show up as a genuine |
| 295 | data loss (missing/wrong records), not just a slow rebuild.""" |
| 296 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 297 | data_dir = tmp_path / "data" |
| 298 | collection = "no_hidden_state" |
| 299 | write_append_schema(data_dir, collection, ALL_FIELDS) |
| 300 | |
| 301 | records = [_exotic_payload(str(i), 3000 + i) for i in range(6)] |
| 302 | for rec in records: |
| 303 | object_records.create_collection_record(collection, rec, base_dir=data_dir, roots=[]) |
| 304 | object_records.update_collection_record( |
| 305 | collection, "rec-1", {"amount_cents": "999999"}, base_dir=data_dir, roots=[] |
| 306 | ) |
| 307 | object_records.delete_collection_record(collection, "rec-4", base_dir=data_dir, roots=[]) |
| 308 | |
| 309 | _clear_caches() |
| 310 | golden = object_records.read_collection_records(collection, base_dir=data_dir, roots=[]) |
| 311 | golden_ids = sorted(r["id"] for r in golden) |
| 312 | assert golden_ids == ["rec-0", "rec-1", "rec-2", "rec-3", "rec-5"] |
| 313 | |
| 314 | path = _records_tsv_path(data_dir, collection) |
| 315 | files_before = _collection_dir_files(data_dir, collection) |
| 316 | print(f"\n[NO-HIDDEN-STATE] files present for '{collection}' before cleanup: " |
| 317 | f"{[p.name for p in files_before]}") |
| 318 | assert path in files_before |
| 319 | |
| 320 | # Delete every file in the collection directory EXCEPT records.tsv. |
| 321 | for f in files_before: |
| 322 | if f != path: |
| 323 | f.unlink() |
| 324 | _clear_caches() |
| 325 | |
| 326 | files_remaining = _collection_dir_files(data_dir, collection) |
| 327 | assert files_remaining == [path], ( |
| 328 | f"expected only records.tsv to remain, found {[p.name for p in files_remaining]}" |
| 329 | ) |
| 330 | |
| 331 | # The dataset must be fully intact reading from records.tsv alone. |
| 332 | reread = object_records.read_collection_records(collection, base_dir=data_dir, roots=[]) |
| 333 | assert sorted(r["id"] for r in reread) == golden_ids |
| 334 | assert {r["id"]: r for r in reread} == {r["id"]: r for r in golden} |
| 335 | |
| 336 | _clear_caches() |
| 337 | for rid in golden_ids: |
| 338 | by_id = object_records.get_collection_record(collection, rid, base_dir=data_dir, roots=[]) |
| 339 | expected = next(r for r in golden if r["id"] == rid) |
| 340 | assert by_id == expected |
| 341 | |
| 342 | print( |
| 343 | "[NO-HIDDEN-STATE] PRINCIPLE HOLDS: after deleting every file except " |
| 344 | "records.tsv, the complete dataset (including the prior update and " |
| 345 | "delete) is intact via every read path -- no byte of truth lives " |
| 346 | "outside the .tsv file." |
| 347 | ) |
| 348 | |
| 349 | |
| 350 | # ============================================================================= |
| 351 | # 3. READ-WITH-STANDARD-TOOLS |
| 352 | # ============================================================================= |
| 353 | |
| 354 | |
| 355 | def test_read_with_standard_tools_self_describing_and_csv_parseable(tmp_path): |
| 356 | """The archivist-in-2125 test: open records.tsv with nothing but |
| 357 | Python's stdlib `csv` module (delimiter='\\t'), independent of |
| 358 | object_records, and confirm (a) the header row names every column |
| 359 | (self-describing -- no external schema needed to know what a column |
| 360 | IS, even though a schema file separately governs validation), and (b) |
| 361 | every data row parses cleanly and reconstructs the exact field values, |
| 362 | for both classic and append storage.""" |
| 363 | data_dir = tmp_path / "data" |
| 364 | |
| 365 | write_schema(data_dir, "std_classic", ALL_FIELDS) |
| 366 | write_append_schema(data_dir, "std_append", ALL_FIELDS) |
| 367 | |
| 368 | records = [_exotic_payload(str(i), 4000 + i) for i in range(5)] |
| 369 | for collection in ("std_classic", "std_append"): |
| 370 | for rec in records: |
| 371 | object_records.create_collection_record(collection, rec, base_dir=data_dir, roots=[]) |
| 372 | |
| 373 | for collection, expect_op_column in (("std_classic", False), ("std_append", True)): |
| 374 | path = _records_tsv_path(data_dir, collection) |
| 375 | with path.open(newline="") as handle: |
| 376 | reader = csv.reader(handle, delimiter="\t") |
| 377 | rows = list(reader) |
| 378 | |
| 379 | header, data_rows = rows[0], rows[1:] |
| 380 | print(f"\n[READ-WITH-STANDARD-TOOLS / {collection}] header: {header}") |
| 381 | |
| 382 | if expect_op_column: |
| 383 | assert header[0] == "_op" |
| 384 | logical_header = header[1:] |
| 385 | else: |
| 386 | logical_header = header |
| 387 | |
| 388 | # Self-describing: the header names exactly the fields we wrote |
| 389 | # (order-independent check on set membership, order-sensitive on |
| 390 | # "id" being present at all). |
| 391 | assert set(logical_header) == set(LOGICAL_FIELD_NAMES), ( |
| 392 | f"{collection}: header does not name the columns we wrote: {logical_header!r}" |
| 393 | ) |
| 394 | assert len(data_rows) == len(records), ( |
| 395 | f"{collection}: expected {len(records)} data rows, got {len(data_rows)}" |
| 396 | ) |
| 397 | |
| 398 | recovered_by_id = {} |
| 399 | for row in data_rows: |
| 400 | assert len(row) == len(header), ( |
| 401 | f"{collection}: row does not match header column count: {row!r} vs {header!r}" |
| 402 | ) |
| 403 | as_dict = dict(zip(header, row)) |
| 404 | if expect_op_column: |
| 405 | assert as_dict.pop("_op") == "" # every row here is a plain upsert |
| 406 | recovered_by_id[as_dict["id"]] = as_dict |
| 407 | |
| 408 | for rec in records: |
| 409 | assert recovered_by_id[rec["id"]] == rec, ( |
| 410 | f"{collection}: naive csv.reader recovery of {rec['id']!r} does not match " |
| 411 | f"what was written.\n wrote: {rec!r}\n recovered: {recovered_by_id[rec['id']]!r}" |
| 412 | ) |
| 413 | |
| 414 | print( |
| 415 | "\n[READ-WITH-STANDARD-TOOLS] PRINCIPLE HOLDS: both classic- and " |
| 416 | "append-mode records.tsv files are well-formed tab-delimited CSV " |
| 417 | "with a self-describing header, and every exotic-content field " |
| 418 | "(tab/newline/quote/backslash/Unicode/CRLF/bare-CR/commas/empty) " |
| 419 | "recovers exactly via a plain csv.reader -- no object_records code " |
| 420 | "involved in this parse." |
| 421 | ) |
| 422 | |
| 423 | |
| 424 | # ============================================================================= |
| 425 | # 4. ARCHIVAL round-trip -- fully naive, zero DBBASIC code on the READ side |
| 426 | # ============================================================================= |
| 427 | |
| 428 | |
| 429 | def _naive_read_classic_tsv(path: Path) -> list[dict]: |
| 430 | """A hand-rolled reader that owes NOTHING to object_records: stdlib |
| 431 | `csv` only, using the header row itself to know the field names (the |
| 432 | only "schema" this function is given). This is what an archivist with |
| 433 | no access to this codebase, in some future year, would write in an |
| 434 | afternoon to recover the data.""" |
| 435 | with path.open(newline="") as handle: |
| 436 | reader = csv.DictReader(handle, delimiter="\t") |
| 437 | return list(reader) |
| 438 | |
| 439 | |
| 440 | def _naive_read_append_tsv_with_manual_fold(path: Path) -> dict[str, dict]: |
| 441 | """Same as above, but additionally applies the append-only op |
| 442 | semantics documented in plain English in object_records.py's own |
| 443 | comments -- op "" upserts, op "del" tombstones, last physical row for |
| 444 | an id wins -- reimplemented here with no reference to any |
| 445 | object_records internals, to show even append-mode data does not |
| 446 | require DBBASIC's own code to recover, only reading the two |
| 447 | human-documented op values in order.""" |
| 448 | live: dict[str, dict] = {} |
| 449 | with path.open(newline="") as handle: |
| 450 | reader = csv.DictReader(handle, delimiter="\t") |
| 451 | for row in reader: |
| 452 | op = row.pop("_op") |
| 453 | rid = row["id"] |
| 454 | if op == "del": |
| 455 | live.pop(rid, None) |
| 456 | else: |
| 457 | live[rid] = row |
| 458 | return live |
| 459 | |
| 460 | |
| 461 | def test_archival_round_trip_naive_reader_classic_mode(tmp_path): |
| 462 | """Write via object_records (realistic production write path), then |
| 463 | read back with ONLY the naive external reader above -- zero calls to |
| 464 | object_records for the read side.""" |
| 465 | data_dir = tmp_path / "data" |
| 466 | collection = "archival_classic" |
| 467 | write_schema(data_dir, collection, ALL_FIELDS) |
| 468 | |
| 469 | records = [_exotic_payload(str(i), 5000 + i) for i in range(7)] |
| 470 | for rec in records: |
| 471 | object_records.create_collection_record(collection, rec, base_dir=data_dir, roots=[]) |
| 472 | |
| 473 | path = _records_tsv_path(data_dir, collection) |
| 474 | recovered = _naive_read_classic_tsv(path) |
| 475 | recovered_by_id = {r["id"]: r for r in recovered} |
| 476 | |
| 477 | assert len(recovered) == len(records) |
| 478 | for rec in records: |
| 479 | assert recovered_by_id[rec["id"]] == rec, ( |
| 480 | f"naive external round-trip failed for {rec['id']!r}: " |
| 481 | f"wrote {rec!r}, naive-recovered {recovered_by_id[rec['id']]!r}" |
| 482 | ) |
| 483 | |
| 484 | print( |
| 485 | "\n[ARCHIVAL round-trip / classic] PRINCIPLE HOLDS: a csv.DictReader " |
| 486 | "with zero DBBASIC code recovers every record exactly, including " |
| 487 | "every hostile-content field." |
| 488 | ) |
| 489 | |
| 490 | |
| 491 | def test_archival_round_trip_naive_reader_append_mode_with_update_and_delete(tmp_path): |
| 492 | """The stronger version: append-mode file, with a real update |
| 493 | (superseding row) and a real delete (tombstone) baked in, recovered |
| 494 | end-to-end by a hand-rolled fold that uses nothing but the two |
| 495 | documented op values -- proving the append-only format itself needs no |
| 496 | proprietary reader, just the (short, human-readable) semantics |
| 497 | documented in object_records.py's own comments.""" |
| 498 | data_dir = tmp_path / "data" |
| 499 | collection = "archival_append" |
| 500 | write_append_schema(data_dir, collection, ALL_FIELDS) |
| 501 | |
| 502 | records = [_exotic_payload(str(i), 6000 + i) for i in range(6)] |
| 503 | for rec in records: |
| 504 | object_records.create_collection_record(collection, rec, base_dir=data_dir, roots=[]) |
| 505 | |
| 506 | updated_note = "ARCHIVAL-UPDATED: café \t\n\"\\ 日本" |
| 507 | object_records.update_collection_record( |
| 508 | collection, "rec-2", {"note": updated_note}, base_dir=data_dir, roots=[] |
| 509 | ) |
| 510 | object_records.delete_collection_record(collection, "rec-5", base_dir=data_dir, roots=[]) |
| 511 | |
| 512 | path = _records_tsv_path(data_dir, collection) |
| 513 | recovered = _naive_read_append_tsv_with_manual_fold(path) |
| 514 | |
| 515 | assert "rec-5" not in recovered, "naive fold failed to honor the 'del' tombstone" |
| 516 | assert set(recovered) == {"rec-0", "rec-1", "rec-2", "rec-3", "rec-4"} |
| 517 | assert recovered["rec-2"]["note"] == updated_note, ( |
| 518 | "naive fold failed to let the later physical row for rec-2 win" |
| 519 | ) |
| 520 | for rid, rec in recovered.items(): |
| 521 | if rid == "rec-2": |
| 522 | continue |
| 523 | expected = next(r for r in records if r["id"] == rid) |
| 524 | assert rec == expected, f"naive fold mismatch for {rid!r}: {rec!r} vs {expected!r}" |
| 525 | |
| 526 | print( |
| 527 | "\n[ARCHIVAL round-trip / append, update+delete] PRINCIPLE HOLDS: " |
| 528 | "a hand-rolled fold using only the two documented op values (\"\" " |
| 529 | "upsert / \"del\" tombstone) and a csv.DictReader recovers the " |
| 530 | "correct live dataset -- update and delete included -- with zero " |
| 531 | "object_records code involved." |
| 532 | ) |