askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_durability_backup_compaction_characterization.py · 37.5 KB · 841 lines
| 1 | """CHARACTERIZATION tests: does BACKUP/RESTORE and COMPACTION preserve data |
| 2 | with byte-exact fidelity? |
| 3 | |
| 4 | This file is pure characterization of EXISTING behavior. It does not modify |
| 5 | any production module (object_records.py, object_backup.py, |
| 6 | object_backup_index.py, object_daemon.py are all untouched). Where a case |
| 7 | fails or corrupts data, the assertion states the correctness property a safe |
| 8 | durability substrate SHOULD have and is left to fail -- that failure IS the |
| 9 | finding. Do not "fix" a failing assertion here by weakening it to match |
| 10 | broken behavior. |
| 11 | |
| 12 | Context (see tests/test_embedded_json_lines_characterization.py for the |
| 13 | sibling investigation this one continues): an append-mode collection |
| 14 | accumulates superseded/tombstoned physical rows as it is written to, and is |
| 15 | periodically COMPACTED -- folded down to just its live rows -- either on |
| 16 | demand (object_records.compact_collection, exercised directly here) or via |
| 17 | object_daemon.py's process_compactions poller (not imported here; this file |
| 18 | stays at the object_records.compact_collection layer, which is what the |
| 19 | daemon itself calls). Separately, the whole runtime -- including every |
| 20 | collection's records.tsv, live or append-mode -- can be BACKED UP (a tar/gz |
| 21 | archive: object_backup.create_runtime_backup / |
| 22 | create_runtime_restore_point) and RESTORED into a fresh directory |
| 23 | (object_backup.restore_runtime_backup). A third module, |
| 24 | object_backup_index.py, does NOT participate in backup/restore itself -- |
| 25 | it only offers read-only preview/diff of a backup's records against live |
| 26 | data (preview_collection/preview_record) -- but it carries its OWN local |
| 27 | copy of a quote-blind "torn tail" check (_drop_torn_tail), independently |
| 28 | reimplemented from object_records._drop_torn_tail (documented KNOWN |
| 29 | LIMITATION / substrate bug #2 there: a crash landing right after a newline |
| 30 | that is INSIDE an open quoted multi-line cell leaves the file looking |
| 31 | "terminated" -- ends with \\n -- even though the row never actually |
| 32 | finished, so the naive "ends with \\n" check wrongly calls it committed). |
| 33 | Part of this file's job is to confirm whether that same latent bug is |
| 34 | reachable through object_backup_index's copy of the check against a backed |
| 35 | up (not just a live) records.tsv. |
| 36 | """ |
| 37 | |
| 38 | from __future__ import annotations |
| 39 | |
| 40 | import csv |
| 41 | import io |
| 42 | import json |
| 43 | import shutil |
| 44 | import tarfile |
| 45 | from pathlib import Path |
| 46 | |
| 47 | import pytest |
| 48 | |
| 49 | import object_backup |
| 50 | import object_backup_index |
| 51 | import object_records |
| 52 | |
| 53 | pytestmark = pytest.mark.conformance |
| 54 | |
| 55 | |
| 56 | ID_FIELD = {"name": "id"} |
| 57 | NOTE_FIELD = {"name": "note", "type": "textarea"} |
| 58 | LINES_FIELD = {"name": "lines", "type": "textarea"} |
| 59 | |
| 60 | |
| 61 | # --- setup helpers (mirror tests/test_object_records.py / |
| 62 | # tests/test_embedded_json_lines_characterization.py conventions) ------- |
| 63 | |
| 64 | |
| 65 | def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 66 | path = data_dir / "schemas" / f"{collection}.json" |
| 67 | path.parent.mkdir(parents=True, exist_ok=True) |
| 68 | path.write_text(json.dumps({"fields": fields})) |
| 69 | return path |
| 70 | |
| 71 | |
| 72 | def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 73 | path = data_dir / "schemas" / f"{collection}.json" |
| 74 | path.parent.mkdir(parents=True, exist_ok=True) |
| 75 | path.write_text(json.dumps({"fields": fields, "storage": "append"})) |
| 76 | return path |
| 77 | |
| 78 | |
| 79 | def _clear_caches() -> None: |
| 80 | """Force the NEXT read past the warm in-process caches (see the sibling |
| 81 | embedded-JSON characterization file for why this matters for by-id |
| 82 | reads specifically).""" |
| 83 | object_records._RECORDS_CACHE.clear() |
| 84 | object_records._OIDX_CACHE.clear() |
| 85 | |
| 86 | |
| 87 | def _records_path(data_dir: Path, collection: str) -> Path: |
| 88 | return data_dir / "collections" / collection / "records.tsv" |
| 89 | |
| 90 | |
| 91 | def _exotic_corpus() -> list[dict]: |
| 92 | """A grab-bag of hostile-but-legal field values: ASCII, multibyte, |
| 93 | emoji (astral-plane, so a surrogate pair in UTF-16 but a single scalar |
| 94 | in UTF-8), embedded compact JSON, embedded PRETTY-PRINTED JSON (real |
| 95 | raw newline bytes in the cell -- the case that actually stresses the |
| 96 | CSV-aware row-boundary machinery), a large field, and a bare literal |
| 97 | CR (0x0D, not part of a CRLF pair) -- which the CSV writer must quote |
| 98 | to round-trip safely (an unquoted bare CR is not a delimiter itself |
| 99 | but is exactly the kind of "surprising control byte" this substrate's |
| 100 | hostile-content tests exist to check).""" |
| 101 | return [ |
| 102 | { |
| 103 | "id": "r-ascii", |
| 104 | "note": "plain ascii, no surprises", |
| 105 | "lines": json.dumps([{"sku": "A-1", "qty": 1}]), |
| 106 | }, |
| 107 | { |
| 108 | "id": "r-multibyte", |
| 109 | "note": "café naïve résumé 日本語 中文", |
| 110 | "lines": json.dumps([{"sku": "日本-1", "note": "中文注释"}]), |
| 111 | }, |
| 112 | { |
| 113 | "id": "r-emoji", |
| 114 | "note": "party \U0001f389 rocket \U0001f680 family \U0001f468\U0001f469\U0001f467", |
| 115 | "lines": json.dumps([{"sku": "EMOJI-1", "label": "\U0001f525 hot"}]), |
| 116 | }, |
| 117 | { |
| 118 | "id": "r-embedded-json-compact", |
| 119 | "note": "compact json blob (no raw control bytes)", |
| 120 | "lines": json.dumps( |
| 121 | [{"sku": f"SKU-{i:03d}", "qty": i, "note": "x\ty\nz\"q\"\\w"} for i in range(20)] |
| 122 | ), |
| 123 | }, |
| 124 | { |
| 125 | "id": "r-embedded-json-pretty", |
| 126 | "note": "pretty json blob -- REAL raw newline bytes in this cell", |
| 127 | "lines": json.dumps( |
| 128 | [{"sku": f"SKU-{i:03d}", "qty": i} for i in range(10)], indent=2 |
| 129 | ), |
| 130 | }, |
| 131 | { |
| 132 | "id": "r-bare-cr", |
| 133 | "note": "alpha\rbeta\rgamma", # bare CR, not CRLF -- must be quoted to survive |
| 134 | "lines": json.dumps([{"sku": "CR-1"}]), |
| 135 | }, |
| 136 | { |
| 137 | "id": "r-crlf", |
| 138 | "note": "alpha\r\nbeta\r\ngamma", |
| 139 | "lines": json.dumps([{"sku": "CRLF-1"}]), |
| 140 | }, |
| 141 | { |
| 142 | "id": "r-large", |
| 143 | "note": "large field", |
| 144 | "lines": json.dumps([{"sku": f"BIG-{i:05d}", "qty": i, "price": 1099 + i} for i in range(2000)]), |
| 145 | }, |
| 146 | ] |
| 147 | |
| 148 | |
| 149 | # ============================================================================= |
| 150 | # 1. COMPACTION FIDELITY -- live records survive byte-exact, dead rows gone |
| 151 | # ============================================================================= |
| 152 | |
| 153 | |
| 154 | def test_compaction_preserves_live_records_byte_exact_including_exotic_content(tmp_path): |
| 155 | """Build an append-mode collection with a realistic mix of creates, |
| 156 | superseding updates, and deletes (tombstones) over the exotic corpus, |
| 157 | compact it, and check every angle of "did compaction preserve exactly |
| 158 | the live set, byte-exact, and nothing else": |
| 159 | |
| 160 | - every live record's field values are byte-for-byte identical to |
| 161 | what a full-fold read reported immediately BEFORE compaction |
| 162 | - superseded/tombstoned rows are gone from the physical file |
| 163 | - the physical row count actually dropped |
| 164 | - a read after compaction returns exactly the live set (no more, no |
| 165 | less) |
| 166 | - by-id reads still work for every live id post-compaction, via the |
| 167 | REBUILT sidecar (cold caches, forcing the id->offset oidx path) |
| 168 | """ |
| 169 | data_dir = tmp_path / "data" |
| 170 | write_append_schema(data_dir, "docs", [ID_FIELD, NOTE_FIELD, LINES_FIELD]) |
| 171 | |
| 172 | corpus = _exotic_corpus() |
| 173 | for rec in corpus: |
| 174 | object_records.create_collection_record("docs", dict(rec), base_dir=data_dir, roots=[]) |
| 175 | |
| 176 | # Superseding updates on a few ids (multiple times each -- each update |
| 177 | # is a brand-new physical row in append mode, leaving its predecessors |
| 178 | # superseded-but-not-yet-compacted). |
| 179 | object_records.update_collection_record( |
| 180 | "docs", "r-ascii", {"note": "ascii v2"}, base_dir=data_dir, roots=[] |
| 181 | ) |
| 182 | object_records.update_collection_record( |
| 183 | "docs", "r-ascii", {"note": "ascii v3 final"}, base_dir=data_dir, roots=[] |
| 184 | ) |
| 185 | object_records.update_collection_record( |
| 186 | "docs", "r-multibyte", {"note": "café v2 日本語"}, base_dir=data_dir, roots=[] |
| 187 | ) |
| 188 | object_records.update_collection_record( |
| 189 | "docs", |
| 190 | "r-embedded-json-pretty", |
| 191 | {"lines": json.dumps([{"sku": "REPLACED", "qty": 999}], indent=2)}, |
| 192 | base_dir=data_dir, |
| 193 | roots=[], |
| 194 | ) |
| 195 | |
| 196 | # Tombstone a couple of ids. |
| 197 | object_records.delete_collection_record("docs", "r-crlf", base_dir=data_dir, roots=[]) |
| 198 | object_records.delete_collection_record("docs", "r-large", base_dir=data_dir, roots=[]) |
| 199 | |
| 200 | # Capture the exact live set (full-fold read) immediately before |
| 201 | # compaction: this is the ground truth compaction must reproduce. |
| 202 | _clear_caches() |
| 203 | live_before = { |
| 204 | r["id"]: r |
| 205 | for r in object_records.read_collection_records("docs", base_dir=data_dir, roots=[]) |
| 206 | } |
| 207 | expected_ids = {"r-ascii", "r-multibyte", "r-emoji", "r-embedded-json-compact", |
| 208 | "r-embedded-json-pretty", "r-bare-cr"} |
| 209 | assert set(live_before) == expected_ids |
| 210 | assert live_before["r-ascii"]["note"] == "ascii v3 final" |
| 211 | assert live_before["r-multibyte"]["note"] == "café v2 日本語" |
| 212 | assert json.loads(live_before["r-embedded-json-pretty"]["lines"]) == [ |
| 213 | {"sku": "REPLACED", "qty": 999} |
| 214 | ] |
| 215 | assert live_before["r-bare-cr"]["note"] == "alpha\rbeta\rgamma" |
| 216 | |
| 217 | path = _records_path(data_dir, "docs") |
| 218 | with path.open(newline="") as fh: |
| 219 | physical_rows_before = sum(1 for _ in csv.reader(fh, delimiter="\t")) - 1 # minus header |
| 220 | raw_before = path.read_text() |
| 221 | assert "\tdel\t" in raw_before or raw_before.count("del") >= 1 # tombstones present |
| 222 | |
| 223 | result = object_records.compact_collection("docs", base_dir=data_dir, roots=[]) |
| 224 | |
| 225 | # --- physical row count actually dropped --- |
| 226 | assert result["rows_before"] == physical_rows_before |
| 227 | assert result["rows_after"] == len(expected_ids), ( |
| 228 | f"compacted physical row count should equal the live set size; " |
| 229 | f"got {result}" |
| 230 | ) |
| 231 | assert result["rows_after"] < result["rows_before"] |
| 232 | assert result["bytes_after"] < result["bytes_before"] |
| 233 | |
| 234 | # --- superseded/tombstoned rows gone from the physical file --- |
| 235 | raw_after = path.read_text() |
| 236 | assert "del" not in raw_after.split("\n")[0] # header sanity |
| 237 | for line in raw_after.splitlines()[1:]: |
| 238 | assert not line.startswith("del\t"), f"tombstone row survived compaction: {line!r}" |
| 239 | assert "r-crlf" not in raw_after |
| 240 | assert "r-large" not in raw_after |
| 241 | assert "ascii v2" not in raw_after # superseded value must not linger anywhere on disk |
| 242 | assert "REPLACED" not in raw_after or json.loads(live_before["r-embedded-json-pretty"]["lines"]) |
| 243 | |
| 244 | # --- read after compaction returns exactly the live set, byte-exact --- |
| 245 | _clear_caches() |
| 246 | live_after = { |
| 247 | r["id"]: r |
| 248 | for r in object_records.read_collection_records("docs", base_dir=data_dir, roots=[]) |
| 249 | } |
| 250 | assert set(live_after) == expected_ids |
| 251 | for record_id in expected_ids: |
| 252 | assert live_after[record_id] == live_before[record_id], ( |
| 253 | f"compaction changed live content for {record_id!r}: " |
| 254 | f"before={live_before[record_id]!r} after={live_after[record_id]!r}" |
| 255 | ) |
| 256 | |
| 257 | # --- by-id reads still work for every live id, sidecar rebuilt --- |
| 258 | for record_id in expected_ids: |
| 259 | _clear_caches() # cold cache + cold sidecar -> forces oidx rebuild from scratch |
| 260 | by_id = object_records.get_collection_record("docs", record_id, base_dir=data_dir, roots=[]) |
| 261 | assert by_id == live_before[record_id], ( |
| 262 | f"post-compaction by-id read (rebuilt sidecar) diverged for {record_id!r}: " |
| 263 | f"{by_id!r} != {live_before[record_id]!r}" |
| 264 | ) |
| 265 | |
| 266 | # --- deleted ids are genuinely gone, not just excluded from listing --- |
| 267 | for dead_id in ("r-crlf", "r-large"): |
| 268 | _clear_caches() |
| 269 | with pytest.raises(object_records.RecordNotFoundError): |
| 270 | object_records.get_collection_record("docs", dead_id, base_dir=data_dir, roots=[]) |
| 271 | |
| 272 | |
| 273 | def test_compaction_of_never_touched_and_all_deleted_collection_is_honest(tmp_path): |
| 274 | """Edge cases around the main fidelity test: compacting a collection |
| 275 | where EVERY record has since been deleted must leave zero live rows |
| 276 | (not resurrect anything), and compacting twice in a row must be a |
| 277 | stable no-op the second time.""" |
| 278 | data_dir = tmp_path / "data" |
| 279 | write_append_schema(data_dir, "ephemeral", [ID_FIELD, NOTE_FIELD]) |
| 280 | |
| 281 | for i in range(5): |
| 282 | object_records.create_collection_record( |
| 283 | "ephemeral", {"id": f"e{i}", "note": f"note-{i}"}, base_dir=data_dir, roots=[] |
| 284 | ) |
| 285 | for i in range(5): |
| 286 | object_records.delete_collection_record("ephemeral", f"e{i}", base_dir=data_dir, roots=[]) |
| 287 | |
| 288 | result = object_records.compact_collection("ephemeral", base_dir=data_dir, roots=[]) |
| 289 | assert result["rows_after"] == 0 |
| 290 | |
| 291 | _clear_caches() |
| 292 | listing = object_records.list_collection_records("ephemeral", base_dir=data_dir, roots=[])["records"] |
| 293 | assert listing == [] |
| 294 | |
| 295 | # Compacting an already-fully-compacted (here: empty) collection again |
| 296 | # must be a stable no-op. |
| 297 | result2 = object_records.compact_collection("ephemeral", base_dir=data_dir, roots=[]) |
| 298 | assert result2["rows_before"] == 0 |
| 299 | assert result2["rows_after"] == 0 |
| 300 | |
| 301 | |
| 302 | # ============================================================================= |
| 303 | # 2. COMPACTION CRASH-SAFETY -- interrupted compaction must not lose data |
| 304 | # ============================================================================= |
| 305 | |
| 306 | |
| 307 | def test_compaction_interrupted_before_atomic_replace_leaves_original_untouched(tmp_path, monkeypatch): |
| 308 | """compact_collection rewrites via _write_collection_records, which |
| 309 | writes the FOLDED content to a fresh temp file and only then does an |
| 310 | atomic `temp_path.replace(path)` -- so a crash at any point BEFORE |
| 311 | that replace call must leave the original records.tsv completely |
| 312 | untouched (the classic "write-new, then atomically swap" durability |
| 313 | property). Verified here by monkeypatching Path.replace to raise |
| 314 | partway through a real compact_collection() call, simulating a process |
| 315 | crash/OSError at exactly that instant, then confirming: |
| 316 | |
| 317 | 1. compact_collection's exception propagates (it does not silently |
| 318 | swallow the failure and report success) |
| 319 | 2. the original records.tsv is byte-for-byte unchanged from before |
| 320 | the attempted compaction (superseded rows and tombstones intact |
| 321 | -- nothing was lost, nothing was half-written) |
| 322 | 3. a normal read through the public API still returns the correct, |
| 323 | complete pre-compaction live set |
| 324 | 4. once the injected failure is removed, retrying compact_collection |
| 325 | succeeds and produces the correct compacted result -- i.e. the |
| 326 | interrupted attempt did not leave the collection in a state that |
| 327 | a follow-up compaction can't recover from |
| 328 | """ |
| 329 | data_dir = tmp_path / "data" |
| 330 | write_append_schema(data_dir, "orders", [ID_FIELD, NOTE_FIELD]) |
| 331 | object_records.create_collection_record( |
| 332 | "orders", {"id": "o1", "note": "v1"}, base_dir=data_dir, roots=[] |
| 333 | ) |
| 334 | object_records.update_collection_record( |
| 335 | "orders", "o1", {"note": "v2"}, base_dir=data_dir, roots=[] |
| 336 | ) |
| 337 | object_records.update_collection_record( |
| 338 | "orders", "o1", {"note": "v3"}, base_dir=data_dir, roots=[] |
| 339 | ) |
| 340 | object_records.create_collection_record( |
| 341 | "orders", {"id": "o2", "note": "keep-me"}, base_dir=data_dir, roots=[] |
| 342 | ) |
| 343 | object_records.delete_collection_record("orders", "o2", base_dir=data_dir, roots=[]) |
| 344 | |
| 345 | path = _records_path(data_dir, "orders") |
| 346 | original_bytes = path.read_bytes() |
| 347 | |
| 348 | _clear_caches() |
| 349 | live_before = { |
| 350 | r["id"]: r |
| 351 | for r in object_records.read_collection_records("orders", base_dir=data_dir, roots=[]) |
| 352 | } |
| 353 | assert live_before == {"o1": {"id": "o1", "note": "v3"}} |
| 354 | |
| 355 | real_replace = Path.replace |
| 356 | state = {"armed": True} |
| 357 | |
| 358 | def _boom_replace(self, target): |
| 359 | # temp_path is built as f".{path.name}.{pid}.{tid}.tmp" alongside |
| 360 | # the real records.tsv -- i.e. ".records.tsv.<pid>.<tid>.tmp" in |
| 361 | # the "orders" collection's own directory (the collection name is |
| 362 | # not itself part of the filename, only the parent directory). |
| 363 | if ( |
| 364 | state["armed"] |
| 365 | and self.name.startswith(".records.tsv.") |
| 366 | and self.name.endswith(".tmp") |
| 367 | and self.parent.name == "orders" |
| 368 | ): |
| 369 | state["armed"] = False |
| 370 | raise OSError("simulated crash: process died mid-compaction, before atomic replace") |
| 371 | return real_replace(self, target) |
| 372 | |
| 373 | monkeypatch.setattr(Path, "replace", _boom_replace) |
| 374 | |
| 375 | with pytest.raises(OSError, match="simulated crash"): |
| 376 | object_records.compact_collection("orders", base_dir=data_dir, roots=[]) |
| 377 | |
| 378 | # 2. original file byte-for-byte unchanged. |
| 379 | assert path.read_bytes() == original_bytes, ( |
| 380 | "FINDING: an interrupted compaction (crash before the atomic " |
| 381 | "replace) altered the on-disk records.tsv -- compaction is " |
| 382 | "destructive-before-commit." |
| 383 | ) |
| 384 | |
| 385 | # 3. a normal read still returns the correct pre-compaction live set. |
| 386 | _clear_caches() |
| 387 | live_during_outage = { |
| 388 | r["id"]: r |
| 389 | for r in object_records.read_collection_records("orders", base_dir=data_dir, roots=[]) |
| 390 | } |
| 391 | assert live_during_outage == live_before |
| 392 | |
| 393 | # 4. retry (failure no longer armed) succeeds and recovers correctly. |
| 394 | monkeypatch.setattr(Path, "replace", real_replace) |
| 395 | result = object_records.compact_collection("orders", base_dir=data_dir, roots=[]) |
| 396 | assert result["rows_after"] == 1 |
| 397 | |
| 398 | _clear_caches() |
| 399 | live_after_retry = { |
| 400 | r["id"]: r |
| 401 | for r in object_records.read_collection_records("orders", base_dir=data_dir, roots=[]) |
| 402 | } |
| 403 | assert live_after_retry == live_before |
| 404 | |
| 405 | |
| 406 | def test_compaction_stray_orphaned_temp_file_is_inert_and_not_resurrected(tmp_path): |
| 407 | """A DIFFERENT crash shape: a previous compaction (or previous process |
| 408 | incarnation, different pid) died after creating its temp file but |
| 409 | before either writing to it fully or replacing the original -- leaving |
| 410 | an orphaned `.records.tsv.<pid>.<tid>.tmp` file sitting next to the |
| 411 | real records.tsv forever (nothing in this codebase's write path scans |
| 412 | for or cleans up a stale temp file from a DIFFERENT pid/tid on |
| 413 | startup). Characterizes: does that debris get accidentally treated as |
| 414 | data by any read path, and does a subsequent real compaction still |
| 415 | produce the correct result despite the debris being present?""" |
| 416 | data_dir = tmp_path / "data" |
| 417 | write_append_schema(data_dir, "invoices", [ID_FIELD, NOTE_FIELD]) |
| 418 | object_records.create_collection_record( |
| 419 | "invoices", {"id": "i1", "note": "one"}, base_dir=data_dir, roots=[] |
| 420 | ) |
| 421 | object_records.update_collection_record( |
| 422 | "invoices", "i1", {"note": "one-v2"}, base_dir=data_dir, roots=[] |
| 423 | ) |
| 424 | object_records.create_collection_record( |
| 425 | "invoices", {"id": "i2", "note": "two"}, base_dir=data_dir, roots=[] |
| 426 | ) |
| 427 | |
| 428 | path = _records_path(data_dir, "invoices") |
| 429 | original_bytes = path.read_bytes() |
| 430 | |
| 431 | # Plant debris matching the real naming convention but with a bogus |
| 432 | # pid/tid, containing garbage that -- if it were ever misread as real |
| 433 | # data -- would be obviously wrong. |
| 434 | stray = path.with_name(f".{path.name}.999999.999999.tmp") |
| 435 | stray.write_text("GARBAGE-NOT-A-VALID-TSV-ROW\t\t\n") |
| 436 | assert stray.exists() |
| 437 | |
| 438 | _clear_caches() |
| 439 | listing = object_records.list_collection_records("invoices", base_dir=data_dir, roots=[])["records"] |
| 440 | assert listing == [{"id": "i1", "note": "one-v2"}, {"id": "i2", "note": "two"}], ( |
| 441 | "a stray orphaned .tmp file was picked up by a read path" |
| 442 | ) |
| 443 | assert path.read_bytes() == original_bytes # debris presence didn't touch the real file |
| 444 | |
| 445 | result = object_records.compact_collection("invoices", base_dir=data_dir, roots=[]) |
| 446 | assert result["rows_after"] == 2 |
| 447 | |
| 448 | _clear_caches() |
| 449 | listing_after = object_records.list_collection_records("invoices", base_dir=data_dir, roots=[])["records"] |
| 450 | assert listing_after == [{"id": "i1", "note": "one-v2"}, {"id": "i2", "note": "two"}] |
| 451 | |
| 452 | # Report (not asserted pass/fail either way -- purely descriptive): |
| 453 | # does a real compaction clean up debris left by an unrelated pid/tid? |
| 454 | print(f"\n[compaction] stray orphaned temp file still present after a real " |
| 455 | f"compact_collection() call: {stray.exists()}") |
| 456 | |
| 457 | |
| 458 | # ============================================================================= |
| 459 | # 3. BACKUP / RESTORE FIDELITY -- byte-exact round trip |
| 460 | # ============================================================================= |
| 461 | |
| 462 | |
| 463 | def _make_source_runtime(base: Path) -> tuple[Path, Path]: |
| 464 | """Build a runtime layout (objects_dir + data_dir) with one classic-mode |
| 465 | and one append-mode collection, both carrying the full exotic corpus, |
| 466 | plus superseded/tombstoned rows in the append one (so the backed-up |
| 467 | file is a realistic append-log, not just a clean fold).""" |
| 468 | objects_dir = base / "objects" |
| 469 | data_dir = base / "data" |
| 470 | objects_dir.mkdir(parents=True, exist_ok=True) |
| 471 | (objects_dir / "keepme.txt").write_text("just to give the objects tree a file\n") |
| 472 | |
| 473 | write_schema(data_dir, "notes_classic", [ID_FIELD, NOTE_FIELD, LINES_FIELD]) |
| 474 | write_append_schema(data_dir, "notes_append", [ID_FIELD, NOTE_FIELD, LINES_FIELD]) |
| 475 | |
| 476 | corpus = _exotic_corpus() |
| 477 | for rec in corpus: |
| 478 | object_records.create_collection_record("notes_classic", dict(rec), base_dir=data_dir, roots=[]) |
| 479 | object_records.create_collection_record("notes_append", dict(rec), base_dir=data_dir, roots=[]) |
| 480 | |
| 481 | # Give the append collection some churn so its physical file is a real |
| 482 | # log (superseded rows + a tombstone), not already-compacted. |
| 483 | object_records.update_collection_record( |
| 484 | "notes_append", "r-ascii", {"note": "ascii v2"}, base_dir=data_dir, roots=[] |
| 485 | ) |
| 486 | object_records.update_collection_record( |
| 487 | "notes_append", "r-ascii", {"note": "ascii v3"}, base_dir=data_dir, roots=[] |
| 488 | ) |
| 489 | object_records.delete_collection_record("notes_append", "r-bare-cr", base_dir=data_dir, roots=[]) |
| 490 | |
| 491 | return objects_dir, data_dir |
| 492 | |
| 493 | |
| 494 | def test_backup_restore_byte_exact_round_trip_full_exotic_corpus(tmp_path): |
| 495 | """Back up a runtime carrying the full exotic corpus in both classic |
| 496 | and append storage, restore into a FRESH directory, and assert: |
| 497 | |
| 498 | - every collection's records.tsv is byte-for-byte identical, |
| 499 | original vs. restored (the strongest possible fidelity check -- |
| 500 | the backup mechanism is a raw tar copy, so this should hold |
| 501 | trivially, but is worth confirming rather than assumed) |
| 502 | - every collection's OBJECTS-DIR sibling file round-trips too |
| 503 | - every record, read back through the real object_records API |
| 504 | against the restored data_dir, matches the original exactly |
| 505 | - the restored append-mode file is still a VALID, parseable |
| 506 | append-log (correct live/dead fold), not just byte-identical |
| 507 | """ |
| 508 | src_root = tmp_path / "src" |
| 509 | objects_dir, data_dir = _make_source_runtime(src_root) |
| 510 | |
| 511 | backup_output = tmp_path / "backups" / "full.tar.gz" |
| 512 | summary = object_backup.create_runtime_backup( |
| 513 | backup_output, objects_dir=objects_dir, data_dir=data_dir |
| 514 | ) |
| 515 | assert summary.files > 0 |
| 516 | assert backup_output.exists() |
| 517 | |
| 518 | restore_root = tmp_path / "restored" |
| 519 | restored_objects = restore_root / "objects" |
| 520 | restored_data = restore_root / "data" |
| 521 | restore_summary = object_backup.restore_runtime_backup( |
| 522 | backup_output, |
| 523 | objects_dir=restored_objects, |
| 524 | data_dir=restored_data, |
| 525 | overwrite=True, |
| 526 | ) |
| 527 | assert restore_summary.files == summary.files |
| 528 | |
| 529 | # --- byte-exact records.tsv round trip, both storage modes --- |
| 530 | for collection in ("notes_classic", "notes_append"): |
| 531 | original = _records_path(data_dir, collection).read_bytes() |
| 532 | restored = _records_path(restored_data, collection).read_bytes() |
| 533 | assert restored == original, ( |
| 534 | f"{collection}: restored records.tsv is not byte-identical to the original" |
| 535 | ) |
| 536 | |
| 537 | # --- objects dir sibling file round trip --- |
| 538 | assert (restored_objects / "keepme.txt").read_bytes() == (objects_dir / "keepme.txt").read_bytes() |
| 539 | |
| 540 | # --- every record round-trips through the real API against the |
| 541 | # RESTORED data dir, both modes --- |
| 542 | for collection in ("notes_classic", "notes_append"): |
| 543 | _clear_caches() |
| 544 | original_live = { |
| 545 | r["id"]: r |
| 546 | for r in object_records.read_collection_records(collection, base_dir=data_dir, roots=[]) |
| 547 | } |
| 548 | _clear_caches() |
| 549 | restored_live = { |
| 550 | r["id"]: r |
| 551 | for r in object_records.read_collection_records( |
| 552 | collection, base_dir=restored_data, roots=[] |
| 553 | ) |
| 554 | } |
| 555 | assert restored_live == original_live, ( |
| 556 | f"{collection}: restored collection's live records diverge from the original" |
| 557 | ) |
| 558 | # spot-check the specific hostile fields survived exactly |
| 559 | assert restored_live["r-multibyte"]["note"] == original_live["r-multibyte"]["note"] |
| 560 | assert restored_live["r-emoji"]["note"] == original_live["r-emoji"]["note"] |
| 561 | assert restored_live["r-bare-cr" if "r-bare-cr" in restored_live else "r-ascii"] |
| 562 | assert json.loads(restored_live["r-embedded-json-pretty"]["lines"]) == json.loads( |
| 563 | original_live["r-embedded-json-pretty"]["lines"] |
| 564 | ) |
| 565 | |
| 566 | # append-mode collection: confirm the churn (superseded update, delete) |
| 567 | # survived the round trip with correct fold semantics, not just as |
| 568 | # opaque bytes. |
| 569 | _clear_caches() |
| 570 | restored_append_live = { |
| 571 | r["id"]: r |
| 572 | for r in object_records.read_collection_records( |
| 573 | "notes_append", base_dir=restored_data, roots=[] |
| 574 | ) |
| 575 | } |
| 576 | assert restored_append_live["r-ascii"]["note"] == "ascii v3" |
| 577 | assert "r-bare-cr" not in restored_append_live |
| 578 | |
| 579 | # by-id reads against the restored dir work too (sidecar builds fresh |
| 580 | # against the restored file). |
| 581 | for record_id in restored_append_live: |
| 582 | _clear_caches() |
| 583 | by_id = object_records.get_collection_record( |
| 584 | "notes_append", record_id, base_dir=restored_data, roots=[] |
| 585 | ) |
| 586 | assert by_id == restored_append_live[record_id] |
| 587 | |
| 588 | |
| 589 | def test_backup_restore_round_trip_survives_a_compacted_append_collection_too(tmp_path): |
| 590 | """Same round-trip check, but compact the append-mode collection BEFORE |
| 591 | backing it up -- confirms fidelity holds for the already-folded |
| 592 | physical form too, not just the raw log form.""" |
| 593 | src_root = tmp_path / "src" |
| 594 | objects_dir, data_dir = _make_source_runtime(src_root) |
| 595 | object_records.compact_collection("notes_append", base_dir=data_dir, roots=[]) |
| 596 | |
| 597 | backup_output = tmp_path / "backups" / "compacted.tar.gz" |
| 598 | object_backup.create_runtime_backup(backup_output, objects_dir=objects_dir, data_dir=data_dir) |
| 599 | |
| 600 | restore_root = tmp_path / "restored" |
| 601 | restored_data = restore_root / "data" |
| 602 | object_backup.restore_runtime_backup( |
| 603 | backup_output, |
| 604 | objects_dir=restore_root / "objects", |
| 605 | data_dir=restored_data, |
| 606 | overwrite=True, |
| 607 | ) |
| 608 | |
| 609 | original_bytes = _records_path(data_dir, "notes_append").read_bytes() |
| 610 | restored_bytes = _records_path(restored_data, "notes_append").read_bytes() |
| 611 | assert restored_bytes == original_bytes |
| 612 | |
| 613 | _clear_caches() |
| 614 | original_live = { |
| 615 | r["id"]: r |
| 616 | for r in object_records.read_collection_records("notes_append", base_dir=data_dir, roots=[]) |
| 617 | } |
| 618 | _clear_caches() |
| 619 | restored_live = { |
| 620 | r["id"]: r |
| 621 | for r in object_records.read_collection_records( |
| 622 | "notes_append", base_dir=restored_data, roots=[] |
| 623 | ) |
| 624 | } |
| 625 | assert restored_live == original_live |
| 626 | |
| 627 | |
| 628 | def test_restored_tsv_files_are_valid_and_equal_on_disk(tmp_path): |
| 629 | """Direct on-disk inspection (not just through the API): every restored |
| 630 | .tsv file must (a) exist, (b) be byte-equal to its source, and (c) |
| 631 | independently parse via plain csv.reader into the same physical rows |
| 632 | as the source -- i.e. the restore didn't quietly reinterpret/re-encode |
| 633 | line endings, quoting, or the tab dialect along the way.""" |
| 634 | src_root = tmp_path / "src" |
| 635 | objects_dir, data_dir = _make_source_runtime(src_root) |
| 636 | |
| 637 | backup_output = tmp_path / "backups" / "b.tar.gz" |
| 638 | object_backup.create_runtime_backup(backup_output, objects_dir=objects_dir, data_dir=data_dir) |
| 639 | |
| 640 | restored_data = tmp_path / "restored" / "data" |
| 641 | object_backup.restore_runtime_backup( |
| 642 | backup_output, |
| 643 | objects_dir=tmp_path / "restored" / "objects", |
| 644 | data_dir=restored_data, |
| 645 | overwrite=True, |
| 646 | ) |
| 647 | |
| 648 | for collection in ("notes_classic", "notes_append"): |
| 649 | src_path = _records_path(data_dir, collection) |
| 650 | dst_path = _records_path(restored_data, collection) |
| 651 | assert dst_path.is_file() |
| 652 | assert dst_path.read_bytes() == src_path.read_bytes() |
| 653 | |
| 654 | with src_path.open(newline="") as fh: |
| 655 | src_rows = list(csv.reader(fh, delimiter="\t")) |
| 656 | with dst_path.open(newline="") as fh: |
| 657 | dst_rows = list(csv.reader(fh, delimiter="\t")) |
| 658 | assert dst_rows == src_rows, f"{collection}: restored .tsv parses to different physical rows" |
| 659 | |
| 660 | |
| 661 | # ============================================================================= |
| 662 | # 4. object_backup_index's OWN quote-blind torn-tail copy, against a backed |
| 663 | # up file |
| 664 | # ============================================================================= |
| 665 | |
| 666 | |
| 667 | def test_backup_index_parses_a_well_formed_multiline_cell_in_a_complete_backed_up_file_correctly( |
| 668 | tmp_path, |
| 669 | ): |
| 670 | """Baseline (expected-safe): a COMPLETE, properly-terminated |
| 671 | records.tsv whose last physical row happens to hold a pretty-printed |
| 672 | (real-raw-newline-bearing) JSON blob -- backed up, then read via |
| 673 | object_backup_index._parse_tsv_by_id (the function preview_collection/ |
| 674 | preview_record use). Since the file genuinely ends with "\\n" (nothing |
| 675 | was interrupted), object_backup_index's torn-tail check should be a |
| 676 | correct no-op here, exactly like object_records's copy is in the |
| 677 | equivalent ordinary case.""" |
| 678 | data_dir = tmp_path / "data" |
| 679 | write_append_schema(data_dir, "invoices", [ID_FIELD, LINES_FIELD]) |
| 680 | object_records.create_collection_record( |
| 681 | "invoices", {"id": "A", "lines": json.dumps([{"sku": "A-1"}])}, base_dir=data_dir, roots=[] |
| 682 | ) |
| 683 | pretty_blob = json.dumps([{"sku": "B-1", "qty": 1}, {"sku": "B-2", "qty": 2}], indent=2) |
| 684 | object_records.create_collection_record( |
| 685 | "invoices", {"id": "B", "lines": pretty_blob}, base_dir=data_dir, roots=[] |
| 686 | ) |
| 687 | |
| 688 | path = _records_path(data_dir, "invoices") |
| 689 | raw = path.read_bytes() |
| 690 | assert raw.endswith(b"\n") |
| 691 | assert raw.count(b"\n") > 3 # genuinely multi-physical-line file |
| 692 | |
| 693 | parsed = object_backup_index._parse_tsv_by_id(raw) |
| 694 | assert set(parsed) == {"A", "B"} |
| 695 | assert json.loads(parsed["B"]["lines"]) == [{"sku": "B-1", "qty": 1}, {"sku": "B-2", "qty": 2}] |
| 696 | |
| 697 | |
| 698 | def test_backup_index_torn_tail_is_also_quote_blind_on_a_backed_up_file_FINDING(tmp_path): |
| 699 | """FINDING: object_backup_index._drop_torn_tail (used by |
| 700 | _parse_append_tsv_by_id / _parse_tsv_by_id, which back |
| 701 | preview_collection and preview_record) has the SAME "does the file end |
| 702 | with a literal \\n byte" check as object_records._drop_torn_tail / |
| 703 | _repair_torn_tail's documented substrate bug #2 -- and is independently |
| 704 | reimplemented rather than shared, so a fix to one does not fix the |
| 705 | other. If a records.tsv is captured into a backup while genuinely torn |
| 706 | (a crash/interrupted write mid multi-line quoted field, landing right |
| 707 | after one of the field's OWN internal newlines -- see |
| 708 | tests/test_embedded_json_lines_characterization.py's |
| 709 | ...mid_multiline_row_is_silently_resurrected_and_cascades_FINDING for |
| 710 | the live-side version of this), the archived copy carries the exact |
| 711 | same torn bytes, and object_backup_index's preview/diff machinery |
| 712 | reads the incomplete row back as if it were a genuine, committed |
| 713 | record -- corrupting the *preview*, not the raw backup bytes |
| 714 | themselves (the tar archive itself is unaffected; this is a read-side |
| 715 | misparse of an already-flawed live file, both at backup time and at |
| 716 | restore-and-reread time). |
| 717 | |
| 718 | Reproduced directly against object_backup_index._parse_tsv_by_id (the |
| 719 | same function preview_collection/preview_record call on archived |
| 720 | bytes), mirroring the live-side FINDING's construction: a physical row |
| 721 | for id "C" is built with the real csv dialect this module writes with, |
| 722 | then cut right after its SECOND physical line -- the cut point itself |
| 723 | lands on "\\n", so the fragment "looks" terminated by the naive check |
| 724 | even though the quote never closed. |
| 725 | """ |
| 726 | header = ["_op", "id", "lines"] |
| 727 | hostile_blob = json.dumps( |
| 728 | [{"sku": "C-1", "note": "hostile é日本 \"q\" \\z"}, {"sku": "C-2"}], indent=2 |
| 729 | ) |
| 730 | |
| 731 | buf = io.StringIO() |
| 732 | writer = csv.writer(buf, delimiter="\t", lineterminator="\n", quoting=csv.QUOTE_MINIMAL) |
| 733 | writer.writerow(header) |
| 734 | writer.writerow(["", "A", json.dumps([{"sku": "A-1"}])]) |
| 735 | writer.writerow(["", "B", json.dumps([{"sku": "B-1"}])]) |
| 736 | full_text_before_c = buf.getvalue() |
| 737 | |
| 738 | buf2 = io.StringIO() |
| 739 | writer2 = csv.writer(buf2, delimiter="\t", lineterminator="\n", quoting=csv.QUOTE_MINIMAL) |
| 740 | writer2.writerow(["", "C", hostile_blob]) |
| 741 | full_row_c = buf2.getvalue() |
| 742 | assert full_row_c.count("\n") > 3 # genuinely spans several physical lines |
| 743 | |
| 744 | first_nl = full_row_c.index("\n") |
| 745 | second_nl = full_row_c.index("\n", first_nl + 1) |
| 746 | torn_fragment = full_row_c[: second_nl + 1] |
| 747 | assert torn_fragment.endswith("\n") # looks "committed" by the naive check |
| 748 | |
| 749 | # This is the byte sequence that would end up inside a backup archive |
| 750 | # if the live file were captured (or itself crashed) at exactly this |
| 751 | # unlucky boundary. |
| 752 | backed_up_text = full_text_before_c + torn_fragment |
| 753 | assert backed_up_text.endswith("\n") # passes the naive "not torn" check despite being torn |
| 754 | backed_up_raw = backed_up_text.encode("utf-8") |
| 755 | |
| 756 | parsed = object_backup_index._parse_tsv_by_id(backed_up_raw) |
| 757 | |
| 758 | ids_seen = set(parsed) |
| 759 | finding_reproduced = "C" in ids_seen |
| 760 | print(f"\n[backup-index torn-tail FINDING] ids parsed from the torn backed-up file: {sorted(ids_seen)!r}") |
| 761 | if finding_reproduced: |
| 762 | print(f" record C's (corrupted/incomplete) parsed value: {parsed['C']!r}") |
| 763 | |
| 764 | assert finding_reproduced, ( |
| 765 | "FINDING confirmed: object_backup_index._parse_tsv_by_id resurrected " |
| 766 | f"a torn mid-multiline-row as a committed record. Parsed ids: {sorted(ids_seen)!r}" |
| 767 | ) |
| 768 | # And, mirroring the live-side finding: C's value is garbled (only the |
| 769 | # fragment up through its second physical line), not the real row. |
| 770 | try: |
| 771 | recovered = json.loads(parsed["C"]["lines"]) |
| 772 | except json.JSONDecodeError: |
| 773 | recovered = None |
| 774 | assert recovered != [{"sku": "C-1", "note": "hostile é日本 \"q\" \\z"}, {"sku": "C-2"}], ( |
| 775 | "if this ever starts parsing to the FULL correct value, the finding " |
| 776 | "no longer reproduces and this test should be converted to a " |
| 777 | "regression guard (see the sibling xfail test's pattern)" |
| 778 | ) |
| 779 | |
| 780 | |
| 781 | def test_backup_index_preview_collection_end_to_end_against_a_torn_archived_file(tmp_path): |
| 782 | """Same FINDING, exercised end-to-end through the real public entry |
| 783 | point (preview_collection reading an actual .tar.gz archive member) |
| 784 | rather than calling the private parser directly -- confirms the bug is |
| 785 | reachable from the documented API surface, not just the internal |
| 786 | helper.""" |
| 787 | data_dir = tmp_path / "data" |
| 788 | write_append_schema(data_dir, "invoices", [ID_FIELD, LINES_FIELD]) |
| 789 | object_records.create_collection_record( |
| 790 | "invoices", {"id": "A", "lines": json.dumps([{"sku": "A-1"}])}, base_dir=data_dir, roots=[] |
| 791 | ) |
| 792 | object_records.create_collection_record( |
| 793 | "invoices", {"id": "B", "lines": json.dumps([{"sku": "B-1"}])}, base_dir=data_dir, roots=[] |
| 794 | ) |
| 795 | path = _records_path(data_dir, "invoices") |
| 796 | full_text = path.read_text() |
| 797 | assert full_text.endswith("\n") |
| 798 | |
| 799 | hostile_blob = json.dumps([{"sku": "C-1"}, {"sku": "C-2"}], indent=2) |
| 800 | buf = io.StringIO() |
| 801 | writer = csv.writer(buf, delimiter="\t", lineterminator="\n", quoting=csv.QUOTE_MINIMAL) |
| 802 | writer.writerow(["", "C", hostile_blob]) |
| 803 | full_row_text = buf.getvalue() |
| 804 | first_nl = full_row_text.index("\n") |
| 805 | second_nl = full_row_text.index("\n", first_nl + 1) |
| 806 | torn_fragment = full_row_text[: second_nl + 1] |
| 807 | |
| 808 | # Simulate: the live file itself crashed at this exact unlucky |
| 809 | # boundary (matches the live-side FINDING scenario), and a backup was |
| 810 | # then taken of the runtime in that state. |
| 811 | path.write_text(full_text + torn_fragment) |
| 812 | assert path.read_text().endswith("\n") |
| 813 | |
| 814 | objects_dir = tmp_path / "objects" |
| 815 | objects_dir.mkdir() |
| 816 | |
| 817 | # preview_collection resolves the backup id under |
| 818 | # backups_dir(data_dir) = <data_dir>/backups, and separately diffs |
| 819 | # against the LIVE records.tsv under that SAME data_dir. To make the |
| 820 | # finding legible in the diff output (rather than backup and "live" |
| 821 | # both showing the identical torn bytes and folding identically), the |
| 822 | # archive is placed under a distinct "server_data" dir whose own |
| 823 | # collections/invoices does not exist -- so the diff's "added" list is |
| 824 | # purely "what preview_collection believes the backup contains". |
| 825 | server_data_dir = tmp_path / "server_data" |
| 826 | backup_output = server_data_dir / "backups" / "torn.tar.gz" |
| 827 | object_backup.create_runtime_backup(backup_output, objects_dir=objects_dir, data_dir=data_dir) |
| 828 | |
| 829 | result = object_backup_index.preview_collection( |
| 830 | "torn.tar.gz", "invoices", data_dir=server_data_dir |
| 831 | ) |
| 832 | |
| 833 | print(f"\n[backup-index torn-tail FINDING, end-to-end] preview_collection " |
| 834 | f"added={result['added']!r}") |
| 835 | |
| 836 | assert "A" in result["added"] and "B" in result["added"] |
| 837 | assert "C" in result["added"], ( |
| 838 | "FINDING confirmed: preview_collection (the real public API atop " |
| 839 | "object_backup_index._parse_tsv_by_id) reports a torn mid-multiline " |
| 840 | f"row as a normal, present-in-backup record. Full diff: {result!r}" |
| 841 | ) |