Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_durability_backup_compaction_characterization.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_durability_backup_compaction_characterization.py · 37.5 KB · 841 lines
1"""CHARACTERIZATION tests: does BACKUP/RESTORE and COMPACTION preserve data
2with byte-exact fidelity?
3
4This file is pure characterization of EXISTING behavior. It does not modify
5any production module (object_records.py, object_backup.py,
6object_backup_index.py, object_daemon.py are all untouched). Where a case
7fails or corrupts data, the assertion states the correctness property a safe
8durability substrate SHOULD have and is left to fail -- that failure IS the
9finding. Do not "fix" a failing assertion here by weakening it to match
10broken behavior.
11
12Context (see tests/test_embedded_json_lines_characterization.py for the
13sibling investigation this one continues): an append-mode collection
14accumulates superseded/tombstoned physical rows as it is written to, and is
15periodically COMPACTED -- folded down to just its live rows -- either on
16demand (object_records.compact_collection, exercised directly here) or via
17object_daemon.py's process_compactions poller (not imported here; this file
18stays at the object_records.compact_collection layer, which is what the
19daemon itself calls). Separately, the whole runtime -- including every
20collection's records.tsv, live or append-mode -- can be BACKED UP (a tar/gz
21archive: object_backup.create_runtime_backup /
22create_runtime_restore_point) and RESTORED into a fresh directory
23(object_backup.restore_runtime_backup). A third module,
24object_backup_index.py, does NOT participate in backup/restore itself --
25it only offers read-only preview/diff of a backup's records against live
26data (preview_collection/preview_record) -- but it carries its OWN local
27copy of a quote-blind "torn tail" check (_drop_torn_tail), independently
28reimplemented from object_records._drop_torn_tail (documented KNOWN
29LIMITATION / substrate bug #2 there: a crash landing right after a newline
30that is INSIDE an open quoted multi-line cell leaves the file looking
31"terminated" -- ends with \\n -- even though the row never actually
32finished, so the naive "ends with \\n" check wrongly calls it committed).
33Part of this file's job is to confirm whether that same latent bug is
34reachable through object_backup_index's copy of the check against a backed
35up (not just a live) records.tsv.
36"""
37
38from __future__ import annotations
39
40import csv
41import io
42import json
43import shutil
44import tarfile
45from pathlib import Path
46
47import pytest
48
49import object_backup
50import object_backup_index
51import object_records
52
53pytestmark = pytest.mark.conformance
54
55
56ID_FIELD = {"name": "id"}
57NOTE_FIELD = {"name": "note", "type": "textarea"}
58LINES_FIELD = {"name": "lines", "type": "textarea"}
59
60
61# --- setup helpers (mirror tests/test_object_records.py /
62# tests/test_embedded_json_lines_characterization.py conventions) -------
63
64
65def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path:
66 path = data_dir / "schemas" / f"{collection}.json"
67 path.parent.mkdir(parents=True, exist_ok=True)
68 path.write_text(json.dumps({"fields": fields}))
69 return path
70
71
72def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path:
73 path = data_dir / "schemas" / f"{collection}.json"
74 path.parent.mkdir(parents=True, exist_ok=True)
75 path.write_text(json.dumps({"fields": fields, "storage": "append"}))
76 return path
77
78
79def _clear_caches() -> None:
80 """Force the NEXT read past the warm in-process caches (see the sibling
81 embedded-JSON characterization file for why this matters for by-id
82 reads specifically)."""
83 object_records._RECORDS_CACHE.clear()
84 object_records._OIDX_CACHE.clear()
85
86
87def _records_path(data_dir: Path, collection: str) -> Path:
88 return data_dir / "collections" / collection / "records.tsv"
89
90
91def _exotic_corpus() -> list[dict]:
92 """A grab-bag of hostile-but-legal field values: ASCII, multibyte,
93 emoji (astral-plane, so a surrogate pair in UTF-16 but a single scalar
94 in UTF-8), embedded compact JSON, embedded PRETTY-PRINTED JSON (real
95 raw newline bytes in the cell -- the case that actually stresses the
96 CSV-aware row-boundary machinery), a large field, and a bare literal
97 CR (0x0D, not part of a CRLF pair) -- which the CSV writer must quote
98 to round-trip safely (an unquoted bare CR is not a delimiter itself
99 but is exactly the kind of "surprising control byte" this substrate's
100 hostile-content tests exist to check)."""
101 return [
102 {
103 "id": "r-ascii",
104 "note": "plain ascii, no surprises",
105 "lines": json.dumps([{"sku": "A-1", "qty": 1}]),
106 },
107 {
108 "id": "r-multibyte",
109 "note": "café naïve résumé 日本語 中文",
110 "lines": json.dumps([{"sku": "日本-1", "note": "中文注释"}]),
111 },
112 {
113 "id": "r-emoji",
114 "note": "party \U0001f389 rocket \U0001f680 family \U0001f468‍\U0001f469‍\U0001f467",
115 "lines": json.dumps([{"sku": "EMOJI-1", "label": "\U0001f525 hot"}]),
116 },
117 {
118 "id": "r-embedded-json-compact",
119 "note": "compact json blob (no raw control bytes)",
120 "lines": json.dumps(
121 [{"sku": f"SKU-{i:03d}", "qty": i, "note": "x\ty\nz\"q\"\\w"} for i in range(20)]
122 ),
123 },
124 {
125 "id": "r-embedded-json-pretty",
126 "note": "pretty json blob -- REAL raw newline bytes in this cell",
127 "lines": json.dumps(
128 [{"sku": f"SKU-{i:03d}", "qty": i} for i in range(10)], indent=2
129 ),
130 },
131 {
132 "id": "r-bare-cr",
133 "note": "alpha\rbeta\rgamma", # bare CR, not CRLF -- must be quoted to survive
134 "lines": json.dumps([{"sku": "CR-1"}]),
135 },
136 {
137 "id": "r-crlf",
138 "note": "alpha\r\nbeta\r\ngamma",
139 "lines": json.dumps([{"sku": "CRLF-1"}]),
140 },
141 {
142 "id": "r-large",
143 "note": "large field",
144 "lines": json.dumps([{"sku": f"BIG-{i:05d}", "qty": i, "price": 1099 + i} for i in range(2000)]),
145 },
146 ]
147
148
149# =============================================================================
150# 1. COMPACTION FIDELITY -- live records survive byte-exact, dead rows gone
151# =============================================================================
152
153
154def test_compaction_preserves_live_records_byte_exact_including_exotic_content(tmp_path):
155 """Build an append-mode collection with a realistic mix of creates,
156 superseding updates, and deletes (tombstones) over the exotic corpus,
157 compact it, and check every angle of "did compaction preserve exactly
158 the live set, byte-exact, and nothing else":
159
160 - every live record's field values are byte-for-byte identical to
161 what a full-fold read reported immediately BEFORE compaction
162 - superseded/tombstoned rows are gone from the physical file
163 - the physical row count actually dropped
164 - a read after compaction returns exactly the live set (no more, no
165 less)
166 - by-id reads still work for every live id post-compaction, via the
167 REBUILT sidecar (cold caches, forcing the id->offset oidx path)
168 """
169 data_dir = tmp_path / "data"
170 write_append_schema(data_dir, "docs", [ID_FIELD, NOTE_FIELD, LINES_FIELD])
171
172 corpus = _exotic_corpus()
173 for rec in corpus:
174 object_records.create_collection_record("docs", dict(rec), base_dir=data_dir, roots=[])
175
176 # Superseding updates on a few ids (multiple times each -- each update
177 # is a brand-new physical row in append mode, leaving its predecessors
178 # superseded-but-not-yet-compacted).
179 object_records.update_collection_record(
180 "docs", "r-ascii", {"note": "ascii v2"}, base_dir=data_dir, roots=[]
181 )
182 object_records.update_collection_record(
183 "docs", "r-ascii", {"note": "ascii v3 final"}, base_dir=data_dir, roots=[]
184 )
185 object_records.update_collection_record(
186 "docs", "r-multibyte", {"note": "café v2 日本語"}, base_dir=data_dir, roots=[]
187 )
188 object_records.update_collection_record(
189 "docs",
190 "r-embedded-json-pretty",
191 {"lines": json.dumps([{"sku": "REPLACED", "qty": 999}], indent=2)},
192 base_dir=data_dir,
193 roots=[],
194 )
195
196 # Tombstone a couple of ids.
197 object_records.delete_collection_record("docs", "r-crlf", base_dir=data_dir, roots=[])
198 object_records.delete_collection_record("docs", "r-large", base_dir=data_dir, roots=[])
199
200 # Capture the exact live set (full-fold read) immediately before
201 # compaction: this is the ground truth compaction must reproduce.
202 _clear_caches()
203 live_before = {
204 r["id"]: r
205 for r in object_records.read_collection_records("docs", base_dir=data_dir, roots=[])
206 }
207 expected_ids = {"r-ascii", "r-multibyte", "r-emoji", "r-embedded-json-compact",
208 "r-embedded-json-pretty", "r-bare-cr"}
209 assert set(live_before) == expected_ids
210 assert live_before["r-ascii"]["note"] == "ascii v3 final"
211 assert live_before["r-multibyte"]["note"] == "café v2 日本語"
212 assert json.loads(live_before["r-embedded-json-pretty"]["lines"]) == [
213 {"sku": "REPLACED", "qty": 999}
214 ]
215 assert live_before["r-bare-cr"]["note"] == "alpha\rbeta\rgamma"
216
217 path = _records_path(data_dir, "docs")
218 with path.open(newline="") as fh:
219 physical_rows_before = sum(1 for _ in csv.reader(fh, delimiter="\t")) - 1 # minus header
220 raw_before = path.read_text()
221 assert "\tdel\t" in raw_before or raw_before.count("del") >= 1 # tombstones present
222
223 result = object_records.compact_collection("docs", base_dir=data_dir, roots=[])
224
225 # --- physical row count actually dropped ---
226 assert result["rows_before"] == physical_rows_before
227 assert result["rows_after"] == len(expected_ids), (
228 f"compacted physical row count should equal the live set size; "
229 f"got {result}"
230 )
231 assert result["rows_after"] < result["rows_before"]
232 assert result["bytes_after"] < result["bytes_before"]
233
234 # --- superseded/tombstoned rows gone from the physical file ---
235 raw_after = path.read_text()
236 assert "del" not in raw_after.split("\n")[0] # header sanity
237 for line in raw_after.splitlines()[1:]:
238 assert not line.startswith("del\t"), f"tombstone row survived compaction: {line!r}"
239 assert "r-crlf" not in raw_after
240 assert "r-large" not in raw_after
241 assert "ascii v2" not in raw_after # superseded value must not linger anywhere on disk
242 assert "REPLACED" not in raw_after or json.loads(live_before["r-embedded-json-pretty"]["lines"])
243
244 # --- read after compaction returns exactly the live set, byte-exact ---
245 _clear_caches()
246 live_after = {
247 r["id"]: r
248 for r in object_records.read_collection_records("docs", base_dir=data_dir, roots=[])
249 }
250 assert set(live_after) == expected_ids
251 for record_id in expected_ids:
252 assert live_after[record_id] == live_before[record_id], (
253 f"compaction changed live content for {record_id!r}: "
254 f"before={live_before[record_id]!r} after={live_after[record_id]!r}"
255 )
256
257 # --- by-id reads still work for every live id, sidecar rebuilt ---
258 for record_id in expected_ids:
259 _clear_caches() # cold cache + cold sidecar -> forces oidx rebuild from scratch
260 by_id = object_records.get_collection_record("docs", record_id, base_dir=data_dir, roots=[])
261 assert by_id == live_before[record_id], (
262 f"post-compaction by-id read (rebuilt sidecar) diverged for {record_id!r}: "
263 f"{by_id!r} != {live_before[record_id]!r}"
264 )
265
266 # --- deleted ids are genuinely gone, not just excluded from listing ---
267 for dead_id in ("r-crlf", "r-large"):
268 _clear_caches()
269 with pytest.raises(object_records.RecordNotFoundError):
270 object_records.get_collection_record("docs", dead_id, base_dir=data_dir, roots=[])
271
272
273def test_compaction_of_never_touched_and_all_deleted_collection_is_honest(tmp_path):
274 """Edge cases around the main fidelity test: compacting a collection
275 where EVERY record has since been deleted must leave zero live rows
276 (not resurrect anything), and compacting twice in a row must be a
277 stable no-op the second time."""
278 data_dir = tmp_path / "data"
279 write_append_schema(data_dir, "ephemeral", [ID_FIELD, NOTE_FIELD])
280
281 for i in range(5):
282 object_records.create_collection_record(
283 "ephemeral", {"id": f"e{i}", "note": f"note-{i}"}, base_dir=data_dir, roots=[]
284 )
285 for i in range(5):
286 object_records.delete_collection_record("ephemeral", f"e{i}", base_dir=data_dir, roots=[])
287
288 result = object_records.compact_collection("ephemeral", base_dir=data_dir, roots=[])
289 assert result["rows_after"] == 0
290
291 _clear_caches()
292 listing = object_records.list_collection_records("ephemeral", base_dir=data_dir, roots=[])["records"]
293 assert listing == []
294
295 # Compacting an already-fully-compacted (here: empty) collection again
296 # must be a stable no-op.
297 result2 = object_records.compact_collection("ephemeral", base_dir=data_dir, roots=[])
298 assert result2["rows_before"] == 0
299 assert result2["rows_after"] == 0
300
301
302# =============================================================================
303# 2. COMPACTION CRASH-SAFETY -- interrupted compaction must not lose data
304# =============================================================================
305
306
307def test_compaction_interrupted_before_atomic_replace_leaves_original_untouched(tmp_path, monkeypatch):
308 """compact_collection rewrites via _write_collection_records, which
309 writes the FOLDED content to a fresh temp file and only then does an
310 atomic `temp_path.replace(path)` -- so a crash at any point BEFORE
311 that replace call must leave the original records.tsv completely
312 untouched (the classic "write-new, then atomically swap" durability
313 property). Verified here by monkeypatching Path.replace to raise
314 partway through a real compact_collection() call, simulating a process
315 crash/OSError at exactly that instant, then confirming:
316
317 1. compact_collection's exception propagates (it does not silently
318 swallow the failure and report success)
319 2. the original records.tsv is byte-for-byte unchanged from before
320 the attempted compaction (superseded rows and tombstones intact
321 -- nothing was lost, nothing was half-written)
322 3. a normal read through the public API still returns the correct,
323 complete pre-compaction live set
324 4. once the injected failure is removed, retrying compact_collection
325 succeeds and produces the correct compacted result -- i.e. the
326 interrupted attempt did not leave the collection in a state that
327 a follow-up compaction can't recover from
328 """
329 data_dir = tmp_path / "data"
330 write_append_schema(data_dir, "orders", [ID_FIELD, NOTE_FIELD])
331 object_records.create_collection_record(
332 "orders", {"id": "o1", "note": "v1"}, base_dir=data_dir, roots=[]
333 )
334 object_records.update_collection_record(
335 "orders", "o1", {"note": "v2"}, base_dir=data_dir, roots=[]
336 )
337 object_records.update_collection_record(
338 "orders", "o1", {"note": "v3"}, base_dir=data_dir, roots=[]
339 )
340 object_records.create_collection_record(
341 "orders", {"id": "o2", "note": "keep-me"}, base_dir=data_dir, roots=[]
342 )
343 object_records.delete_collection_record("orders", "o2", base_dir=data_dir, roots=[])
344
345 path = _records_path(data_dir, "orders")
346 original_bytes = path.read_bytes()
347
348 _clear_caches()
349 live_before = {
350 r["id"]: r
351 for r in object_records.read_collection_records("orders", base_dir=data_dir, roots=[])
352 }
353 assert live_before == {"o1": {"id": "o1", "note": "v3"}}
354
355 real_replace = Path.replace
356 state = {"armed": True}
357
358 def _boom_replace(self, target):
359 # temp_path is built as f".{path.name}.{pid}.{tid}.tmp" alongside
360 # the real records.tsv -- i.e. ".records.tsv.<pid>.<tid>.tmp" in
361 # the "orders" collection's own directory (the collection name is
362 # not itself part of the filename, only the parent directory).
363 if (
364 state["armed"]
365 and self.name.startswith(".records.tsv.")
366 and self.name.endswith(".tmp")
367 and self.parent.name == "orders"
368 ):
369 state["armed"] = False
370 raise OSError("simulated crash: process died mid-compaction, before atomic replace")
371 return real_replace(self, target)
372
373 monkeypatch.setattr(Path, "replace", _boom_replace)
374
375 with pytest.raises(OSError, match="simulated crash"):
376 object_records.compact_collection("orders", base_dir=data_dir, roots=[])
377
378 # 2. original file byte-for-byte unchanged.
379 assert path.read_bytes() == original_bytes, (
380 "FINDING: an interrupted compaction (crash before the atomic "
381 "replace) altered the on-disk records.tsv -- compaction is "
382 "destructive-before-commit."
383 )
384
385 # 3. a normal read still returns the correct pre-compaction live set.
386 _clear_caches()
387 live_during_outage = {
388 r["id"]: r
389 for r in object_records.read_collection_records("orders", base_dir=data_dir, roots=[])
390 }
391 assert live_during_outage == live_before
392
393 # 4. retry (failure no longer armed) succeeds and recovers correctly.
394 monkeypatch.setattr(Path, "replace", real_replace)
395 result = object_records.compact_collection("orders", base_dir=data_dir, roots=[])
396 assert result["rows_after"] == 1
397
398 _clear_caches()
399 live_after_retry = {
400 r["id"]: r
401 for r in object_records.read_collection_records("orders", base_dir=data_dir, roots=[])
402 }
403 assert live_after_retry == live_before
404
405
406def test_compaction_stray_orphaned_temp_file_is_inert_and_not_resurrected(tmp_path):
407 """A DIFFERENT crash shape: a previous compaction (or previous process
408 incarnation, different pid) died after creating its temp file but
409 before either writing to it fully or replacing the original -- leaving
410 an orphaned `.records.tsv.<pid>.<tid>.tmp` file sitting next to the
411 real records.tsv forever (nothing in this codebase's write path scans
412 for or cleans up a stale temp file from a DIFFERENT pid/tid on
413 startup). Characterizes: does that debris get accidentally treated as
414 data by any read path, and does a subsequent real compaction still
415 produce the correct result despite the debris being present?"""
416 data_dir = tmp_path / "data"
417 write_append_schema(data_dir, "invoices", [ID_FIELD, NOTE_FIELD])
418 object_records.create_collection_record(
419 "invoices", {"id": "i1", "note": "one"}, base_dir=data_dir, roots=[]
420 )
421 object_records.update_collection_record(
422 "invoices", "i1", {"note": "one-v2"}, base_dir=data_dir, roots=[]
423 )
424 object_records.create_collection_record(
425 "invoices", {"id": "i2", "note": "two"}, base_dir=data_dir, roots=[]
426 )
427
428 path = _records_path(data_dir, "invoices")
429 original_bytes = path.read_bytes()
430
431 # Plant debris matching the real naming convention but with a bogus
432 # pid/tid, containing garbage that -- if it were ever misread as real
433 # data -- would be obviously wrong.
434 stray = path.with_name(f".{path.name}.999999.999999.tmp")
435 stray.write_text("GARBAGE-NOT-A-VALID-TSV-ROW\t\t\n")
436 assert stray.exists()
437
438 _clear_caches()
439 listing = object_records.list_collection_records("invoices", base_dir=data_dir, roots=[])["records"]
440 assert listing == [{"id": "i1", "note": "one-v2"}, {"id": "i2", "note": "two"}], (
441 "a stray orphaned .tmp file was picked up by a read path"
442 )
443 assert path.read_bytes() == original_bytes # debris presence didn't touch the real file
444
445 result = object_records.compact_collection("invoices", base_dir=data_dir, roots=[])
446 assert result["rows_after"] == 2
447
448 _clear_caches()
449 listing_after = object_records.list_collection_records("invoices", base_dir=data_dir, roots=[])["records"]
450 assert listing_after == [{"id": "i1", "note": "one-v2"}, {"id": "i2", "note": "two"}]
451
452 # Report (not asserted pass/fail either way -- purely descriptive):
453 # does a real compaction clean up debris left by an unrelated pid/tid?
454 print(f"\n[compaction] stray orphaned temp file still present after a real "
455 f"compact_collection() call: {stray.exists()}")
456
457
458# =============================================================================
459# 3. BACKUP / RESTORE FIDELITY -- byte-exact round trip
460# =============================================================================
461
462
463def _make_source_runtime(base: Path) -> tuple[Path, Path]:
464 """Build a runtime layout (objects_dir + data_dir) with one classic-mode
465 and one append-mode collection, both carrying the full exotic corpus,
466 plus superseded/tombstoned rows in the append one (so the backed-up
467 file is a realistic append-log, not just a clean fold)."""
468 objects_dir = base / "objects"
469 data_dir = base / "data"
470 objects_dir.mkdir(parents=True, exist_ok=True)
471 (objects_dir / "keepme.txt").write_text("just to give the objects tree a file\n")
472
473 write_schema(data_dir, "notes_classic", [ID_FIELD, NOTE_FIELD, LINES_FIELD])
474 write_append_schema(data_dir, "notes_append", [ID_FIELD, NOTE_FIELD, LINES_FIELD])
475
476 corpus = _exotic_corpus()
477 for rec in corpus:
478 object_records.create_collection_record("notes_classic", dict(rec), base_dir=data_dir, roots=[])
479 object_records.create_collection_record("notes_append", dict(rec), base_dir=data_dir, roots=[])
480
481 # Give the append collection some churn so its physical file is a real
482 # log (superseded rows + a tombstone), not already-compacted.
483 object_records.update_collection_record(
484 "notes_append", "r-ascii", {"note": "ascii v2"}, base_dir=data_dir, roots=[]
485 )
486 object_records.update_collection_record(
487 "notes_append", "r-ascii", {"note": "ascii v3"}, base_dir=data_dir, roots=[]
488 )
489 object_records.delete_collection_record("notes_append", "r-bare-cr", base_dir=data_dir, roots=[])
490
491 return objects_dir, data_dir
492
493
494def test_backup_restore_byte_exact_round_trip_full_exotic_corpus(tmp_path):
495 """Back up a runtime carrying the full exotic corpus in both classic
496 and append storage, restore into a FRESH directory, and assert:
497
498 - every collection's records.tsv is byte-for-byte identical,
499 original vs. restored (the strongest possible fidelity check --
500 the backup mechanism is a raw tar copy, so this should hold
501 trivially, but is worth confirming rather than assumed)
502 - every collection's OBJECTS-DIR sibling file round-trips too
503 - every record, read back through the real object_records API
504 against the restored data_dir, matches the original exactly
505 - the restored append-mode file is still a VALID, parseable
506 append-log (correct live/dead fold), not just byte-identical
507 """
508 src_root = tmp_path / "src"
509 objects_dir, data_dir = _make_source_runtime(src_root)
510
511 backup_output = tmp_path / "backups" / "full.tar.gz"
512 summary = object_backup.create_runtime_backup(
513 backup_output, objects_dir=objects_dir, data_dir=data_dir
514 )
515 assert summary.files > 0
516 assert backup_output.exists()
517
518 restore_root = tmp_path / "restored"
519 restored_objects = restore_root / "objects"
520 restored_data = restore_root / "data"
521 restore_summary = object_backup.restore_runtime_backup(
522 backup_output,
523 objects_dir=restored_objects,
524 data_dir=restored_data,
525 overwrite=True,
526 )
527 assert restore_summary.files == summary.files
528
529 # --- byte-exact records.tsv round trip, both storage modes ---
530 for collection in ("notes_classic", "notes_append"):
531 original = _records_path(data_dir, collection).read_bytes()
532 restored = _records_path(restored_data, collection).read_bytes()
533 assert restored == original, (
534 f"{collection}: restored records.tsv is not byte-identical to the original"
535 )
536
537 # --- objects dir sibling file round trip ---
538 assert (restored_objects / "keepme.txt").read_bytes() == (objects_dir / "keepme.txt").read_bytes()
539
540 # --- every record round-trips through the real API against the
541 # RESTORED data dir, both modes ---
542 for collection in ("notes_classic", "notes_append"):
543 _clear_caches()
544 original_live = {
545 r["id"]: r
546 for r in object_records.read_collection_records(collection, base_dir=data_dir, roots=[])
547 }
548 _clear_caches()
549 restored_live = {
550 r["id"]: r
551 for r in object_records.read_collection_records(
552 collection, base_dir=restored_data, roots=[]
553 )
554 }
555 assert restored_live == original_live, (
556 f"{collection}: restored collection's live records diverge from the original"
557 )
558 # spot-check the specific hostile fields survived exactly
559 assert restored_live["r-multibyte"]["note"] == original_live["r-multibyte"]["note"]
560 assert restored_live["r-emoji"]["note"] == original_live["r-emoji"]["note"]
561 assert restored_live["r-bare-cr" if "r-bare-cr" in restored_live else "r-ascii"]
562 assert json.loads(restored_live["r-embedded-json-pretty"]["lines"]) == json.loads(
563 original_live["r-embedded-json-pretty"]["lines"]
564 )
565
566 # append-mode collection: confirm the churn (superseded update, delete)
567 # survived the round trip with correct fold semantics, not just as
568 # opaque bytes.
569 _clear_caches()
570 restored_append_live = {
571 r["id"]: r
572 for r in object_records.read_collection_records(
573 "notes_append", base_dir=restored_data, roots=[]
574 )
575 }
576 assert restored_append_live["r-ascii"]["note"] == "ascii v3"
577 assert "r-bare-cr" not in restored_append_live
578
579 # by-id reads against the restored dir work too (sidecar builds fresh
580 # against the restored file).
581 for record_id in restored_append_live:
582 _clear_caches()
583 by_id = object_records.get_collection_record(
584 "notes_append", record_id, base_dir=restored_data, roots=[]
585 )
586 assert by_id == restored_append_live[record_id]
587
588
589def test_backup_restore_round_trip_survives_a_compacted_append_collection_too(tmp_path):
590 """Same round-trip check, but compact the append-mode collection BEFORE
591 backing it up -- confirms fidelity holds for the already-folded
592 physical form too, not just the raw log form."""
593 src_root = tmp_path / "src"
594 objects_dir, data_dir = _make_source_runtime(src_root)
595 object_records.compact_collection("notes_append", base_dir=data_dir, roots=[])
596
597 backup_output = tmp_path / "backups" / "compacted.tar.gz"
598 object_backup.create_runtime_backup(backup_output, objects_dir=objects_dir, data_dir=data_dir)
599
600 restore_root = tmp_path / "restored"
601 restored_data = restore_root / "data"
602 object_backup.restore_runtime_backup(
603 backup_output,
604 objects_dir=restore_root / "objects",
605 data_dir=restored_data,
606 overwrite=True,
607 )
608
609 original_bytes = _records_path(data_dir, "notes_append").read_bytes()
610 restored_bytes = _records_path(restored_data, "notes_append").read_bytes()
611 assert restored_bytes == original_bytes
612
613 _clear_caches()
614 original_live = {
615 r["id"]: r
616 for r in object_records.read_collection_records("notes_append", base_dir=data_dir, roots=[])
617 }
618 _clear_caches()
619 restored_live = {
620 r["id"]: r
621 for r in object_records.read_collection_records(
622 "notes_append", base_dir=restored_data, roots=[]
623 )
624 }
625 assert restored_live == original_live
626
627
628def test_restored_tsv_files_are_valid_and_equal_on_disk(tmp_path):
629 """Direct on-disk inspection (not just through the API): every restored
630 .tsv file must (a) exist, (b) be byte-equal to its source, and (c)
631 independently parse via plain csv.reader into the same physical rows
632 as the source -- i.e. the restore didn't quietly reinterpret/re-encode
633 line endings, quoting, or the tab dialect along the way."""
634 src_root = tmp_path / "src"
635 objects_dir, data_dir = _make_source_runtime(src_root)
636
637 backup_output = tmp_path / "backups" / "b.tar.gz"
638 object_backup.create_runtime_backup(backup_output, objects_dir=objects_dir, data_dir=data_dir)
639
640 restored_data = tmp_path / "restored" / "data"
641 object_backup.restore_runtime_backup(
642 backup_output,
643 objects_dir=tmp_path / "restored" / "objects",
644 data_dir=restored_data,
645 overwrite=True,
646 )
647
648 for collection in ("notes_classic", "notes_append"):
649 src_path = _records_path(data_dir, collection)
650 dst_path = _records_path(restored_data, collection)
651 assert dst_path.is_file()
652 assert dst_path.read_bytes() == src_path.read_bytes()
653
654 with src_path.open(newline="") as fh:
655 src_rows = list(csv.reader(fh, delimiter="\t"))
656 with dst_path.open(newline="") as fh:
657 dst_rows = list(csv.reader(fh, delimiter="\t"))
658 assert dst_rows == src_rows, f"{collection}: restored .tsv parses to different physical rows"
659
660
661# =============================================================================
662# 4. object_backup_index's OWN quote-blind torn-tail copy, against a backed
663# up file
664# =============================================================================
665
666
667def test_backup_index_parses_a_well_formed_multiline_cell_in_a_complete_backed_up_file_correctly(
668 tmp_path,
669):
670 """Baseline (expected-safe): a COMPLETE, properly-terminated
671 records.tsv whose last physical row happens to hold a pretty-printed
672 (real-raw-newline-bearing) JSON blob -- backed up, then read via
673 object_backup_index._parse_tsv_by_id (the function preview_collection/
674 preview_record use). Since the file genuinely ends with "\\n" (nothing
675 was interrupted), object_backup_index's torn-tail check should be a
676 correct no-op here, exactly like object_records's copy is in the
677 equivalent ordinary case."""
678 data_dir = tmp_path / "data"
679 write_append_schema(data_dir, "invoices", [ID_FIELD, LINES_FIELD])
680 object_records.create_collection_record(
681 "invoices", {"id": "A", "lines": json.dumps([{"sku": "A-1"}])}, base_dir=data_dir, roots=[]
682 )
683 pretty_blob = json.dumps([{"sku": "B-1", "qty": 1}, {"sku": "B-2", "qty": 2}], indent=2)
684 object_records.create_collection_record(
685 "invoices", {"id": "B", "lines": pretty_blob}, base_dir=data_dir, roots=[]
686 )
687
688 path = _records_path(data_dir, "invoices")
689 raw = path.read_bytes()
690 assert raw.endswith(b"\n")
691 assert raw.count(b"\n") > 3 # genuinely multi-physical-line file
692
693 parsed = object_backup_index._parse_tsv_by_id(raw)
694 assert set(parsed) == {"A", "B"}
695 assert json.loads(parsed["B"]["lines"]) == [{"sku": "B-1", "qty": 1}, {"sku": "B-2", "qty": 2}]
696
697
698def test_backup_index_torn_tail_is_also_quote_blind_on_a_backed_up_file_FINDING(tmp_path):
699 """FINDING: object_backup_index._drop_torn_tail (used by
700 _parse_append_tsv_by_id / _parse_tsv_by_id, which back
701 preview_collection and preview_record) has the SAME "does the file end
702 with a literal \\n byte" check as object_records._drop_torn_tail /
703 _repair_torn_tail's documented substrate bug #2 -- and is independently
704 reimplemented rather than shared, so a fix to one does not fix the
705 other. If a records.tsv is captured into a backup while genuinely torn
706 (a crash/interrupted write mid multi-line quoted field, landing right
707 after one of the field's OWN internal newlines -- see
708 tests/test_embedded_json_lines_characterization.py's
709 ...mid_multiline_row_is_silently_resurrected_and_cascades_FINDING for
710 the live-side version of this), the archived copy carries the exact
711 same torn bytes, and object_backup_index's preview/diff machinery
712 reads the incomplete row back as if it were a genuine, committed
713 record -- corrupting the *preview*, not the raw backup bytes
714 themselves (the tar archive itself is unaffected; this is a read-side
715 misparse of an already-flawed live file, both at backup time and at
716 restore-and-reread time).
717
718 Reproduced directly against object_backup_index._parse_tsv_by_id (the
719 same function preview_collection/preview_record call on archived
720 bytes), mirroring the live-side FINDING's construction: a physical row
721 for id "C" is built with the real csv dialect this module writes with,
722 then cut right after its SECOND physical line -- the cut point itself
723 lands on "\\n", so the fragment "looks" terminated by the naive check
724 even though the quote never closed.
725 """
726 header = ["_op", "id", "lines"]
727 hostile_blob = json.dumps(
728 [{"sku": "C-1", "note": "hostile é日本 \"q\" \\z"}, {"sku": "C-2"}], indent=2
729 )
730
731 buf = io.StringIO()
732 writer = csv.writer(buf, delimiter="\t", lineterminator="\n", quoting=csv.QUOTE_MINIMAL)
733 writer.writerow(header)
734 writer.writerow(["", "A", json.dumps([{"sku": "A-1"}])])
735 writer.writerow(["", "B", json.dumps([{"sku": "B-1"}])])
736 full_text_before_c = buf.getvalue()
737
738 buf2 = io.StringIO()
739 writer2 = csv.writer(buf2, delimiter="\t", lineterminator="\n", quoting=csv.QUOTE_MINIMAL)
740 writer2.writerow(["", "C", hostile_blob])
741 full_row_c = buf2.getvalue()
742 assert full_row_c.count("\n") > 3 # genuinely spans several physical lines
743
744 first_nl = full_row_c.index("\n")
745 second_nl = full_row_c.index("\n", first_nl + 1)
746 torn_fragment = full_row_c[: second_nl + 1]
747 assert torn_fragment.endswith("\n") # looks "committed" by the naive check
748
749 # This is the byte sequence that would end up inside a backup archive
750 # if the live file were captured (or itself crashed) at exactly this
751 # unlucky boundary.
752 backed_up_text = full_text_before_c + torn_fragment
753 assert backed_up_text.endswith("\n") # passes the naive "not torn" check despite being torn
754 backed_up_raw = backed_up_text.encode("utf-8")
755
756 parsed = object_backup_index._parse_tsv_by_id(backed_up_raw)
757
758 ids_seen = set(parsed)
759 finding_reproduced = "C" in ids_seen
760 print(f"\n[backup-index torn-tail FINDING] ids parsed from the torn backed-up file: {sorted(ids_seen)!r}")
761 if finding_reproduced:
762 print(f" record C's (corrupted/incomplete) parsed value: {parsed['C']!r}")
763
764 assert finding_reproduced, (
765 "FINDING confirmed: object_backup_index._parse_tsv_by_id resurrected "
766 f"a torn mid-multiline-row as a committed record. Parsed ids: {sorted(ids_seen)!r}"
767 )
768 # And, mirroring the live-side finding: C's value is garbled (only the
769 # fragment up through its second physical line), not the real row.
770 try:
771 recovered = json.loads(parsed["C"]["lines"])
772 except json.JSONDecodeError:
773 recovered = None
774 assert recovered != [{"sku": "C-1", "note": "hostile é日本 \"q\" \\z"}, {"sku": "C-2"}], (
775 "if this ever starts parsing to the FULL correct value, the finding "
776 "no longer reproduces and this test should be converted to a "
777 "regression guard (see the sibling xfail test's pattern)"
778 )
779
780
781def test_backup_index_preview_collection_end_to_end_against_a_torn_archived_file(tmp_path):
782 """Same FINDING, exercised end-to-end through the real public entry
783 point (preview_collection reading an actual .tar.gz archive member)
784 rather than calling the private parser directly -- confirms the bug is
785 reachable from the documented API surface, not just the internal
786 helper."""
787 data_dir = tmp_path / "data"
788 write_append_schema(data_dir, "invoices", [ID_FIELD, LINES_FIELD])
789 object_records.create_collection_record(
790 "invoices", {"id": "A", "lines": json.dumps([{"sku": "A-1"}])}, base_dir=data_dir, roots=[]
791 )
792 object_records.create_collection_record(
793 "invoices", {"id": "B", "lines": json.dumps([{"sku": "B-1"}])}, base_dir=data_dir, roots=[]
794 )
795 path = _records_path(data_dir, "invoices")
796 full_text = path.read_text()
797 assert full_text.endswith("\n")
798
799 hostile_blob = json.dumps([{"sku": "C-1"}, {"sku": "C-2"}], indent=2)
800 buf = io.StringIO()
801 writer = csv.writer(buf, delimiter="\t", lineterminator="\n", quoting=csv.QUOTE_MINIMAL)
802 writer.writerow(["", "C", hostile_blob])
803 full_row_text = buf.getvalue()
804 first_nl = full_row_text.index("\n")
805 second_nl = full_row_text.index("\n", first_nl + 1)
806 torn_fragment = full_row_text[: second_nl + 1]
807
808 # Simulate: the live file itself crashed at this exact unlucky
809 # boundary (matches the live-side FINDING scenario), and a backup was
810 # then taken of the runtime in that state.
811 path.write_text(full_text + torn_fragment)
812 assert path.read_text().endswith("\n")
813
814 objects_dir = tmp_path / "objects"
815 objects_dir.mkdir()
816
817 # preview_collection resolves the backup id under
818 # backups_dir(data_dir) = <data_dir>/backups, and separately diffs
819 # against the LIVE records.tsv under that SAME data_dir. To make the
820 # finding legible in the diff output (rather than backup and "live"
821 # both showing the identical torn bytes and folding identically), the
822 # archive is placed under a distinct "server_data" dir whose own
823 # collections/invoices does not exist -- so the diff's "added" list is
824 # purely "what preview_collection believes the backup contains".
825 server_data_dir = tmp_path / "server_data"
826 backup_output = server_data_dir / "backups" / "torn.tar.gz"
827 object_backup.create_runtime_backup(backup_output, objects_dir=objects_dir, data_dir=data_dir)
828
829 result = object_backup_index.preview_collection(
830 "torn.tar.gz", "invoices", data_dir=server_data_dir
831 )
832
833 print(f"\n[backup-index torn-tail FINDING, end-to-end] preview_collection "
834 f"added={result['added']!r}")
835
836 assert "A" in result["added"] and "B" in result["added"]
837 assert "C" in result["added"], (
838 "FINDING confirmed: preview_collection (the real public API atop "
839 "object_backup_index._parse_tsv_by_id) reports a torn mid-multiline "
840 f"row as a normal, present-in-backup record. Full diff: {result!r}"
841 )