Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_embedded_json_lines_characterization.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_embedded_json_lines_characterization.py · 39.3 KB · 868 lines
1"""CHARACTERIZATION tests: does the current TSV substrate (object_records.py)
2safely hold an embedded-JSON array field (a document's "lines") through both
3storage modes -- especially the append-mode, byte-offset-indexed single-record
4read path (the id->offset sidecar, ".records.oidx")?
5
6This file is pure characterization of EXISTING behavior. It does not modify
7any production module. Where a case fails or corrupts data, the assertion is
8written to state the correctness property that a "safe to hold embedded JSON"
9substrate SHOULD have, and is left to fail -- that failure IS the finding, and
10is documented in this file's docstrings and in the accompanying report. Do
11not "fix" a failing assertion here by weakening it to match broken behavior.
12
13Context: the design question being evaluated is whether order/invoice/journal
14LINE ITEMS can be modeled as a single embedded JSON array on the parent
15document (schemaless TSV + a `lines` field holding `json.dumps(...)`) rather
16than a separate `*_lines` collection. Two risk axes, per collection storage
17mode:
18
19 1. Hostile content: does a `lines` value containing TAB, NEWLINE,
20 DOUBLE-QUOTE, BACKSLASH, CRLF, or Unicode survive an exact round trip
21 through the TSV/CSV layer, in both the "read everything and fold" path
22 and the single-record-by-id path?
23 2. Scale: many items per array, and many documents, in both correctness and
24 wall-clock terms -- including the cost of MUTATING an item inside an
25 already-large array (the write-amplification concern for this model).
26
27One important nuance threaded through every "hostile content" test below:
28`json.dumps(...)` (the realistic way a `lines` value would be produced)
29ESCAPES every control character in a string VALUE -- a literal tab or
30newline typed by a user becomes the two-character sequence "\\t"/"\\n" in the
31JSON text, never a raw 0x09/0x0A byte. So a compact `json.dumps(...)` blob,
32however hostile the *logical* content, never actually presents a raw
33tab/newline byte to the TSV/CSV layer -- only literal double-quotes,
34backslashes, and Unicode do. The one realistic way a `lines` blob DOES end up
35carrying literal raw newline BYTES is pretty-printing (`json.dumps(...,
36indent=2)`, or any other human-readable serialization choice) -- so tests
37that need to exercise a genuinely multi-physical-line row use `indent=2`
38deliberately, and say so.
39"""
40
41from __future__ import annotations
42
43import csv
44import io
45import json
46import time
47from pathlib import Path
48
49import pytest
50
51import object_records
52
53
54ID_FIELD = {"name": "id"}
55LINES_FIELD = {"name": "lines", "type": "textarea"}
56
57
58# --- setup helpers (mirror tests/test_object_records.py's conventions) -----
59
60
61def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path:
62 path = data_dir / "schemas" / f"{collection}.json"
63 path.parent.mkdir(parents=True, exist_ok=True)
64 path.write_text(json.dumps({"fields": fields}))
65 return path
66
67
68def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path:
69 path = data_dir / "schemas" / f"{collection}.json"
70 path.parent.mkdir(parents=True, exist_ok=True)
71 path.write_text(json.dumps({"fields": fields, "storage": "append"}))
72 return path
73
74
75def _clear_caches() -> None:
76 """Force the NEXT read to go past the warm in-process caches: a cold
77 _RECORDS_CACHE + _OIDX_CACHE is what makes get_collection_record on an
78 append-mode file actually exercise the id->offset sidecar / _oidx_get_row
79 (see _fast_record_lookup) instead of serving from the ordinary records
80 cache -- which is what this whole file needs to test the right thing."""
81 object_records._RECORDS_CACHE.clear()
82 object_records._OIDX_CACHE.clear()
83
84
85def _hostile_lines() -> list[dict]:
86 return [
87 {
88 "sku": "SKU-1",
89 "note": "café \t x\n y \"q\" \\z 日本",
90 "crlf_field": "alpha\r\nbeta",
91 "amount_cents": 1099,
92 },
93 {
94 "sku": "SKU-2",
95 "note": "plain ascii, no surprises",
96 "amount_cents": -50,
97 },
98 ]
99
100
101# =============================================================================
102# 1. HOSTILE CONTENT, CLASSIC mode
103# =============================================================================
104
105
106def test_hostile_content_classic_mode_round_trip(tmp_path):
107 """Classic-mode parsing (_parse_classic_records) uses csv.reader over the
108 whole file handle, which is quote-aware ACROSS physical lines -- so a
109 pretty-printed (literal-raw-newline-bearing) JSON blob should round-trip
110 exactly, both via the fold-all read and the by-id read (classic mode
111 never touches the id->offset sidecar at all)."""
112 data_dir = tmp_path / "data"
113 write_schema(data_dir, "invoices", [ID_FIELD, LINES_FIELD])
114 original = _hostile_lines()
115 # Pretty-printed deliberately: this is what forces literal raw "\n"
116 # bytes into the stored TSV cell (see module docstring) -- the worst
117 # case for a naive delimiter scan, and the case classic mode's
118 # file-level csv.reader is expected to still get right.
119 blob = json.dumps(original, indent=2)
120
121 object_records.create_collection_record(
122 "invoices", {"id": "inv-1", "lines": blob}, base_dir=data_dir, roots=[]
123 )
124
125 _clear_caches()
126 all_records = object_records.read_collection_records("invoices", base_dir=data_dir, roots=[])
127 assert len(all_records) == 1
128 assert json.loads(all_records[0]["lines"]) == original
129
130 _clear_caches()
131 by_id = object_records.get_collection_record("invoices", "inv-1", base_dir=data_dir, roots=[])
132 assert json.loads(by_id["lines"]) == original
133
134
135# =============================================================================
136# 2. HOSTILE CONTENT, APPEND mode
137# =============================================================================
138
139
140def test_hostile_content_append_mode_compact_json_round_trip(tmp_path, monkeypatch):
141 """Compact json.dumps (no raw control bytes, but real literal
142 double-quotes/backslashes/Unicode) through append mode, forcing the
143 by-id read through the cold-cache id->offset sidecar path
144 (DBBASIC_RECORDS_CACHE_MAX_ROWS=0, same trick
145 tests/test_object_records.py's oidx tests use). Expected, and observed,
146 to round-trip correctly: nothing about this content spans more than one
147 physical line."""
148 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
149 data_dir = tmp_path / "data"
150 write_append_schema(data_dir, "invoices", [ID_FIELD, LINES_FIELD])
151 original = _hostile_lines()
152 blob = json.dumps(original)
153
154 object_records.create_collection_record(
155 "invoices", {"id": "inv-1", "lines": blob}, base_dir=data_dir, roots=[]
156 )
157
158 _clear_caches()
159 all_records = object_records.read_collection_records("invoices", base_dir=data_dir, roots=[])
160 assert json.loads(all_records[0]["lines"]) == original
161
162 _clear_caches()
163 by_id = object_records.get_collection_record("invoices", "inv-1", base_dir=data_dir, roots=[])
164 assert json.loads(by_id["lines"]) == original
165
166
167def test_hostile_content_append_mode_pretty_json_by_id_read_round_trips(
168 tmp_path, monkeypatch
169):
170 """REGRESSION GUARD for substrate fix #1 (was a FINDING, now fixed):
171 _oidx_get_row -- the function behind get_collection_record's
172 byte-offset-indexed single-record read, once the ordinary records cache
173 is cold -- used to read the row with a raw `handle.readline()` that
174 stopped at the FIRST b'\\n' byte, silently TRUNCATING any value that spans
175 multiple physical lines (e.g. this pretty-printed JSON blob) at its first
176 embedded newline, and (because "lines" is the last column) returning the
177 wrong value with no exception or fallback. The fix reads the full LOGICAL
178 row via csv.reader from the row offset, exactly as the full-fold path
179 (_parse_append_body) already did -- so the by-id read now consumes the
180 quoted multi-line field correctly. This test asserts both paths agree.
181 """
182 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
183 data_dir = tmp_path / "data"
184 write_append_schema(data_dir, "invoices", [ID_FIELD, LINES_FIELD])
185 original = _hostile_lines()
186 blob = json.dumps(original, indent=2) # pretty-printed: literal raw "\n" bytes in the cell
187
188 object_records.create_collection_record(
189 "invoices", {"id": "inv-1", "lines": blob}, base_dir=data_dir, roots=[]
190 )
191
192 _clear_caches()
193 all_records = object_records.read_collection_records("invoices", base_dir=data_dir, roots=[])
194 assert json.loads(all_records[0]["lines"]) == original # full fold: correct
195
196 _clear_caches()
197 by_id = object_records.get_collection_record("invoices", "inv-1", base_dir=data_dir, roots=[])
198 try:
199 loaded = json.loads(by_id["lines"])
200 except json.JSONDecodeError as exc:
201 pytest.fail(
202 "FINDING confirmed: the offset-indexed by-id read returned "
203 f"truncated/unparseable JSON for a row with an embedded literal "
204 f"newline (json.loads raised {exc!r}). Raw value returned: "
205 f"{by_id['lines']!r}"
206 )
207 assert loaded == original, (
208 "FINDING confirmed: the offset-indexed by-id read returned wrong "
209 f"data for a row with an embedded literal newline. Got: {loaded!r}"
210 )
211
212
213# =============================================================================
214# 3. OFFSET-INDEX CRUX: does the sidecar delimit rows CSV-aware, or naively?
215# =============================================================================
216
217
218def test_offset_index_crux_reads_survive_an_embedded_newline_row_between_them(
219 tmp_path, monkeypatch
220):
221 """THE central question this file exists to answer: when the id->offset
222 sidecar is built/caught-up (_scan_append_tail, shared by
223 _scan_append_rows_for_offsets and the record-cache's own tail-delta
224 fold), does it delimit physical rows with real CSV parsing, or with a
225 naive '\\n' scan? If naive, record B's embedded newlines (a
226 pretty-printed JSON blob) would throw off every row_start/row_end AFTER
227 it, corrupting C's and D's offsets even though neither of THEIR rows has
228 anything hostile in it.
229
230 ANSWER (read from the source, then verified here): _scan_append_tail
231 parses via `csv.reader` over `io.StringIO(text)` and computes each row's
232 byte span from the csv reader's OWN `stream.tell()` position (character
233 offset, converted to a byte length by re-encoding only the newly
234 consumed slice) -- genuinely CSV-aware, not a `.split("\\n")` or
235 `.count("\\n")` scan. This test forces the sidecar cold on every point
236 op (DBBASIC_RECORDS_CACHE_MAX_ROWS=0) and confirms C and D come back
237 correct by id, both before and after B is updated (a second physical
238 row, also multi-line) -- i.e. the scan's row-boundary tracking is
239 correct across a multi-physical-line row. (This is a DIFFERENT question
240 from "is B's OWN by-id read correct" -- it isn't, see the FINDING test
241 above; that is a distinct bug in how a row is RE-READ from its offset,
242 not in how offsets are COMPUTED during the scan.)
243 """
244 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
245 data_dir = tmp_path / "data"
246 write_append_schema(data_dir, "invoices", [ID_FIELD, LINES_FIELD])
247
248 a_lines = [{"sku": "A-1", "amount_cents": 100}]
249 b_lines_v1 = _hostile_lines()
250 c_lines = [{"sku": "C-1", "amount_cents": 300}, {"sku": "C-2", "amount_cents": 301}]
251 d_lines = [{"sku": "D-1", "amount_cents": 400}]
252
253 object_records.create_collection_record(
254 "invoices", {"id": "A", "lines": json.dumps(a_lines)}, base_dir=data_dir, roots=[]
255 )
256 object_records.create_collection_record(
257 "invoices",
258 {"id": "B", "lines": json.dumps(b_lines_v1, indent=2)}, # multi-physical-line row
259 base_dir=data_dir,
260 roots=[],
261 )
262 object_records.create_collection_record(
263 "invoices", {"id": "C", "lines": json.dumps(c_lines)}, base_dir=data_dir, roots=[]
264 )
265 object_records.create_collection_record(
266 "invoices", {"id": "D", "lines": json.dumps(d_lines)}, base_dir=data_dir, roots=[]
267 )
268
269 _clear_caches()
270 rec_c = object_records.get_collection_record("invoices", "C", base_dir=data_dir, roots=[])
271 assert json.loads(rec_c["lines"]) == c_lines
272
273 _clear_caches()
274 rec_d = object_records.get_collection_record("invoices", "D", base_dir=data_dir, roots=[])
275 assert json.loads(rec_d["lines"]) == d_lines
276
277 # Update B (a SECOND multi-physical-line physical row, superseding the
278 # first) and confirm C/D still resolve correctly afterwards -- the
279 # sidecar's catch-up scan must correctly skip over BOTH of B's physical
280 # rows to keep later offsets right.
281 b_lines_v2 = _hostile_lines() + [{"sku": "B-extra", "amount_cents": 999}]
282 object_records.update_collection_record(
283 "invoices", "B", {"lines": json.dumps(b_lines_v2, indent=2)}, base_dir=data_dir, roots=[]
284 )
285
286 _clear_caches()
287 rec_c2 = object_records.get_collection_record("invoices", "C", base_dir=data_dir, roots=[])
288 assert json.loads(rec_c2["lines"]) == c_lines
289
290 _clear_caches()
291 rec_d2 = object_records.get_collection_record("invoices", "D", base_dir=data_dir, roots=[])
292 assert json.loads(rec_d2["lines"]) == d_lines
293
294
295# =============================================================================
296# 4. SCALE -- many lines per doc
297# =============================================================================
298
299
300def _item_shape(i: int) -> dict:
301 return {"sku": f"SKU-{i:05d}", "qty": (i % 7) + 1, "price_cents": 1099 + i, "tax_cents": 87}
302
303
304def test_scale_many_lines_per_doc_round_trip(tmp_path):
305 """Correctness (not timing -- see the many-docs test below) at
306 increasing embedded-array size, both storage modes: 10/100/1000 line
307 items -- all comfortably under Python's csv module's default
308 field_size_limit (131072 bytes / 128 KiB; see the dedicated FINDING
309 test right below this one for what happens once a `lines` blob crosses
310 that ceiling, which this item shape does well before 5000 items).
311 Reports the serialized blob's byte size at each size, since that --
312 not the item count itself -- is what actually drives write/parse
313 cost."""
314 data_dir = tmp_path / "data"
315 write_schema(data_dir, "orders_classic", [ID_FIELD, LINES_FIELD])
316 write_append_schema(data_dir, "orders_append", [ID_FIELD, LINES_FIELD])
317
318 sizes_report: dict[int, int] = {}
319 for n in (10, 100, 1000):
320 lines = [_item_shape(i) for i in range(n)]
321 blob = json.dumps(lines)
322 sizes_report[n] = len(blob.encode("utf-8"))
323 record_id = f"ord-{n}"
324
325 object_records.create_collection_record(
326 "orders_classic", {"id": record_id, "lines": blob}, base_dir=data_dir, roots=[]
327 )
328 object_records.create_collection_record(
329 "orders_append", {"id": record_id, "lines": blob}, base_dir=data_dir, roots=[]
330 )
331
332 for collection in ("orders_classic", "orders_append"):
333 _clear_caches()
334 all_recs = object_records.read_collection_records(
335 collection, base_dir=data_dir, roots=[]
336 )
337 match = next(r for r in all_recs if r["id"] == record_id)
338 assert json.loads(match["lines"]) == lines
339
340 _clear_caches()
341 by_id = object_records.get_collection_record(
342 collection, record_id, base_dir=data_dir, roots=[]
343 )
344 assert json.loads(by_id["lines"]) == lines
345
346 print("\n[scale] serialized 'lines' blob size (bytes) by item count:")
347 for n, nbytes in sizes_report.items():
348 print(f" n={n:>5} bytes={nbytes:>8}")
349
350
351def test_scale_large_field_round_trips_and_oversize_rejected_on_write(tmp_path):
352 """REGRESSION GUARD for substrate fix #3 (was a FINDING, now fixed):
353 csv.field_size_limit is raised to MAX_TSV_FIELD_BYTES at module load and
354 create/update reject any field over that cap -- so a large-but-under-cap
355 cell round-trips and an over-cap write is refused, instead of silently
356 corrupting reads. The original finding this guards against:
357
358 FINDING (see report -- independent of, and arguably more
359 fundamental than, the offset-index/torn-tail findings above): nothing
360 in object_records.py ever raises Python's `csv` module's default
361 `field_size_limit()` (131072 bytes / 128 KiB) above its stdlib
362 default. A `lines` blob past that size -- 5000 items of this shape
363 serializes to ~350 KB, well over it; the crossover for THIS item shape
364 (~70 bytes/item) is roughly 1800-1900 items, printed below -- is
365 written to disk just fine (csv.writer enforces no such limit), but
366 becomes UNREADABLE the moment anything tries to read the collection
367 back, and the failure mode is different, and worse, in append mode
368 than in classic mode:
369
370 - CLASSIC mode: `_parse_classic_records` has no try/except around
371 row iteration at all, so `_csv.Error: field larger than field
372 limit (131072)` propagates UNCAUGHT out of
373 read_collection_records/get_collection_record. Loud, but total:
374 the entire collection (every OTHER record in it too, not just the
375 oversized one) becomes unreadable via any read path until the
376 oversized value is removed or the process's field_size_limit is
377 raised -- there is no per-record isolation.
378 - APPEND mode: both `_parse_append_body` and `_scan_append_tail`
379 catch `csv.Error` and treat it exactly like a torn tail (`break`,
380 stop consuming, don't raise) -- which is the right instinct for a
381 genuinely torn write, but wrong here: the oversized field is
382 legitimately, fully committed. The practical effect: since the
383 oversized row in this test is the FIRST physical row, parsing
384 breaks immediately and the fold produces ZERO records -- the
385 entire collection (every record, not just the oversized one)
386 silently reads back as EMPTY on every path (list, and by-id, since
387 an incoherent/empty sidecar rebuild falls through to the same
388 empty fold) -- no exception at all. This is silent, total,
389 collection-wide data loss on the READ side (the bytes are still on
390 disk; nothing can see them) triggered by ONE oversized document.
391
392 This is a hard, item-shape-independent ceiling any embedded-JSON-array
393 design must budget for: a single richly-detailed invoice/order/journal
394 (many line items, or line items with long descriptions) can trivially
395 exceed 128 KB.
396 """
397 data_dir = tmp_path / "data"
398 write_schema(data_dir, "orders_classic", [ID_FIELD, LINES_FIELD])
399 write_append_schema(data_dir, "orders_append", [ID_FIELD, LINES_FIELD])
400
401 # A large-but-under-cap value (~350 KB, far below the 16 MiB
402 # MAX_TSV_FIELD_BYTES) now round-trips on EVERY path in BOTH modes. Before
403 # fix #3 this same value silently emptied the whole append collection on
404 # read; now the raised csv.field_size_limit lets it parse.
405 n = 5000
406 lines = [_item_shape(i) for i in range(n)]
407 blob = json.dumps(lines)
408 blob_bytes = len(blob.encode("utf-8"))
409 assert blob_bytes > 131072 # over the old 128 KiB stdlib default...
410 assert blob_bytes < object_records.MAX_TSV_FIELD_BYTES # ...but under our cap
411 print(f"\n[scale] blob={blob_bytes} bytes: over 128 KiB, under the "
412 f"{object_records.MAX_TSV_FIELD_BYTES}-byte cap -- must round-trip")
413
414 for coll in ("orders_classic", "orders_append"):
415 object_records.create_collection_record(
416 coll, {"id": "big-1", "lines": blob}, base_dir=data_dir, roots=[]
417 )
418 object_records.create_collection_record(
419 coll, {"id": "small-2", "lines": json.dumps([{"sku": "small"}])},
420 base_dir=data_dir, roots=[],
421 )
422 _clear_caches()
423 all_rows = object_records.read_collection_records(coll, base_dir=data_dir, roots=[])
424 assert {r["id"] for r in all_rows} == {"big-1", "small-2"}, (
425 f"{coll}: an oversize-but-under-cap field must not break the read "
426 f"(nor drop its siblings); got {all_rows!r}"
427 )
428 _clear_caches()
429 got = object_records.get_collection_record(coll, "big-1", base_dir=data_dir, roots=[])
430 assert json.loads(got["lines"]) == lines, (
431 f"{coll}: by-id read of the large field must round-trip exactly"
432 )
433
434 # Over the cap: create REJECTS the write with a structured error rather
435 # than persist a row that cannot be read back.
436 over = "x" * (object_records.MAX_TSV_FIELD_BYTES + 1)
437 for coll in ("orders_classic", "orders_append"):
438 with pytest.raises(object_records.InvalidRecordPayloadError):
439 object_records.create_collection_record(
440 coll, {"id": "too-big", "lines": over}, base_dir=data_dir, roots=[]
441 )
442
443
444# =============================================================================
445# 5. SCALE -- many docs (timing)
446# =============================================================================
447
448
449def test_scale_many_docs_ecommerce_shape_append_mode_timing(tmp_path):
450 """1000 docs x 5-line arrays (the ecommerce shape: many small documents),
451 then 200 docs x 500-line arrays, append mode. Wall-clock characterization
452 only (time.perf_counter) -- the only hard assertions are that a sampled
453 read from each population is correct."""
454 data_dir = tmp_path / "data"
455 write_append_schema(data_dir, "orders_small", [ID_FIELD, LINES_FIELD])
456 write_append_schema(data_dir, "orders_big", [ID_FIELD, LINES_FIELD])
457
458 def make_lines(n: int, seed: int) -> list[dict]:
459 return [
460 {"sku": f"SKU-{seed}-{i:04d}", "qty": (i % 5) + 1, "price_cents": 500 + i}
461 for i in range(n)
462 ]
463
464 docs_small = 1000
465 lines_per_small_doc = 5
466 t0 = time.perf_counter()
467 for i in range(docs_small):
468 blob = json.dumps(make_lines(lines_per_small_doc, i))
469 object_records.create_collection_record(
470 "orders_small", {"id": f"small-{i:05d}", "lines": blob}, base_dir=data_dir, roots=[]
471 )
472 write_elapsed_small = time.perf_counter() - t0
473
474 _clear_caches()
475 t0 = time.perf_counter()
476 sample_small = object_records.get_collection_record(
477 "orders_small", "small-00500", base_dir=data_dir, roots=[]
478 )
479 read_elapsed_small = time.perf_counter() - t0
480 assert json.loads(sample_small["lines"]) == make_lines(lines_per_small_doc, 500)
481
482 docs_big = 200
483 lines_per_big_doc = 500
484 t0 = time.perf_counter()
485 for i in range(docs_big):
486 blob = json.dumps(make_lines(lines_per_big_doc, i))
487 object_records.create_collection_record(
488 "orders_big", {"id": f"big-{i:04d}", "lines": blob}, base_dir=data_dir, roots=[]
489 )
490 write_elapsed_big = time.perf_counter() - t0
491
492 _clear_caches()
493 t0 = time.perf_counter()
494 sample_big = object_records.get_collection_record(
495 "orders_big", "big-0100", base_dir=data_dir, roots=[]
496 )
497 read_elapsed_big = time.perf_counter() - t0
498 assert json.loads(sample_big["lines"]) == make_lines(lines_per_big_doc, 100)
499
500 print("\n[scale] many-docs append-mode timing:")
501 print(
502 f" {docs_small} docs x {lines_per_small_doc} lines: "
503 f"total write={write_elapsed_small:.4f}s "
504 f"({write_elapsed_small / docs_small * 1000:.3f}ms/doc), "
505 f"single by-id read={read_elapsed_small * 1000:.3f}ms"
506 )
507 print(
508 f" {docs_big} docs x {lines_per_big_doc} lines: "
509 f"total write={write_elapsed_big:.4f}s "
510 f"({write_elapsed_big / docs_big * 1000:.3f}ms/doc), "
511 f"single by-id read={read_elapsed_big * 1000:.3f}ms"
512 )
513
514
515# =============================================================================
516# 6. TORN TAIL, APPEND mode -- with an embedded newline in the partial row
517# =============================================================================
518
519
520def test_append_mode_torn_tail_ordinary_case_still_self_heals_with_lines_field(tmp_path):
521 """Baseline (expected-safe) case, mirroring
522 test_append_mode_torn_tail_is_ignored_and_self_heals in
523 tests/test_object_records.py exactly, just with a `lines` field/compact
524 JSON payload in the mix: a crash that lands in the middle of a SINGLE
525 physical line (no embedded newline in the torn fragment itself) is
526 correctly detected and dropped, and self-heals on the next write."""
527 data_dir = tmp_path / "data"
528 write_append_schema(data_dir, "invoices", [ID_FIELD, LINES_FIELD])
529 object_records.create_collection_record(
530 "invoices",
531 {"id": "A", "lines": json.dumps([{"sku": "A-1"}])},
532 base_dir=data_dir,
533 roots=[],
534 )
535 object_records.create_collection_record(
536 "invoices",
537 {"id": "B", "lines": json.dumps([{"sku": "B-1"}])},
538 base_dir=data_dir,
539 roots=[],
540 )
541 path = data_dir / "collections" / "invoices" / "records.tsv"
542 full_text = path.read_text()
543
544 # Chop the last few bytes off (no embedded newline involved -- ordinary
545 # single-physical-line torn tail).
546 path.write_text(full_text[:-4])
547 assert not path.read_text().endswith("\n")
548
549 _clear_caches()
550 before_stat = path.stat()
551 listing = object_records.list_collection_records("invoices", base_dir=data_dir, roots=[])[
552 "records"
553 ]
554 after_stat = path.stat()
555 assert listing == [{"id": "A", "lines": json.dumps([{"sku": "A-1"}])}]
556 assert before_stat.st_mtime_ns == after_stat.st_mtime_ns
557 assert before_stat.st_size == after_stat.st_size
558
559 object_records.create_collection_record(
560 "invoices",
561 {"id": "C", "lines": json.dumps([{"sku": "C-1"}])},
562 base_dir=data_dir,
563 roots=[],
564 )
565 _clear_caches()
566 listing_after = object_records.list_collection_records("invoices", base_dir=data_dir, roots=[])[
567 "records"
568 ]
569 assert listing_after == [
570 {"id": "A", "lines": json.dumps([{"sku": "A-1"}])},
571 {"id": "C", "lines": json.dumps([{"sku": "C-1"}])},
572 ]
573
574
575def test_append_mode_torn_tail_mid_multiline_row_is_dropped_and_never_cascades(
576 tmp_path,
577):
578 """The original bug #2 finding, now its regression guard.
579
580 The scenario is unchanged from the FINDING version -- record C torn
581 mid-multiline-row at the unluckiest boundary, right after an internal
582 newline, so the file's raw bytes end in "\n" and the old quote-blind
583 check saw a committed tail. What is asserted flipped from the two
584 failure effects to their absence: a bare read shows [A, B] with no
585 phantom C, and a following create of D lands as its own record instead
586 of being swallowed into C's unclosed quote.
587
588 Original finding text, kept because the mechanism is the documentation:
589
590 FORMER FINDING (was the most serious one): the torn-tail
591 safety net (_drop_torn_tail / _repair_torn_tail) decides "is the file's
592 tail committed?" with exactly one check: does the file end with a
593 literal b'\\n' byte. That check is unsound for any row whose value spans
594 multiple physical lines (e.g. a pretty-printed embedded-JSON `lines`
595 blob): a real crash can flush and stop at ANY byte boundary, including
596 one that lands right after an INTERNAL newline inside the still-open
597 quoted field -- at which point the file "ends with \\n" even though the
598 row (and the write) never actually completed.
599
600 Reproduced here: a row for id "C" is deliberately cut after its SECOND
601 physical line (mid-quote, but the cut point itself is right after a
602 newline, so the file's raw bytes end in \\n). Observed effects, via the
603 real public API only (create_collection_record / list_collection_records
604 -- no internal function called directly):
605
606 1. A plain read (no write at all) immediately reports C as a normal,
607 fully-formed committed record -- with a truncated/garbled `lines`
608 value -- even though C's write never completed. This is silent
609 data resurrection from a torn write, not "torn tail correctly
610 dropped."
611 2. Because C's quoted field was never actually closed on disk, EVERY
612 row appended after it (self-heal does not trigger -- the file
613 already "ends with \\n" per the same broken check, so
614 _repair_torn_tail sees nothing to truncate) gets silently absorbed
615 AS TEXT into C's still-open "lines" value. A subsequent normal
616 create of record "D" does not produce a new, distinct record D at
617 all -- D's entire physical row becomes part of C's corrupted
618 value, and D is permanently missing from every future read.
619
620 This is worse than the offset-index truncation FINDING above: it is not
621 limited to the (already cold-cache-gated) by-id read path, it requires
622 no cache-warmth conditions to observe, and it is not self-limiting to
623 one row -- it corrupts and swallows every subsequent write until the
624 collection is compacted (compaction rewrites from the in-memory FOLDED
625 view, which already contains C's corrupted value and never recovers D).
626 """
627 data_dir = tmp_path / "data"
628 write_append_schema(data_dir, "invoices", [ID_FIELD, LINES_FIELD])
629 object_records.create_collection_record(
630 "invoices",
631 {"id": "A", "lines": json.dumps([{"sku": "A-1"}])},
632 base_dir=data_dir,
633 roots=[],
634 )
635 object_records.create_collection_record(
636 "invoices",
637 {"id": "B", "lines": json.dumps([{"sku": "B-1"}])},
638 base_dir=data_dir,
639 roots=[],
640 )
641 path = data_dir / "collections" / "invoices" / "records.tsv"
642 full_text = path.read_text()
643 assert full_text.endswith("\n")
644
645 # Build what row C's bytes WOULD have been (a full, well-formed physical
646 # row for a pretty-printed, multi-line "lines" value), via the exact
647 # same csv dialect object_records itself writes with -- then keep only a
648 # PREFIX that cuts right after the row's second physical line, so the
649 # leftover fragment (a) is genuinely incomplete (the quote never
650 # closes) and (b) itself ends with "\n" -- simulating a crash that
651 # landed at that unlucky byte boundary.
652 hostile_blob = json.dumps(_hostile_lines(), indent=2)
653 buf = io.StringIO()
654 writer = csv.writer(buf, delimiter="\t", lineterminator="\n", quoting=csv.QUOTE_MINIMAL)
655 writer.writerow(["", "C", hostile_blob])
656 full_row_text = buf.getvalue()
657 assert full_row_text.count("\n") > 3 # sanity: genuinely spans several physical lines
658
659 first_nl = full_row_text.index("\n")
660 second_nl = full_row_text.index("\n", first_nl + 1)
661 torn_fragment = full_row_text[: second_nl + 1]
662 assert torn_fragment.endswith("\n") # the crux: looks "committed" by the naive check
663
664 path.write_text(full_text + torn_fragment)
665 assert path.read_text().endswith("\n") # file passes the "torn tail" check despite being torn
666
667 # Guard 1: a plain read DROPS the torn C rather than resurrecting it.
668 _clear_caches()
669 listing = object_records.list_collection_records("invoices", base_dir=data_dir, roots=[])[
670 "records"
671 ]
672 ids = [r["id"] for r in listing]
673 assert ids == ["A", "B"], (
674 f"read-side regression: the torn C must be dropped, got {ids!r}"
675 )
676
677 # Guard 2: the next NORMAL write self-heals -- the repair truncates the
678 # unclosed-quote fragment before appending, so D lands as its own row.
679 object_records.create_collection_record(
680 "invoices",
681 {"id": "D", "lines": json.dumps([{"sku": "D-1"}])},
682 base_dir=data_dir,
683 roots=[],
684 )
685 _clear_caches()
686 listing_after = object_records.list_collection_records(
687 "invoices", base_dir=data_dir, roots=[]
688 )["records"]
689 ids_after = [r["id"] for r in listing_after]
690 assert ids_after == ["A", "B", "D"], (
691 f"write-side regression: D must land as its own record after the "
692 f"quote-aware repair, got {ids_after!r} / {listing_after!r}"
693 )
694 by_id = {r["id"]: r for r in listing_after}
695 assert by_id["D"]["lines"] == json.dumps([{"sku": "D-1"}])
696
697
698# =============================================================================
699# 7. MUTATION PERFORMANCE -- editing one item inside an already-stored array
700# =============================================================================
701
702
703def test_mutation_edit_one_item_performance_classic_vs_append(tmp_path):
704 """Characterization: cost of editing ONE item inside an N-item embedded
705 JSON array and re-writing the record, classic vs append storage, at
706 N = 5/50/500/1000. Each collection also carries a fixed backdrop of
707 SIBLING_COUNT small sibling documents, so the thing this comparison is
708 actually about -- classic mode's full-FILE rewrite touching every OTHER
709 document too, vs append mode's O(1) delta row -- has something to show
710 up against. (Without any sibling backdrop, a single-document collection
711 makes classic and append cost about the same: both are then dominated
712 purely by the edited blob's own size, since there's nothing else to
713 amplify.)
714
715 Not a hard-assertion timing test -- see the printed table in the pytest
716 -s output -- but round-trip correctness of the edited record IS
717 asserted, in both modes, at every N."""
718 SIBLING_COUNT = 150
719 ITEM_SIZES = (5, 50, 500, 1000)
720 results: dict[tuple[str, int], float] = {}
721
722 for storage in ("classic", "append"):
723 for n in ITEM_SIZES:
724 collection = f"mut1_{storage}_{n}"
725 data_dir = tmp_path / f"data_{storage}_{n}"
726 if storage == "classic":
727 write_schema(data_dir, collection, [ID_FIELD, LINES_FIELD])
728 else:
729 write_append_schema(data_dir, collection, [ID_FIELD, LINES_FIELD])
730
731 for i in range(SIBLING_COUNT):
732 sib_lines = [{"sku": f"sib-{i}-{j}", "price_cents": 100 + j} for j in range(3)]
733 object_records.create_collection_record(
734 collection,
735 {"id": f"sib-{i:04d}", "lines": json.dumps(sib_lines)},
736 base_dir=data_dir,
737 roots=[],
738 )
739
740 target_lines = [
741 {"sku": f"item-{i:04d}", "qty": 1, "price_cents": 1000 + i} for i in range(n)
742 ]
743 object_records.create_collection_record(
744 collection,
745 {"id": "target", "lines": json.dumps(target_lines)},
746 base_dir=data_dir,
747 roots=[],
748 )
749
750 # The edit: bump item[0]'s price by one cent, then re-serialize
751 # the WHOLE array -- there is no partial-field update for an
752 # embedded array; the entire blob must be rewritten even though
753 # only one of its N items changed.
754 current = object_records.get_collection_record(
755 collection, "target", base_dir=data_dir, roots=[]
756 )
757 items = json.loads(current["lines"])
758 original_first_price = items[0]["price_cents"]
759 items[0]["price_cents"] += 1
760 new_blob = json.dumps(items)
761
762 t0 = time.perf_counter()
763 object_records.update_collection_record(
764 collection, "target", {"lines": new_blob}, base_dir=data_dir, roots=[]
765 )
766 elapsed = time.perf_counter() - t0
767 results[(storage, n)] = elapsed
768
769 after = object_records.get_collection_record(
770 collection, "target", base_dir=data_dir, roots=[]
771 )
772 after_items = json.loads(after["lines"])
773 assert after_items[0]["price_cents"] == original_first_price + 1
774 assert after_items[1:] == items[1:]
775
776 print(
777 f"\n[mutation] edit-one-item latency, classic vs append "
778 f"({SIBLING_COUNT} sibling docs backdrop):"
779 )
780 print(f" {'N items':>8} {'classic (ms)':>14} {'append (ms)':>14}")
781 for n in ITEM_SIZES:
782 c = results[("classic", n)] * 1000
783 a = results[("append", n)] * 1000
784 print(f" {n:>8} {c:>14.3f} {a:>14.3f}")
785
786
787def test_mutation_repeated_edits_compaction_effect_on_read_latency(tmp_path, monkeypatch):
788 """K=50 repeated edits of the SAME big (N=500-item) embedded-array
789 document, append mode: each edit re-writes the whole 500-item blob as
790 one new physical row (append mode's write amplification per the
791 previous test), so 50 edits leaves 50 superseded physical rows plus the
792 original sitting in records.tsv before compaction runs. A backdrop of
793 BACKDROP_COUNT small sibling rows is included so the id->offset
794 sidecar's on-disk body (one line per PHYSICAL row of the WHOLE
795 collection, not just this one document) is big enough for a cold
796 reload's cost to be plausibly measurable.
797 DBBASIC_RECORDS_CACHE_MAX_ROWS=0 forces every point op through the
798 sidecar rather than the ordinary records cache, matching how a
799 collection past that threshold behaves in production; caches are then
800 explicitly cleared (_clear_caches) immediately before each of the two
801 timed reads so both measure a genuinely cold reload, not a warm hit."""
802 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
803 data_dir = tmp_path / "data"
804 collection = "mut2"
805 write_append_schema(data_dir, collection, [ID_FIELD, LINES_FIELD])
806
807 BACKDROP_COUNT = 1500
808 N = 500
809 K = 50
810
811 for i in range(BACKDROP_COUNT):
812 sib_lines = [{"sku": f"sib-{i}-{j}"} for j in range(3)]
813 object_records.create_collection_record(
814 collection,
815 {"id": f"sib-{i:05d}", "lines": json.dumps(sib_lines)},
816 base_dir=data_dir,
817 roots=[],
818 )
819
820 big_lines = [{"sku": f"item-{i:04d}", "price_cents": 1000 + i} for i in range(N)]
821 object_records.create_collection_record(
822 collection, {"id": "big", "lines": json.dumps(big_lines)}, base_dir=data_dir, roots=[]
823 )
824
825 for k in range(K):
826 current = object_records.get_collection_record(collection, "big", base_dir=data_dir, roots=[])
827 items = json.loads(current["lines"])
828 items[k % N]["price_cents"] += 1
829 object_records.update_collection_record(
830 collection, "big", {"lines": json.dumps(items)}, base_dir=data_dir, roots=[]
831 )
832
833 stats_before = object_records.append_collection_stats(collection, base_dir=data_dir, roots=[])
834
835 _clear_caches()
836 t0 = time.perf_counter()
837 before_read = object_records.get_collection_record(collection, "big", base_dir=data_dir, roots=[])
838 read_before = time.perf_counter() - t0
839 before_items = json.loads(before_read["lines"])
840 assert len(before_items) == N # correctness held through 50 edits
841
842 compact_result = object_records.compact_collection(collection, base_dir=data_dir, roots=[])
843 stats_after = object_records.append_collection_stats(collection, base_dir=data_dir, roots=[])
844
845 _clear_caches()
846 t0 = time.perf_counter()
847 after_read = object_records.get_collection_record(collection, "big", base_dir=data_dir, roots=[])
848 read_after = time.perf_counter() - t0
849 after_items = json.loads(after_read["lines"])
850
851 assert after_items == before_items # compaction must not change live content
852 assert compact_result["rows_after"] < compact_result["rows_before"]
853
854 print(
855 f"\n[mutation] {K} repeated edits of one {N}-item document "
856 f"(+{BACKDROP_COUNT} sibling docs backdrop), append mode:"
857 )
858 print(
859 f" physical_rows before compaction: {stats_before['physical_rows']} "
860 f"(live_rows={stats_before['live_rows']}, bloat_ratio={stats_before['bloat_ratio']})"
861 )
862 print(
863 f" physical_rows after compaction: {stats_after['physical_rows']} "
864 f"(live_rows={stats_after['live_rows']}, bloat_ratio={stats_after['bloat_ratio']})"
865 )
866 print(f" compact_collection result: {compact_result}")
867 print(f" cold single-record-by-id read BEFORE compaction: {read_before * 1000:.3f}ms")
868 print(f" cold single-record-by-id read AFTER compaction: {read_after * 1000:.3f}ms")