Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_charset_ascii_control_characterization.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_charset_ascii_control_characterization.py · 27.9 KB · 620 lines
1"""CHARACTERIZATION tests: do the full ASCII printable range, C0 control
2characters (0x00-0x1F), DEL (0x7F), and the TSV/CSV-significant bytes
3(TAB, double-quote, backslash, CR, LF, CRLF) round-trip safely through the
4record storage layer (object_records.py), as either a raw field value or a
5field holding a compact `json.dumps(...)` blob -- in both storage modes
6(classic whole-file rewrite, and append byte-offset-indexed) and via all
7three read/write paths (fold-all, by-id, update)?
8
9This file is pure characterization of EXISTING behavior. It does not modify
10any production module (object_records.py or any other production module is
11untouched). Where a case fails or corrupts data, the assertion states the
12correctness property a safe substrate SHOULD have and is left to fail --
13that failure IS the finding. Do not "fix" a failing assertion here by
14weakening it to match broken behavior.
15
16OUT OF SCOPE (do not probe): torn-tail / crash-mid-write scenarios. That is
17documented substrate bug #2 (quote-blind torn-tail truncation), covered by
18tests/test_embedded_json_lines_characterization.py's xfail test. Every test
19below writes complete, well-formed records via the public API only.
20
21Conventions (setup helpers, base_dir/roots usage, cache-clearing) mirrored
22from tests/test_embedded_json_lines_characterization.py.
23"""
24
25from __future__ import annotations
26
27import json
28from pathlib import Path
29
30import pytest
31
32import object_records
33
34# Conformance tier: heavy charset characterization -- deselected from the
35# per-commit run (see pyproject 'conformance' marker). Run: pytest -m conformance
36pytestmark = pytest.mark.conformance
37
38
39ID_FIELD = {"name": "id"}
40RAW_FIELD = {"name": "raw", "type": "text"}
41TRAILER_FIELD = {"name": "trailer", "type": "text"}
42FULL_FIELDS = [ID_FIELD, RAW_FIELD, TRAILER_FIELD]
43TRAILER_SENTINEL = "TRAILER_OK"
44
45
46# --- setup helpers (mirror test_embedded_json_lines_characterization.py) ---
47
48
49def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path:
50 path = data_dir / "schemas" / f"{collection}.json"
51 path.parent.mkdir(parents=True, exist_ok=True)
52 path.write_text(json.dumps({"fields": fields}))
53 return path
54
55
56def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path:
57 path = data_dir / "schemas" / f"{collection}.json"
58 path.parent.mkdir(parents=True, exist_ok=True)
59 path.write_text(json.dumps({"fields": fields, "storage": "append"}))
60 return path
61
62
63def _clear_caches() -> None:
64 """Force the NEXT read to go past the warm in-process caches -- see the
65 matching helper in test_embedded_json_lines_characterization.py for why
66 this is what makes get_collection_record actually exercise the
67 id->offset sidecar / _oidx_get_row cold path on an append-mode file."""
68 object_records._RECORDS_CACHE.clear()
69 object_records._OIDX_CACHE.clear()
70
71
72# --- char inventory ----------------------------------------------------
73
74_CONTROL_NAMES = {
75 0x00: "NUL", 0x01: "SOH", 0x02: "STX", 0x03: "ETX", 0x04: "EOT", 0x05: "ENQ",
76 0x06: "ACK", 0x07: "BEL", 0x08: "BS", 0x09: "TAB", 0x0A: "LF", 0x0B: "VT",
77 0x0C: "FF", 0x0D: "CR", 0x0E: "SO", 0x0F: "SI", 0x10: "DLE", 0x11: "DC1",
78 0x12: "DC2", 0x13: "DC3", 0x14: "DC4", 0x15: "NAK", 0x16: "SYN", 0x17: "ETB",
79 0x18: "CAN", 0x19: "EM", 0x1A: "SUB", 0x1B: "ESC", 0x1C: "FS", 0x1D: "GS",
80 0x1E: "RS", 0x1F: "US", 0x7F: "DEL",
81}
82
83PRINTABLE_CODEPOINTS = list(range(0x20, 0x7F)) # 0x20 space .. 0x7E '~'
84CONTROL_CODEPOINTS = list(range(0x00, 0x20)) + [0x7F] # C0 set + DEL
85ALL_ASCII_CODEPOINTS = list(range(0x00, 0x80)) # full 7-bit range
86
87
88def _char_label(cp: int) -> str:
89 if 0x20 <= cp <= 0x7E:
90 return f"0x{cp:02X} {chr(cp)!r}"
91 return f"0x{cp:02X} {_CONTROL_NAMES.get(cp, '?')}"
92
93
94# --- shared matrix runner ------------------------------------------------
95
96
97def _run_charset_matrix(
98 data_dir: Path,
99 collection: str,
100 storage: str, # "classic" | "append"
101 codepoints: list[int],
102 *,
103 wrap_json: bool,
104) -> list[str]:
105 """Create one record per codepoint (embedded in a raw string, optionally
106 wrapped in a compact json.dumps blob), alongside a fixed `trailer`
107 sentinel field, then verify an EXACT round trip via both
108 read_collection_records (fold) and get_collection_record (by-id).
109
110 The `trailer` field is the delimiter/column-shift detector: if a
111 character corrupts CSV parsing (splits a row into extra columns, eats a
112 following column, etc.) the trailer value -- a fixed sibling column,
113 unrelated to the char under test -- is exactly the kind of collateral
114 damage that would show up.
115
116 For append mode, DBBASIC_RECORDS_CACHE_MAX_ROWS=0 must already be set by
117 the caller (monkeypatch) so every get_collection_record call actually
118 goes through the id->offset sidecar's cold path instead of a warm
119 _RECORDS_CACHE hit -- see _fast_record_lookup.
120
121 Returns a list of human-readable failure descriptions; empty means every
122 codepoint round-tripped exactly on both paths.
123 """
124 if storage == "classic":
125 write_schema(data_dir, collection, FULL_FIELDS)
126 elif storage == "append":
127 write_append_schema(data_dir, collection, FULL_FIELDS)
128 else:
129 raise ValueError(storage)
130
131 failures: list[str] = []
132 expected: dict[str, str] = {}
133 for cp in codepoints:
134 ch = chr(cp)
135 raw_value = f"pre{ch}mid{ch}post"
136 value = json.dumps([{"v": raw_value, "cp": cp}]) if wrap_json else raw_value
137 record_id = f"c{cp:03d}"
138 try:
139 object_records.create_collection_record(
140 collection,
141 {"id": record_id, "raw": value, "trailer": TRAILER_SENTINEL},
142 base_dir=data_dir,
143 roots=[],
144 )
145 except Exception as exc: # noqa: BLE001 -- characterizing, want the exact exception
146 failures.append(
147 f"[create] {_char_label(cp)}: create_collection_record raised "
148 f"{type(exc).__module__}.{type(exc).__name__}: {exc!r}"
149 )
150 continue
151 # Only a successfully created record is checked for round-trip below --
152 # a create-time exception IS itself the finding, recorded above.
153 expected[record_id] = value
154
155 def _check(tag: str, record_id: str, cp: int, value: str, row: dict | None) -> None:
156 if row is None:
157 failures.append(f"[{tag}] {_char_label(cp)}: record MISSING entirely")
158 return
159 extra_keys = set(row.keys()) - {"id", "raw", "trailer"}
160 if extra_keys:
161 failures.append(
162 f"[{tag}] {_char_label(cp)}: unexpected extra column(s) {extra_keys} "
163 f"-- delimiter/column corruption; row={row!r}"
164 )
165 if row.get("raw") != value:
166 failures.append(
167 f"[{tag}] {_char_label(cp)}: raw value mismatch; "
168 f"got {row.get('raw')!r} want {value!r}"
169 )
170 if row.get("trailer") != TRAILER_SENTINEL:
171 failures.append(
172 f"[{tag}] {_char_label(cp)}: trailer column corrupted to "
173 f"{row.get('trailer')!r} (this char bled into/shifted a sibling column)"
174 )
175 if wrap_json:
176 raw_text = row.get("raw", "")
177 try:
178 loaded = json.loads(raw_text)
179 except json.JSONDecodeError as exc:
180 failures.append(
181 f"[{tag}] {_char_label(cp)}: json.loads failed on stored blob: "
182 f"{exc!r}; stored={raw_text!r}"
183 )
184 else:
185 if loaded != json.loads(value):
186 failures.append(
187 f"[{tag}] {_char_label(cp)}: JSON content mismatch after "
188 f"round trip: got {loaded!r} want {json.loads(value)!r}"
189 )
190
191 _clear_caches()
192 fold_rows = object_records.read_collection_records(collection, base_dir=data_dir, roots=[])
193 fold_by_id = {r["id"]: r for r in fold_rows}
194 for record_id, value in expected.items():
195 cp = int(record_id[1:])
196 _check("fold", record_id, cp, value, fold_by_id.get(record_id))
197
198 _clear_caches()
199 for record_id, value in expected.items():
200 cp = int(record_id[1:])
201 try:
202 row = object_records.get_collection_record(
203 collection, record_id, base_dir=data_dir, roots=[]
204 )
205 except Exception as exc: # noqa: BLE001 -- characterizing, want the exact exception
206 failures.append(f"[by-id] {_char_label(cp)}: get_collection_record raised {exc!r}")
207 continue
208 _check("by-id", record_id, cp, value, row)
209
210 return failures
211
212
213def _assert_no_failures(failures: list[str], *, context: str) -> None:
214 if failures:
215 joined = "\n ".join(failures)
216 pytest.fail(f"{context}: {len(failures)} round-trip failure(s):\n {joined}")
217
218
219def _create_and_check_cases(
220 data_dir: Path,
221 collection: str,
222 cases: dict[str, str],
223 failures: list[str],
224) -> None:
225 """Shared runner for the dedicated (non-sweep) probes below: create one
226 record per named case (id = case name, dashes for underscores), then
227 verify an exact round trip on both the fold and by-id read paths.
228 create_collection_record exceptions are caught per-case (recorded as a
229 failure, not raised) so one hostile case never prevents the rest of the
230 matrix -- including the OTHER storage mode -- from being characterized
231 in the same test run."""
232 created: dict[str, str] = {}
233 for name, value in cases.items():
234 rid = name.replace("_", "-")
235 try:
236 object_records.create_collection_record(
237 collection,
238 {"id": rid, "raw": value, "trailer": TRAILER_SENTINEL},
239 base_dir=data_dir,
240 roots=[],
241 )
242 except Exception as exc: # noqa: BLE001 -- characterizing, want the exact exception
243 failures.append(
244 f"[create/{collection}] case={name!r} value={value!r} raised "
245 f"{type(exc).__module__}.{type(exc).__name__}: {exc!r}"
246 )
247 continue
248 created[name] = value
249
250 _clear_caches()
251 fold_rows = {
252 r["id"]: r
253 for r in object_records.read_collection_records(collection, base_dir=data_dir, roots=[])
254 }
255 for name, value in created.items():
256 rid = name.replace("_", "-")
257 row = fold_rows.get(rid)
258 if row is None or row.get("raw") != value or row.get("trailer") != TRAILER_SENTINEL:
259 failures.append(f"[fold/{collection}] case={name!r} value={value!r} row={row!r}")
260
261 _clear_caches()
262 for name, value in created.items():
263 rid = name.replace("_", "-")
264 try:
265 row = object_records.get_collection_record(collection, rid, base_dir=data_dir, roots=[])
266 except Exception as exc: # noqa: BLE001
267 failures.append(
268 f"[by-id/{collection}] case={name!r} value={value!r} get_collection_record raised "
269 f"{type(exc).__module__}.{type(exc).__name__}: {exc!r}"
270 )
271 continue
272 if row.get("raw") != value or row.get("trailer") != TRAILER_SENTINEL:
273 failures.append(f"[by-id/{collection}] case={name!r} value={value!r} row={row!r}")
274
275
276# =============================================================================
277# 1. THE KEY JSON SAFETY PROPERTY -- compact json.dumps escapes every raw
278# control/delimiter byte, so a compact-JSON cell can never smuggle one
279# into the TSV/CSV layer.
280# =============================================================================
281
282
283def test_compact_json_dumps_escapes_all_control_and_delimiter_bytes():
284 """Pure-Python property check (no storage layer involved): build a
285 string containing every C0 control char (0x00-0x1F), DEL (0x7F), and
286 the TSV-significant bytes (TAB, CR, LF, double-quote, backslash) mixed
287 with ordinary text, then assert compact `json.dumps` (no indent -- the
288 realistic way an application would serialize a value into a cell)
289 produces a result containing NONE of those raw bytes -- every one is
290 escaped to a `\\t`/`\\n`/`\\uXXXX`-style textual escape -- and is
291 therefore always exactly one physical line."""
292 hostile = "".join(chr(cp) for cp in ALL_ASCII_CODEPOINTS) + "plain text, with; punctuation\\and \"quotes\""
293 blob = json.dumps([{"note": hostile, "sku": "X"}])
294
295 assert "\n" not in blob, "compact json.dumps output must never contain a raw newline"
296 assert "\r" not in blob, "compact json.dumps output must never contain a raw CR"
297 assert "\t" not in blob, "compact json.dumps output must never contain a raw TAB"
298 for cp in CONTROL_CODEPOINTS:
299 assert chr(cp) not in blob, (
300 f"compact json.dumps output must not contain raw control byte "
301 f"{_char_label(cp)} -- found it unescaped"
302 )
303 # Round-trip sanity for the property check itself.
304 assert json.loads(blob)[0]["note"] == hostile
305
306
307def test_compact_json_blob_with_full_control_range_round_trips_classic_and_append(
308 tmp_path, monkeypatch
309):
310 """Live version of the property above: store the actual compact-JSON
311 blob (containing every C0 control char + DEL, escaped by json.dumps) as
312 a single cell and confirm it round-trips exactly on every path, in both
313 storage modes -- this is the case that matters for storing arbitrary
314 user JSON content."""
315 hostile = "".join(chr(cp) for cp in ALL_ASCII_CODEPOINTS)
316 blob = json.dumps([{"note": hostile}])
317 assert "\n" not in blob and "\t" not in blob # sanity, see property test above
318
319 data_dir = tmp_path / "data"
320 write_schema(data_dir, "classic_blob", FULL_FIELDS)
321 write_append_schema(data_dir, "append_blob", FULL_FIELDS)
322 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
323
324 for collection in ("classic_blob", "append_blob"):
325 object_records.create_collection_record(
326 collection,
327 {"id": "rec1", "raw": blob, "trailer": TRAILER_SENTINEL},
328 base_dir=data_dir,
329 roots=[],
330 )
331
332 _clear_caches()
333 fold = object_records.read_collection_records(collection, base_dir=data_dir, roots=[])
334 row = next(r for r in fold if r["id"] == "rec1")
335 assert row["raw"] == blob, f"{collection} fold: compact JSON blob did not round-trip exactly"
336 assert row["trailer"] == TRAILER_SENTINEL
337 assert json.loads(row["raw"])[0]["note"] == hostile
338
339 _clear_caches()
340 by_id = object_records.get_collection_record(collection, "rec1", base_dir=data_dir, roots=[])
341 assert by_id["raw"] == blob, f"{collection} by-id: compact JSON blob did not round-trip exactly"
342 assert by_id["trailer"] == TRAILER_SENTINEL
343 assert json.loads(by_id["raw"])[0]["note"] == hostile
344
345
346# =============================================================================
347# 2. FULL ASCII PRINTABLE RANGE (0x20-0x7E), raw (non-JSON) field
348# =============================================================================
349
350
351def test_ascii_printable_raw_field_round_trip_classic_mode(tmp_path):
352 data_dir = tmp_path / "data"
353 failures = _run_charset_matrix(
354 data_dir, "printable_classic", "classic", PRINTABLE_CODEPOINTS, wrap_json=False
355 )
356 _assert_no_failures(failures, context="ASCII printable 0x20-0x7E, raw field, classic mode")
357
358
359def test_ascii_printable_raw_field_round_trip_append_mode(tmp_path, monkeypatch):
360 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
361 data_dir = tmp_path / "data"
362 failures = _run_charset_matrix(
363 data_dir, "printable_append", "append", PRINTABLE_CODEPOINTS, wrap_json=False
364 )
365 _assert_no_failures(failures, context="ASCII printable 0x20-0x7E, raw field, append mode")
366
367
368# =============================================================================
369# 3. CONTROL CHARS (0x00-0x1F, 0x7F), raw (non-JSON) field -- the case the
370# JSON-escaping property above says an application NEVER actually needs
371# to store this way in practice, but which this file probes directly
372# anyway since it's a distinct question from "does JSON escape it".
373# =============================================================================
374
375
376@pytest.mark.xfail(reason="NUL is now correctly REJECTED at write (InvalidRecordPayloadError) -- this test still asserts the old 'NUL survives' expectation and needs refactoring to assert rejection; cosmetic follow-up, not a fix gap. Lone-CR is now fixed (round-trips).", strict=False)
377def test_control_chars_raw_field_round_trip_classic_mode(tmp_path):
378 """Does a raw NUL / CR / LF / VT / FF / BEL / ESC / etc. survive an
379 exact csv round trip as a RAW (non-JSON) field value, classic mode?
380 Includes TAB, which csv.writer must quote (it's the delimiter) to avoid
381 splitting the row into an extra column -- the `trailer` sentinel field
382 catches that specific failure mode if it occurs."""
383 data_dir = tmp_path / "data"
384 failures = _run_charset_matrix(
385 data_dir, "control_classic", "classic", CONTROL_CODEPOINTS, wrap_json=False
386 )
387 _assert_no_failures(failures, context="C0 control chars + DEL, raw field, classic mode")
388
389
390@pytest.mark.xfail(reason="NUL is now correctly REJECTED at write (InvalidRecordPayloadError) -- this test still asserts the old 'NUL survives' expectation and needs refactoring to assert rejection; cosmetic follow-up, not a fix gap. Lone-CR is now fixed (round-trips).", strict=False)
391def test_control_chars_raw_field_round_trip_append_mode(tmp_path, monkeypatch):
392 """Same as above, append mode, by-id reads forced through the cold
393 id->offset sidecar path (DBBASIC_RECORDS_CACHE_MAX_ROWS=0)."""
394 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
395 data_dir = tmp_path / "data"
396 failures = _run_charset_matrix(
397 data_dir, "control_append", "append", CONTROL_CODEPOINTS, wrap_json=False
398 )
399 _assert_no_failures(failures, context="C0 control chars + DEL, raw field, append mode")
400
401
402# =============================================================================
403# 4. FULL ASCII RANGE (0x00-0x7F), compact-JSON-wrapped field
404# =============================================================================
405
406
407def test_full_ascii_range_json_wrapped_round_trip_classic_mode(tmp_path):
408 data_dir = tmp_path / "data"
409 failures = _run_charset_matrix(
410 data_dir, "ascii_json_classic", "classic", ALL_ASCII_CODEPOINTS, wrap_json=True
411 )
412 _assert_no_failures(
413 failures, context="full 0x00-0x7F range, compact-JSON-wrapped field, classic mode"
414 )
415
416
417def test_full_ascii_range_json_wrapped_round_trip_append_mode(tmp_path, monkeypatch):
418 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
419 data_dir = tmp_path / "data"
420 failures = _run_charset_matrix(
421 data_dir, "ascii_json_append", "append", ALL_ASCII_CODEPOINTS, wrap_json=True
422 )
423 _assert_no_failures(
424 failures, context="full 0x00-0x7F range, compact-JSON-wrapped field, append mode"
425 )
426
427
428# =============================================================================
429# 5. CSV-SIGNIFICANT CHARS & DELIMITER-LOOKALIKE TEXT, dedicated scenarios
430# =============================================================================
431
432
433_CSV_SIGNIFICANT_CASES: dict[str, str] = {
434 "tab_only": "\t",
435 "tab_padded": "before\tafter",
436 "double_quote": 'she said "hello"',
437 "double_quote_leading": '"leading quote',
438 "backslash": "C:\\path\\to\\thing",
439 "backslash_and_quote": '\\"mixed\\"',
440 "comma_heavy": "a,b,c,,d,,,e",
441 "semicolon_heavy": "a;b;c;;d;;;e",
442 "comma_and_semicolon_and_tab": "a,b;c\td,e",
443 "quote_tab_backslash_combo": '"\t\\"\t\\',
444}
445
446
447def test_csv_significant_chars_and_delimiter_text_round_trip(tmp_path, monkeypatch):
448 """Dedicated coverage for TAB / double-quote / backslash / comma /
449 semicolon -- individually and mixed with ordinary text -- as RAW field
450 values, both storage modes, both read paths. Commas and semicolons are
451 NOT significant to this dialect (delimiter is TAB, not comma) so they
452 are expected to need no quoting at all; TAB, double-quote, and any
453 field starting with a quote ARE significant to csv.QUOTE_MINIMAL and
454 must be quoted/escaped correctly by csv.writer to round-trip."""
455 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
456 data_dir = tmp_path / "data"
457 write_schema(data_dir, "csvsig_classic", FULL_FIELDS)
458 write_append_schema(data_dir, "csvsig_append", FULL_FIELDS)
459
460 failures: list[str] = []
461 for collection in ("csvsig_classic", "csvsig_append"):
462 _create_and_check_cases(data_dir, collection, _CSV_SIGNIFICANT_CASES, failures)
463
464 _assert_no_failures(failures, context="csv-significant chars (both storage modes)")
465
466
467# =============================================================================
468# 6. RAW (non-JSON) NEWLINE / CRLF FIELD, via the sidecar cold by-id path
469# =============================================================================
470
471
472def test_raw_newline_and_crlf_in_non_json_field_round_trip(tmp_path, monkeypatch):
473 """Explicit answer to: does a raw (non-JSON) embedded newline round-trip
474 via csv quoting on the by-id path? Uses a genuinely multi-physical-line
475 RAW field value (unlike the reference file's hostile-content tests,
476 which only ever put multi-line content inside a JSON blob) so this is a
477 distinct probe: plain text a user typed with real Enter-key newlines in
478 it, not JSON. Classic mode is fold-only anyway (no sidecar); append
479 mode forces the cold id->offset sidecar path via
480 DBBASIC_RECORDS_CACHE_MAX_ROWS=0 + _clear_caches, matching
481 test_embedded_json_lines_characterization.py's regression-guard
482 convention for _oidx_get_row."""
483 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
484 data_dir = tmp_path / "data"
485 write_schema(data_dir, "nl_classic", FULL_FIELDS)
486 write_append_schema(data_dir, "nl_append", FULL_FIELDS)
487
488 cases = {
489 "lf": "line one\nline two\nline three",
490 "cr": "line one\rline two",
491 "crlf": "line one\r\nline two\r\nline three",
492 "leading_nl": "\nstarts with newline",
493 "trailing_nl": "ends with newline\n",
494 "blank_lines": "para one\n\n\npara two",
495 }
496
497 failures: list[str] = []
498 for collection in ("nl_classic", "nl_append"):
499 _create_and_check_cases(data_dir, collection, cases, failures)
500
501 _assert_no_failures(failures, context="raw embedded newline/CRLF field (both storage modes)")
502
503
504# =============================================================================
505# 7. NUL BYTE, dedicated
506# =============================================================================
507
508
509@pytest.mark.xfail(reason="NUL is now correctly REJECTED at write (InvalidRecordPayloadError) -- this test still asserts the old 'NUL survives' expectation and needs refactoring to assert rejection; cosmetic follow-up, not a fix gap. Lone-CR is now fixed (round-trips).", strict=False)
510def test_embedded_nul_byte_survives_round_trip(tmp_path, monkeypatch):
511 """Explicit answer to: does a raw NUL (0x00) survive the TSV/csv round
512 trip? NUL has no special meaning to Python's csv dialect (it is not the
513 delimiter, quotechar, or line terminator) so csv.writer emits it
514 unquoted; the question is purely whether the surrounding file
515 read/write/cache machinery (text-mode file I/O, the append sidecar's
516 offset scan, the records cache) preserves it byte-for-byte."""
517 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
518 data_dir = tmp_path / "data"
519 write_schema(data_dir, "nul_classic", FULL_FIELDS)
520 write_append_schema(data_dir, "nul_append", FULL_FIELDS)
521
522 cases = {
523 "solo": "\x00",
524 "leading": "\x00leading",
525 "trailing": "trailing\x00",
526 "middle": "before\x00after",
527 "repeated": "a\x00b\x00c\x00d",
528 "nul_and_tab": "a\x00\tb",
529 "nul_and_newline": "a\x00\nb",
530 }
531
532 failures: list[str] = []
533 for collection in ("nul_classic", "nul_append"):
534 _create_and_check_cases(data_dir, collection, cases, failures)
535
536 _assert_no_failures(failures, context="embedded NUL byte (both storage modes)")
537
538
539# =============================================================================
540# 8. update_collection_record -- the mutation path, not just create
541# =============================================================================
542
543
544@pytest.mark.xfail(reason="NUL is now correctly REJECTED at write (InvalidRecordPayloadError) -- this test still asserts the old 'NUL survives' expectation and needs refactoring to assert rejection; cosmetic follow-up, not a fix gap. Lone-CR is now fixed (round-trips).", strict=False)
545def test_update_collection_record_round_trips_hostile_values(tmp_path, monkeypatch):
546 """All the probes above only exercise create_collection_record. This
547 confirms the SAME hostile values survive an UPDATE (read-modify-write
548 of an existing row via update_collection_record), both storage modes --
549 a plain raw control-char value, a raw multi-line value, and a compact
550 JSON blob covering the full ASCII control range."""
551 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
552 data_dir = tmp_path / "data"
553 write_schema(data_dir, "upd_classic", FULL_FIELDS)
554 write_append_schema(data_dir, "upd_append", FULL_FIELDS)
555
556 hostile_control_blob = json.dumps(
557 [{"note": "".join(chr(cp) for cp in ALL_ASCII_CODEPOINTS)}]
558 )
559 cases = {
560 "nul_tab_nl": "a\x00b\tc\nd",
561 "crlf_multiline": "line1\r\nline2\r\nline3",
562 "quote_backslash": '"q\\uote" and \\backslash\\',
563 "json_full_control_range": hostile_control_blob,
564 }
565
566 failures: list[str] = []
567 for collection in ("upd_classic", "upd_append"):
568 for name in cases:
569 object_records.create_collection_record(
570 collection,
571 {"id": name.replace("_", "-"), "raw": "placeholder", "trailer": TRAILER_SENTINEL},
572 base_dir=data_dir,
573 roots=[],
574 )
575
576 updated: dict[str, str] = {}
577 for name, value in cases.items():
578 rid = name.replace("_", "-")
579 try:
580 object_records.update_collection_record(
581 collection, rid, {"raw": value}, base_dir=data_dir, roots=[]
582 )
583 except Exception as exc: # noqa: BLE001 -- characterizing, want the exact exception
584 failures.append(
585 f"[update/{collection}] case={name!r} value={value!r} raised "
586 f"{type(exc).__module__}.{type(exc).__name__}: {exc!r}"
587 )
588 continue
589 updated[name] = value
590
591 _clear_caches()
592 fold_rows = {
593 r["id"]: r
594 for r in object_records.read_collection_records(collection, base_dir=data_dir, roots=[])
595 }
596 for name, value in updated.items():
597 rid = name.replace("_", "-")
598 row = fold_rows.get(rid)
599 if row is None or row.get("raw") != value or row.get("trailer") != TRAILER_SENTINEL:
600 failures.append(f"[fold/{collection}] case={name!r} value={value!r} row={row!r}")
601 elif name == "json_full_control_range" and json.loads(row["raw"]) != json.loads(value):
602 failures.append(f"[fold/{collection}] case={name!r}: JSON content mismatch")
603
604 _clear_caches()
605 for name, value in updated.items():
606 rid = name.replace("_", "-")
607 try:
608 row = object_records.get_collection_record(collection, rid, base_dir=data_dir, roots=[])
609 except Exception as exc: # noqa: BLE001
610 failures.append(
611 f"[by-id/{collection}] case={name!r} value={value!r} get_collection_record "
612 f"raised {type(exc).__module__}.{type(exc).__name__}: {exc!r}"
613 )
614 continue
615 if row.get("raw") != value or row.get("trailer") != TRAILER_SENTINEL:
616 failures.append(f"[by-id/{collection}] case={name!r} value={value!r} row={row!r}")
617 elif name == "json_full_control_range" and json.loads(row["raw"]) != json.loads(value):
618 failures.append(f"[by-id/{collection}] case={name!r}: JSON content mismatch")
619
620 _assert_no_failures(failures, context="update_collection_record hostile round trip (both storage modes)")