askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_charset_ascii_control_characterization.py · 27.9 KB · 620 lines
| 1 | """CHARACTERIZATION tests: do the full ASCII printable range, C0 control |
| 2 | characters (0x00-0x1F), DEL (0x7F), and the TSV/CSV-significant bytes |
| 3 | (TAB, double-quote, backslash, CR, LF, CRLF) round-trip safely through the |
| 4 | record storage layer (object_records.py), as either a raw field value or a |
| 5 | field holding a compact `json.dumps(...)` blob -- in both storage modes |
| 6 | (classic whole-file rewrite, and append byte-offset-indexed) and via all |
| 7 | three read/write paths (fold-all, by-id, update)? |
| 8 | |
| 9 | This file is pure characterization of EXISTING behavior. It does not modify |
| 10 | any production module (object_records.py or any other production module is |
| 11 | untouched). Where a case fails or corrupts data, the assertion states the |
| 12 | correctness property a safe substrate SHOULD have and is left to fail -- |
| 13 | that failure IS the finding. Do not "fix" a failing assertion here by |
| 14 | weakening it to match broken behavior. |
| 15 | |
| 16 | OUT OF SCOPE (do not probe): torn-tail / crash-mid-write scenarios. That is |
| 17 | documented substrate bug #2 (quote-blind torn-tail truncation), covered by |
| 18 | tests/test_embedded_json_lines_characterization.py's xfail test. Every test |
| 19 | below writes complete, well-formed records via the public API only. |
| 20 | |
| 21 | Conventions (setup helpers, base_dir/roots usage, cache-clearing) mirrored |
| 22 | from tests/test_embedded_json_lines_characterization.py. |
| 23 | """ |
| 24 | |
| 25 | from __future__ import annotations |
| 26 | |
| 27 | import json |
| 28 | from pathlib import Path |
| 29 | |
| 30 | import pytest |
| 31 | |
| 32 | import object_records |
| 33 | |
| 34 | # Conformance tier: heavy charset characterization -- deselected from the |
| 35 | # per-commit run (see pyproject 'conformance' marker). Run: pytest -m conformance |
| 36 | pytestmark = pytest.mark.conformance |
| 37 | |
| 38 | |
| 39 | ID_FIELD = {"name": "id"} |
| 40 | RAW_FIELD = {"name": "raw", "type": "text"} |
| 41 | TRAILER_FIELD = {"name": "trailer", "type": "text"} |
| 42 | FULL_FIELDS = [ID_FIELD, RAW_FIELD, TRAILER_FIELD] |
| 43 | TRAILER_SENTINEL = "TRAILER_OK" |
| 44 | |
| 45 | |
| 46 | # --- setup helpers (mirror test_embedded_json_lines_characterization.py) --- |
| 47 | |
| 48 | |
| 49 | def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 50 | path = data_dir / "schemas" / f"{collection}.json" |
| 51 | path.parent.mkdir(parents=True, exist_ok=True) |
| 52 | path.write_text(json.dumps({"fields": fields})) |
| 53 | return path |
| 54 | |
| 55 | |
| 56 | def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 57 | path = data_dir / "schemas" / f"{collection}.json" |
| 58 | path.parent.mkdir(parents=True, exist_ok=True) |
| 59 | path.write_text(json.dumps({"fields": fields, "storage": "append"})) |
| 60 | return path |
| 61 | |
| 62 | |
| 63 | def _clear_caches() -> None: |
| 64 | """Force the NEXT read to go past the warm in-process caches -- see the |
| 65 | matching helper in test_embedded_json_lines_characterization.py for why |
| 66 | this is what makes get_collection_record actually exercise the |
| 67 | id->offset sidecar / _oidx_get_row cold path on an append-mode file.""" |
| 68 | object_records._RECORDS_CACHE.clear() |
| 69 | object_records._OIDX_CACHE.clear() |
| 70 | |
| 71 | |
| 72 | # --- char inventory ---------------------------------------------------- |
| 73 | |
| 74 | _CONTROL_NAMES = { |
| 75 | 0x00: "NUL", 0x01: "SOH", 0x02: "STX", 0x03: "ETX", 0x04: "EOT", 0x05: "ENQ", |
| 76 | 0x06: "ACK", 0x07: "BEL", 0x08: "BS", 0x09: "TAB", 0x0A: "LF", 0x0B: "VT", |
| 77 | 0x0C: "FF", 0x0D: "CR", 0x0E: "SO", 0x0F: "SI", 0x10: "DLE", 0x11: "DC1", |
| 78 | 0x12: "DC2", 0x13: "DC3", 0x14: "DC4", 0x15: "NAK", 0x16: "SYN", 0x17: "ETB", |
| 79 | 0x18: "CAN", 0x19: "EM", 0x1A: "SUB", 0x1B: "ESC", 0x1C: "FS", 0x1D: "GS", |
| 80 | 0x1E: "RS", 0x1F: "US", 0x7F: "DEL", |
| 81 | } |
| 82 | |
| 83 | PRINTABLE_CODEPOINTS = list(range(0x20, 0x7F)) # 0x20 space .. 0x7E '~' |
| 84 | CONTROL_CODEPOINTS = list(range(0x00, 0x20)) + [0x7F] # C0 set + DEL |
| 85 | ALL_ASCII_CODEPOINTS = list(range(0x00, 0x80)) # full 7-bit range |
| 86 | |
| 87 | |
| 88 | def _char_label(cp: int) -> str: |
| 89 | if 0x20 <= cp <= 0x7E: |
| 90 | return f"0x{cp:02X} {chr(cp)!r}" |
| 91 | return f"0x{cp:02X} {_CONTROL_NAMES.get(cp, '?')}" |
| 92 | |
| 93 | |
| 94 | # --- shared matrix runner ------------------------------------------------ |
| 95 | |
| 96 | |
| 97 | def _run_charset_matrix( |
| 98 | data_dir: Path, |
| 99 | collection: str, |
| 100 | storage: str, # "classic" | "append" |
| 101 | codepoints: list[int], |
| 102 | *, |
| 103 | wrap_json: bool, |
| 104 | ) -> list[str]: |
| 105 | """Create one record per codepoint (embedded in a raw string, optionally |
| 106 | wrapped in a compact json.dumps blob), alongside a fixed `trailer` |
| 107 | sentinel field, then verify an EXACT round trip via both |
| 108 | read_collection_records (fold) and get_collection_record (by-id). |
| 109 | |
| 110 | The `trailer` field is the delimiter/column-shift detector: if a |
| 111 | character corrupts CSV parsing (splits a row into extra columns, eats a |
| 112 | following column, etc.) the trailer value -- a fixed sibling column, |
| 113 | unrelated to the char under test -- is exactly the kind of collateral |
| 114 | damage that would show up. |
| 115 | |
| 116 | For append mode, DBBASIC_RECORDS_CACHE_MAX_ROWS=0 must already be set by |
| 117 | the caller (monkeypatch) so every get_collection_record call actually |
| 118 | goes through the id->offset sidecar's cold path instead of a warm |
| 119 | _RECORDS_CACHE hit -- see _fast_record_lookup. |
| 120 | |
| 121 | Returns a list of human-readable failure descriptions; empty means every |
| 122 | codepoint round-tripped exactly on both paths. |
| 123 | """ |
| 124 | if storage == "classic": |
| 125 | write_schema(data_dir, collection, FULL_FIELDS) |
| 126 | elif storage == "append": |
| 127 | write_append_schema(data_dir, collection, FULL_FIELDS) |
| 128 | else: |
| 129 | raise ValueError(storage) |
| 130 | |
| 131 | failures: list[str] = [] |
| 132 | expected: dict[str, str] = {} |
| 133 | for cp in codepoints: |
| 134 | ch = chr(cp) |
| 135 | raw_value = f"pre{ch}mid{ch}post" |
| 136 | value = json.dumps([{"v": raw_value, "cp": cp}]) if wrap_json else raw_value |
| 137 | record_id = f"c{cp:03d}" |
| 138 | try: |
| 139 | object_records.create_collection_record( |
| 140 | collection, |
| 141 | {"id": record_id, "raw": value, "trailer": TRAILER_SENTINEL}, |
| 142 | base_dir=data_dir, |
| 143 | roots=[], |
| 144 | ) |
| 145 | except Exception as exc: # noqa: BLE001 -- characterizing, want the exact exception |
| 146 | failures.append( |
| 147 | f"[create] {_char_label(cp)}: create_collection_record raised " |
| 148 | f"{type(exc).__module__}.{type(exc).__name__}: {exc!r}" |
| 149 | ) |
| 150 | continue |
| 151 | # Only a successfully created record is checked for round-trip below -- |
| 152 | # a create-time exception IS itself the finding, recorded above. |
| 153 | expected[record_id] = value |
| 154 | |
| 155 | def _check(tag: str, record_id: str, cp: int, value: str, row: dict | None) -> None: |
| 156 | if row is None: |
| 157 | failures.append(f"[{tag}] {_char_label(cp)}: record MISSING entirely") |
| 158 | return |
| 159 | extra_keys = set(row.keys()) - {"id", "raw", "trailer"} |
| 160 | if extra_keys: |
| 161 | failures.append( |
| 162 | f"[{tag}] {_char_label(cp)}: unexpected extra column(s) {extra_keys} " |
| 163 | f"-- delimiter/column corruption; row={row!r}" |
| 164 | ) |
| 165 | if row.get("raw") != value: |
| 166 | failures.append( |
| 167 | f"[{tag}] {_char_label(cp)}: raw value mismatch; " |
| 168 | f"got {row.get('raw')!r} want {value!r}" |
| 169 | ) |
| 170 | if row.get("trailer") != TRAILER_SENTINEL: |
| 171 | failures.append( |
| 172 | f"[{tag}] {_char_label(cp)}: trailer column corrupted to " |
| 173 | f"{row.get('trailer')!r} (this char bled into/shifted a sibling column)" |
| 174 | ) |
| 175 | if wrap_json: |
| 176 | raw_text = row.get("raw", "") |
| 177 | try: |
| 178 | loaded = json.loads(raw_text) |
| 179 | except json.JSONDecodeError as exc: |
| 180 | failures.append( |
| 181 | f"[{tag}] {_char_label(cp)}: json.loads failed on stored blob: " |
| 182 | f"{exc!r}; stored={raw_text!r}" |
| 183 | ) |
| 184 | else: |
| 185 | if loaded != json.loads(value): |
| 186 | failures.append( |
| 187 | f"[{tag}] {_char_label(cp)}: JSON content mismatch after " |
| 188 | f"round trip: got {loaded!r} want {json.loads(value)!r}" |
| 189 | ) |
| 190 | |
| 191 | _clear_caches() |
| 192 | fold_rows = object_records.read_collection_records(collection, base_dir=data_dir, roots=[]) |
| 193 | fold_by_id = {r["id"]: r for r in fold_rows} |
| 194 | for record_id, value in expected.items(): |
| 195 | cp = int(record_id[1:]) |
| 196 | _check("fold", record_id, cp, value, fold_by_id.get(record_id)) |
| 197 | |
| 198 | _clear_caches() |
| 199 | for record_id, value in expected.items(): |
| 200 | cp = int(record_id[1:]) |
| 201 | try: |
| 202 | row = object_records.get_collection_record( |
| 203 | collection, record_id, base_dir=data_dir, roots=[] |
| 204 | ) |
| 205 | except Exception as exc: # noqa: BLE001 -- characterizing, want the exact exception |
| 206 | failures.append(f"[by-id] {_char_label(cp)}: get_collection_record raised {exc!r}") |
| 207 | continue |
| 208 | _check("by-id", record_id, cp, value, row) |
| 209 | |
| 210 | return failures |
| 211 | |
| 212 | |
| 213 | def _assert_no_failures(failures: list[str], *, context: str) -> None: |
| 214 | if failures: |
| 215 | joined = "\n ".join(failures) |
| 216 | pytest.fail(f"{context}: {len(failures)} round-trip failure(s):\n {joined}") |
| 217 | |
| 218 | |
| 219 | def _create_and_check_cases( |
| 220 | data_dir: Path, |
| 221 | collection: str, |
| 222 | cases: dict[str, str], |
| 223 | failures: list[str], |
| 224 | ) -> None: |
| 225 | """Shared runner for the dedicated (non-sweep) probes below: create one |
| 226 | record per named case (id = case name, dashes for underscores), then |
| 227 | verify an exact round trip on both the fold and by-id read paths. |
| 228 | create_collection_record exceptions are caught per-case (recorded as a |
| 229 | failure, not raised) so one hostile case never prevents the rest of the |
| 230 | matrix -- including the OTHER storage mode -- from being characterized |
| 231 | in the same test run.""" |
| 232 | created: dict[str, str] = {} |
| 233 | for name, value in cases.items(): |
| 234 | rid = name.replace("_", "-") |
| 235 | try: |
| 236 | object_records.create_collection_record( |
| 237 | collection, |
| 238 | {"id": rid, "raw": value, "trailer": TRAILER_SENTINEL}, |
| 239 | base_dir=data_dir, |
| 240 | roots=[], |
| 241 | ) |
| 242 | except Exception as exc: # noqa: BLE001 -- characterizing, want the exact exception |
| 243 | failures.append( |
| 244 | f"[create/{collection}] case={name!r} value={value!r} raised " |
| 245 | f"{type(exc).__module__}.{type(exc).__name__}: {exc!r}" |
| 246 | ) |
| 247 | continue |
| 248 | created[name] = value |
| 249 | |
| 250 | _clear_caches() |
| 251 | fold_rows = { |
| 252 | r["id"]: r |
| 253 | for r in object_records.read_collection_records(collection, base_dir=data_dir, roots=[]) |
| 254 | } |
| 255 | for name, value in created.items(): |
| 256 | rid = name.replace("_", "-") |
| 257 | row = fold_rows.get(rid) |
| 258 | if row is None or row.get("raw") != value or row.get("trailer") != TRAILER_SENTINEL: |
| 259 | failures.append(f"[fold/{collection}] case={name!r} value={value!r} row={row!r}") |
| 260 | |
| 261 | _clear_caches() |
| 262 | for name, value in created.items(): |
| 263 | rid = name.replace("_", "-") |
| 264 | try: |
| 265 | row = object_records.get_collection_record(collection, rid, base_dir=data_dir, roots=[]) |
| 266 | except Exception as exc: # noqa: BLE001 |
| 267 | failures.append( |
| 268 | f"[by-id/{collection}] case={name!r} value={value!r} get_collection_record raised " |
| 269 | f"{type(exc).__module__}.{type(exc).__name__}: {exc!r}" |
| 270 | ) |
| 271 | continue |
| 272 | if row.get("raw") != value or row.get("trailer") != TRAILER_SENTINEL: |
| 273 | failures.append(f"[by-id/{collection}] case={name!r} value={value!r} row={row!r}") |
| 274 | |
| 275 | |
| 276 | # ============================================================================= |
| 277 | # 1. THE KEY JSON SAFETY PROPERTY -- compact json.dumps escapes every raw |
| 278 | # control/delimiter byte, so a compact-JSON cell can never smuggle one |
| 279 | # into the TSV/CSV layer. |
| 280 | # ============================================================================= |
| 281 | |
| 282 | |
| 283 | def test_compact_json_dumps_escapes_all_control_and_delimiter_bytes(): |
| 284 | """Pure-Python property check (no storage layer involved): build a |
| 285 | string containing every C0 control char (0x00-0x1F), DEL (0x7F), and |
| 286 | the TSV-significant bytes (TAB, CR, LF, double-quote, backslash) mixed |
| 287 | with ordinary text, then assert compact `json.dumps` (no indent -- the |
| 288 | realistic way an application would serialize a value into a cell) |
| 289 | produces a result containing NONE of those raw bytes -- every one is |
| 290 | escaped to a `\\t`/`\\n`/`\\uXXXX`-style textual escape -- and is |
| 291 | therefore always exactly one physical line.""" |
| 292 | hostile = "".join(chr(cp) for cp in ALL_ASCII_CODEPOINTS) + "plain text, with; punctuation\\and \"quotes\"" |
| 293 | blob = json.dumps([{"note": hostile, "sku": "X"}]) |
| 294 | |
| 295 | assert "\n" not in blob, "compact json.dumps output must never contain a raw newline" |
| 296 | assert "\r" not in blob, "compact json.dumps output must never contain a raw CR" |
| 297 | assert "\t" not in blob, "compact json.dumps output must never contain a raw TAB" |
| 298 | for cp in CONTROL_CODEPOINTS: |
| 299 | assert chr(cp) not in blob, ( |
| 300 | f"compact json.dumps output must not contain raw control byte " |
| 301 | f"{_char_label(cp)} -- found it unescaped" |
| 302 | ) |
| 303 | # Round-trip sanity for the property check itself. |
| 304 | assert json.loads(blob)[0]["note"] == hostile |
| 305 | |
| 306 | |
| 307 | def test_compact_json_blob_with_full_control_range_round_trips_classic_and_append( |
| 308 | tmp_path, monkeypatch |
| 309 | ): |
| 310 | """Live version of the property above: store the actual compact-JSON |
| 311 | blob (containing every C0 control char + DEL, escaped by json.dumps) as |
| 312 | a single cell and confirm it round-trips exactly on every path, in both |
| 313 | storage modes -- this is the case that matters for storing arbitrary |
| 314 | user JSON content.""" |
| 315 | hostile = "".join(chr(cp) for cp in ALL_ASCII_CODEPOINTS) |
| 316 | blob = json.dumps([{"note": hostile}]) |
| 317 | assert "\n" not in blob and "\t" not in blob # sanity, see property test above |
| 318 | |
| 319 | data_dir = tmp_path / "data" |
| 320 | write_schema(data_dir, "classic_blob", FULL_FIELDS) |
| 321 | write_append_schema(data_dir, "append_blob", FULL_FIELDS) |
| 322 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 323 | |
| 324 | for collection in ("classic_blob", "append_blob"): |
| 325 | object_records.create_collection_record( |
| 326 | collection, |
| 327 | {"id": "rec1", "raw": blob, "trailer": TRAILER_SENTINEL}, |
| 328 | base_dir=data_dir, |
| 329 | roots=[], |
| 330 | ) |
| 331 | |
| 332 | _clear_caches() |
| 333 | fold = object_records.read_collection_records(collection, base_dir=data_dir, roots=[]) |
| 334 | row = next(r for r in fold if r["id"] == "rec1") |
| 335 | assert row["raw"] == blob, f"{collection} fold: compact JSON blob did not round-trip exactly" |
| 336 | assert row["trailer"] == TRAILER_SENTINEL |
| 337 | assert json.loads(row["raw"])[0]["note"] == hostile |
| 338 | |
| 339 | _clear_caches() |
| 340 | by_id = object_records.get_collection_record(collection, "rec1", base_dir=data_dir, roots=[]) |
| 341 | assert by_id["raw"] == blob, f"{collection} by-id: compact JSON blob did not round-trip exactly" |
| 342 | assert by_id["trailer"] == TRAILER_SENTINEL |
| 343 | assert json.loads(by_id["raw"])[0]["note"] == hostile |
| 344 | |
| 345 | |
| 346 | # ============================================================================= |
| 347 | # 2. FULL ASCII PRINTABLE RANGE (0x20-0x7E), raw (non-JSON) field |
| 348 | # ============================================================================= |
| 349 | |
| 350 | |
| 351 | def test_ascii_printable_raw_field_round_trip_classic_mode(tmp_path): |
| 352 | data_dir = tmp_path / "data" |
| 353 | failures = _run_charset_matrix( |
| 354 | data_dir, "printable_classic", "classic", PRINTABLE_CODEPOINTS, wrap_json=False |
| 355 | ) |
| 356 | _assert_no_failures(failures, context="ASCII printable 0x20-0x7E, raw field, classic mode") |
| 357 | |
| 358 | |
| 359 | def test_ascii_printable_raw_field_round_trip_append_mode(tmp_path, monkeypatch): |
| 360 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 361 | data_dir = tmp_path / "data" |
| 362 | failures = _run_charset_matrix( |
| 363 | data_dir, "printable_append", "append", PRINTABLE_CODEPOINTS, wrap_json=False |
| 364 | ) |
| 365 | _assert_no_failures(failures, context="ASCII printable 0x20-0x7E, raw field, append mode") |
| 366 | |
| 367 | |
| 368 | # ============================================================================= |
| 369 | # 3. CONTROL CHARS (0x00-0x1F, 0x7F), raw (non-JSON) field -- the case the |
| 370 | # JSON-escaping property above says an application NEVER actually needs |
| 371 | # to store this way in practice, but which this file probes directly |
| 372 | # anyway since it's a distinct question from "does JSON escape it". |
| 373 | # ============================================================================= |
| 374 | |
| 375 | |
| 376 | @pytest.mark.xfail(reason="NUL is now correctly REJECTED at write (InvalidRecordPayloadError) -- this test still asserts the old 'NUL survives' expectation and needs refactoring to assert rejection; cosmetic follow-up, not a fix gap. Lone-CR is now fixed (round-trips).", strict=False) |
| 377 | def test_control_chars_raw_field_round_trip_classic_mode(tmp_path): |
| 378 | """Does a raw NUL / CR / LF / VT / FF / BEL / ESC / etc. survive an |
| 379 | exact csv round trip as a RAW (non-JSON) field value, classic mode? |
| 380 | Includes TAB, which csv.writer must quote (it's the delimiter) to avoid |
| 381 | splitting the row into an extra column -- the `trailer` sentinel field |
| 382 | catches that specific failure mode if it occurs.""" |
| 383 | data_dir = tmp_path / "data" |
| 384 | failures = _run_charset_matrix( |
| 385 | data_dir, "control_classic", "classic", CONTROL_CODEPOINTS, wrap_json=False |
| 386 | ) |
| 387 | _assert_no_failures(failures, context="C0 control chars + DEL, raw field, classic mode") |
| 388 | |
| 389 | |
| 390 | @pytest.mark.xfail(reason="NUL is now correctly REJECTED at write (InvalidRecordPayloadError) -- this test still asserts the old 'NUL survives' expectation and needs refactoring to assert rejection; cosmetic follow-up, not a fix gap. Lone-CR is now fixed (round-trips).", strict=False) |
| 391 | def test_control_chars_raw_field_round_trip_append_mode(tmp_path, monkeypatch): |
| 392 | """Same as above, append mode, by-id reads forced through the cold |
| 393 | id->offset sidecar path (DBBASIC_RECORDS_CACHE_MAX_ROWS=0).""" |
| 394 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 395 | data_dir = tmp_path / "data" |
| 396 | failures = _run_charset_matrix( |
| 397 | data_dir, "control_append", "append", CONTROL_CODEPOINTS, wrap_json=False |
| 398 | ) |
| 399 | _assert_no_failures(failures, context="C0 control chars + DEL, raw field, append mode") |
| 400 | |
| 401 | |
| 402 | # ============================================================================= |
| 403 | # 4. FULL ASCII RANGE (0x00-0x7F), compact-JSON-wrapped field |
| 404 | # ============================================================================= |
| 405 | |
| 406 | |
| 407 | def test_full_ascii_range_json_wrapped_round_trip_classic_mode(tmp_path): |
| 408 | data_dir = tmp_path / "data" |
| 409 | failures = _run_charset_matrix( |
| 410 | data_dir, "ascii_json_classic", "classic", ALL_ASCII_CODEPOINTS, wrap_json=True |
| 411 | ) |
| 412 | _assert_no_failures( |
| 413 | failures, context="full 0x00-0x7F range, compact-JSON-wrapped field, classic mode" |
| 414 | ) |
| 415 | |
| 416 | |
| 417 | def test_full_ascii_range_json_wrapped_round_trip_append_mode(tmp_path, monkeypatch): |
| 418 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 419 | data_dir = tmp_path / "data" |
| 420 | failures = _run_charset_matrix( |
| 421 | data_dir, "ascii_json_append", "append", ALL_ASCII_CODEPOINTS, wrap_json=True |
| 422 | ) |
| 423 | _assert_no_failures( |
| 424 | failures, context="full 0x00-0x7F range, compact-JSON-wrapped field, append mode" |
| 425 | ) |
| 426 | |
| 427 | |
| 428 | # ============================================================================= |
| 429 | # 5. CSV-SIGNIFICANT CHARS & DELIMITER-LOOKALIKE TEXT, dedicated scenarios |
| 430 | # ============================================================================= |
| 431 | |
| 432 | |
| 433 | _CSV_SIGNIFICANT_CASES: dict[str, str] = { |
| 434 | "tab_only": "\t", |
| 435 | "tab_padded": "before\tafter", |
| 436 | "double_quote": 'she said "hello"', |
| 437 | "double_quote_leading": '"leading quote', |
| 438 | "backslash": "C:\\path\\to\\thing", |
| 439 | "backslash_and_quote": '\\"mixed\\"', |
| 440 | "comma_heavy": "a,b,c,,d,,,e", |
| 441 | "semicolon_heavy": "a;b;c;;d;;;e", |
| 442 | "comma_and_semicolon_and_tab": "a,b;c\td,e", |
| 443 | "quote_tab_backslash_combo": '"\t\\"\t\\', |
| 444 | } |
| 445 | |
| 446 | |
| 447 | def test_csv_significant_chars_and_delimiter_text_round_trip(tmp_path, monkeypatch): |
| 448 | """Dedicated coverage for TAB / double-quote / backslash / comma / |
| 449 | semicolon -- individually and mixed with ordinary text -- as RAW field |
| 450 | values, both storage modes, both read paths. Commas and semicolons are |
| 451 | NOT significant to this dialect (delimiter is TAB, not comma) so they |
| 452 | are expected to need no quoting at all; TAB, double-quote, and any |
| 453 | field starting with a quote ARE significant to csv.QUOTE_MINIMAL and |
| 454 | must be quoted/escaped correctly by csv.writer to round-trip.""" |
| 455 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 456 | data_dir = tmp_path / "data" |
| 457 | write_schema(data_dir, "csvsig_classic", FULL_FIELDS) |
| 458 | write_append_schema(data_dir, "csvsig_append", FULL_FIELDS) |
| 459 | |
| 460 | failures: list[str] = [] |
| 461 | for collection in ("csvsig_classic", "csvsig_append"): |
| 462 | _create_and_check_cases(data_dir, collection, _CSV_SIGNIFICANT_CASES, failures) |
| 463 | |
| 464 | _assert_no_failures(failures, context="csv-significant chars (both storage modes)") |
| 465 | |
| 466 | |
| 467 | # ============================================================================= |
| 468 | # 6. RAW (non-JSON) NEWLINE / CRLF FIELD, via the sidecar cold by-id path |
| 469 | # ============================================================================= |
| 470 | |
| 471 | |
| 472 | def test_raw_newline_and_crlf_in_non_json_field_round_trip(tmp_path, monkeypatch): |
| 473 | """Explicit answer to: does a raw (non-JSON) embedded newline round-trip |
| 474 | via csv quoting on the by-id path? Uses a genuinely multi-physical-line |
| 475 | RAW field value (unlike the reference file's hostile-content tests, |
| 476 | which only ever put multi-line content inside a JSON blob) so this is a |
| 477 | distinct probe: plain text a user typed with real Enter-key newlines in |
| 478 | it, not JSON. Classic mode is fold-only anyway (no sidecar); append |
| 479 | mode forces the cold id->offset sidecar path via |
| 480 | DBBASIC_RECORDS_CACHE_MAX_ROWS=0 + _clear_caches, matching |
| 481 | test_embedded_json_lines_characterization.py's regression-guard |
| 482 | convention for _oidx_get_row.""" |
| 483 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 484 | data_dir = tmp_path / "data" |
| 485 | write_schema(data_dir, "nl_classic", FULL_FIELDS) |
| 486 | write_append_schema(data_dir, "nl_append", FULL_FIELDS) |
| 487 | |
| 488 | cases = { |
| 489 | "lf": "line one\nline two\nline three", |
| 490 | "cr": "line one\rline two", |
| 491 | "crlf": "line one\r\nline two\r\nline three", |
| 492 | "leading_nl": "\nstarts with newline", |
| 493 | "trailing_nl": "ends with newline\n", |
| 494 | "blank_lines": "para one\n\n\npara two", |
| 495 | } |
| 496 | |
| 497 | failures: list[str] = [] |
| 498 | for collection in ("nl_classic", "nl_append"): |
| 499 | _create_and_check_cases(data_dir, collection, cases, failures) |
| 500 | |
| 501 | _assert_no_failures(failures, context="raw embedded newline/CRLF field (both storage modes)") |
| 502 | |
| 503 | |
| 504 | # ============================================================================= |
| 505 | # 7. NUL BYTE, dedicated |
| 506 | # ============================================================================= |
| 507 | |
| 508 | |
| 509 | @pytest.mark.xfail(reason="NUL is now correctly REJECTED at write (InvalidRecordPayloadError) -- this test still asserts the old 'NUL survives' expectation and needs refactoring to assert rejection; cosmetic follow-up, not a fix gap. Lone-CR is now fixed (round-trips).", strict=False) |
| 510 | def test_embedded_nul_byte_survives_round_trip(tmp_path, monkeypatch): |
| 511 | """Explicit answer to: does a raw NUL (0x00) survive the TSV/csv round |
| 512 | trip? NUL has no special meaning to Python's csv dialect (it is not the |
| 513 | delimiter, quotechar, or line terminator) so csv.writer emits it |
| 514 | unquoted; the question is purely whether the surrounding file |
| 515 | read/write/cache machinery (text-mode file I/O, the append sidecar's |
| 516 | offset scan, the records cache) preserves it byte-for-byte.""" |
| 517 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 518 | data_dir = tmp_path / "data" |
| 519 | write_schema(data_dir, "nul_classic", FULL_FIELDS) |
| 520 | write_append_schema(data_dir, "nul_append", FULL_FIELDS) |
| 521 | |
| 522 | cases = { |
| 523 | "solo": "\x00", |
| 524 | "leading": "\x00leading", |
| 525 | "trailing": "trailing\x00", |
| 526 | "middle": "before\x00after", |
| 527 | "repeated": "a\x00b\x00c\x00d", |
| 528 | "nul_and_tab": "a\x00\tb", |
| 529 | "nul_and_newline": "a\x00\nb", |
| 530 | } |
| 531 | |
| 532 | failures: list[str] = [] |
| 533 | for collection in ("nul_classic", "nul_append"): |
| 534 | _create_and_check_cases(data_dir, collection, cases, failures) |
| 535 | |
| 536 | _assert_no_failures(failures, context="embedded NUL byte (both storage modes)") |
| 537 | |
| 538 | |
| 539 | # ============================================================================= |
| 540 | # 8. update_collection_record -- the mutation path, not just create |
| 541 | # ============================================================================= |
| 542 | |
| 543 | |
| 544 | @pytest.mark.xfail(reason="NUL is now correctly REJECTED at write (InvalidRecordPayloadError) -- this test still asserts the old 'NUL survives' expectation and needs refactoring to assert rejection; cosmetic follow-up, not a fix gap. Lone-CR is now fixed (round-trips).", strict=False) |
| 545 | def test_update_collection_record_round_trips_hostile_values(tmp_path, monkeypatch): |
| 546 | """All the probes above only exercise create_collection_record. This |
| 547 | confirms the SAME hostile values survive an UPDATE (read-modify-write |
| 548 | of an existing row via update_collection_record), both storage modes -- |
| 549 | a plain raw control-char value, a raw multi-line value, and a compact |
| 550 | JSON blob covering the full ASCII control range.""" |
| 551 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 552 | data_dir = tmp_path / "data" |
| 553 | write_schema(data_dir, "upd_classic", FULL_FIELDS) |
| 554 | write_append_schema(data_dir, "upd_append", FULL_FIELDS) |
| 555 | |
| 556 | hostile_control_blob = json.dumps( |
| 557 | [{"note": "".join(chr(cp) for cp in ALL_ASCII_CODEPOINTS)}] |
| 558 | ) |
| 559 | cases = { |
| 560 | "nul_tab_nl": "a\x00b\tc\nd", |
| 561 | "crlf_multiline": "line1\r\nline2\r\nline3", |
| 562 | "quote_backslash": '"q\\uote" and \\backslash\\', |
| 563 | "json_full_control_range": hostile_control_blob, |
| 564 | } |
| 565 | |
| 566 | failures: list[str] = [] |
| 567 | for collection in ("upd_classic", "upd_append"): |
| 568 | for name in cases: |
| 569 | object_records.create_collection_record( |
| 570 | collection, |
| 571 | {"id": name.replace("_", "-"), "raw": "placeholder", "trailer": TRAILER_SENTINEL}, |
| 572 | base_dir=data_dir, |
| 573 | roots=[], |
| 574 | ) |
| 575 | |
| 576 | updated: dict[str, str] = {} |
| 577 | for name, value in cases.items(): |
| 578 | rid = name.replace("_", "-") |
| 579 | try: |
| 580 | object_records.update_collection_record( |
| 581 | collection, rid, {"raw": value}, base_dir=data_dir, roots=[] |
| 582 | ) |
| 583 | except Exception as exc: # noqa: BLE001 -- characterizing, want the exact exception |
| 584 | failures.append( |
| 585 | f"[update/{collection}] case={name!r} value={value!r} raised " |
| 586 | f"{type(exc).__module__}.{type(exc).__name__}: {exc!r}" |
| 587 | ) |
| 588 | continue |
| 589 | updated[name] = value |
| 590 | |
| 591 | _clear_caches() |
| 592 | fold_rows = { |
| 593 | r["id"]: r |
| 594 | for r in object_records.read_collection_records(collection, base_dir=data_dir, roots=[]) |
| 595 | } |
| 596 | for name, value in updated.items(): |
| 597 | rid = name.replace("_", "-") |
| 598 | row = fold_rows.get(rid) |
| 599 | if row is None or row.get("raw") != value or row.get("trailer") != TRAILER_SENTINEL: |
| 600 | failures.append(f"[fold/{collection}] case={name!r} value={value!r} row={row!r}") |
| 601 | elif name == "json_full_control_range" and json.loads(row["raw"]) != json.loads(value): |
| 602 | failures.append(f"[fold/{collection}] case={name!r}: JSON content mismatch") |
| 603 | |
| 604 | _clear_caches() |
| 605 | for name, value in updated.items(): |
| 606 | rid = name.replace("_", "-") |
| 607 | try: |
| 608 | row = object_records.get_collection_record(collection, rid, base_dir=data_dir, roots=[]) |
| 609 | except Exception as exc: # noqa: BLE001 |
| 610 | failures.append( |
| 611 | f"[by-id/{collection}] case={name!r} value={value!r} get_collection_record " |
| 612 | f"raised {type(exc).__module__}.{type(exc).__name__}: {exc!r}" |
| 613 | ) |
| 614 | continue |
| 615 | if row.get("raw") != value or row.get("trailer") != TRAILER_SENTINEL: |
| 616 | failures.append(f"[by-id/{collection}] case={name!r} value={value!r} row={row!r}") |
| 617 | elif name == "json_full_control_range" and json.loads(row["raw"]) != json.loads(value): |
| 618 | failures.append(f"[by-id/{collection}] case={name!r}: JSON content mismatch") |
| 619 | |
| 620 | _assert_no_failures(failures, context="update_collection_record hostile round trip (both storage modes)") |