askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_charset_unicode_adversarial_characterization.py · 34.7 KB · 718 lines
| 1 | """CHARACTERIZATION tests: does the TSV substrate (object_records.py) safely |
| 2 | hold ADVERSARIAL / edge-case Unicode -- the inputs most likely to break a |
| 3 | naive text-storage layer -- through create/update/read (fold-all and by-id), |
| 4 | in both classic and append storage modes? |
| 5 | |
| 6 | This file is pure characterization of EXISTING behavior. It does not modify |
| 7 | any production module. Where a case legitimately SHOULD raise (content that |
| 8 | cannot be represented -- a lone UTF-16 surrogate has no UTF-8 encoding), the |
| 9 | assertion checks that the failure is CLEAN (a well-defined exception, at |
| 10 | write time, before anything touches disk) rather than corruption or a silent |
| 11 | drop. Where a case round-trips, the assertion is byte-exact equality -- not |
| 12 | "looks about right" -- because silent normalization/stripping/mangling of |
| 13 | zero-width, bidi, or combining-mark content is exactly the failure mode this |
| 14 | file exists to catch. A clean raise or an exact round-trip is a PASS; a |
| 15 | mismatch, silent alteration, or unhandled corruption is a FINDING and is |
| 16 | documented as such in the failing test's message, not weakened to pass. |
| 17 | |
| 18 | Context (see module docstring / comments in object_records.py): |
| 19 | - Values are stored as TSV cells via Python's csv module; writes encode |
| 20 | text to UTF-8 (or whatever the file's default open() encoding resolves |
| 21 | to -- see the delimiter-safety section below), reads decode UTF-8. |
| 22 | - csv.field_size_limit has already been raised (MAX_TSV_FIELD_BYTES, 16 |
| 23 | MiB) and create/update enforce the same ceiling on write |
| 24 | (_check_field_sizes) -- not retested here. |
| 25 | - _oidx_get_row (the append-mode id->offset sidecar's single-record read) |
| 26 | is already CSV-aware, not a raw readline() -- not retested here. |
| 27 | - The append-mode torn-tail self-heal check (_drop_torn_tail: "does the |
| 28 | file end with a literal \\n byte") is unsound for a row whose value |
| 29 | spans multiple physical lines -- known, deferred bug #2. NOT probed |
| 30 | here; every value below is either single-physical-line or (for the |
| 31 | zalgo/noncharacter cases) still written via compact json.dumps, never |
| 32 | pretty-printed, so no case in this file exercises that bug by accident. |
| 33 | |
| 34 | Setup helpers below mirror tests/test_embedded_json_lines_characterization.py. |
| 35 | """ |
| 36 | |
| 37 | from __future__ import annotations |
| 38 | |
| 39 | import json |
| 40 | import unicodedata |
| 41 | from pathlib import Path |
| 42 | |
| 43 | import pytest |
| 44 | |
| 45 | import object_records |
| 46 | |
| 47 | # Conformance tier: heavy charset characterization -- deselected from the |
| 48 | # per-commit run (see pyproject 'conformance' marker). Run: pytest -m conformance |
| 49 | pytestmark = pytest.mark.conformance |
| 50 | |
| 51 | |
| 52 | ID_FIELD = {"name": "id"} |
| 53 | TEXT_FIELD = {"name": "note", "type": "textarea"} |
| 54 | |
| 55 | |
| 56 | def _required_text_field() -> dict: |
| 57 | return {"name": "note", "type": "textarea", "required": True} |
| 58 | |
| 59 | |
| 60 | # --- setup helpers (mirror tests/test_embedded_json_lines_characterization.py) --- |
| 61 | |
| 62 | |
| 63 | def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 64 | path = data_dir / "schemas" / f"{collection}.json" |
| 65 | path.parent.mkdir(parents=True, exist_ok=True) |
| 66 | path.write_text(json.dumps({"fields": fields})) |
| 67 | return path |
| 68 | |
| 69 | |
| 70 | def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 71 | path = data_dir / "schemas" / f"{collection}.json" |
| 72 | path.parent.mkdir(parents=True, exist_ok=True) |
| 73 | path.write_text(json.dumps({"fields": fields, "storage": "append"})) |
| 74 | return path |
| 75 | |
| 76 | |
| 77 | def _clear_caches() -> None: |
| 78 | """Force the NEXT read past the warm in-process caches -- see the |
| 79 | matching helper in test_embedded_json_lines_characterization.py. Needed |
| 80 | so a by-id read on an append-mode collection actually exercises the |
| 81 | id->offset sidecar path instead of serving from the ordinary records |
| 82 | cache.""" |
| 83 | object_records._RECORDS_CACHE.clear() |
| 84 | object_records._OIDX_CACHE.clear() |
| 85 | |
| 86 | |
| 87 | # --- adversarial content constants ----------------------------------------- |
| 88 | |
| 89 | ZWSP = "" # zero-width space |
| 90 | ZWNJ = "" # zero-width non-joiner |
| 91 | ZWJ = "" # zero-width joiner |
| 92 | WORD_JOINER = "" |
| 93 | |
| 94 | RLO = "" # right-to-left override |
| 95 | LRO = "" # left-to-right override |
| 96 | RLI = "" # right-to-left isolate |
| 97 | LRI = "" # left-to-right isolate |
| 98 | PDI = "" # pop directional isolate |
| 99 | |
| 100 | NONCHARACTERS = ["", "", "\U0001fffe", "\U0001ffff", "", "", ""] |
| 101 | |
| 102 | CYRILLIC_A = "а" # Cyrillic "а" -- visually identical to Latin "a" (U+0061) |
| 103 | LATIN_A = "a" |
| 104 | |
| 105 | |
| 106 | def _round_trip_both_modes( |
| 107 | tmp_path, monkeypatch, collection_prefix: str, field_value: str, *, field_name: str = "note" |
| 108 | ) -> None: |
| 109 | """Store `field_value` raw under `field_name` in both a classic-mode and |
| 110 | an append-mode collection; assert byte-exact equality on BOTH the |
| 111 | full-fold read and the by-id read (the latter forced through the cold |
| 112 | id->offset sidecar in append mode via DBBASIC_RECORDS_CACHE_MAX_ROWS=0).""" |
| 113 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 114 | data_dir = tmp_path / "data" |
| 115 | classic_coll = f"{collection_prefix}_classic" |
| 116 | append_coll = f"{collection_prefix}_append" |
| 117 | write_schema(data_dir, classic_coll, [ID_FIELD, TEXT_FIELD]) |
| 118 | write_append_schema(data_dir, append_coll, [ID_FIELD, TEXT_FIELD]) |
| 119 | |
| 120 | for coll in (classic_coll, append_coll): |
| 121 | object_records.create_collection_record( |
| 122 | coll, {"id": "r1", field_name: field_value}, base_dir=data_dir, roots=[] |
| 123 | ) |
| 124 | |
| 125 | _clear_caches() |
| 126 | all_records = object_records.read_collection_records(coll, base_dir=data_dir, roots=[]) |
| 127 | assert len(all_records) == 1 |
| 128 | assert all_records[0][field_name] == field_value, ( |
| 129 | f"{coll}: FINDING -- fold-all read did not byte-exact round-trip " |
| 130 | f"the stored value. Expected {field_value!r}, got " |
| 131 | f"{all_records[0][field_name]!r}" |
| 132 | ) |
| 133 | |
| 134 | _clear_caches() |
| 135 | by_id = object_records.get_collection_record(coll, "r1", base_dir=data_dir, roots=[]) |
| 136 | assert by_id[field_name] == field_value, ( |
| 137 | f"{coll}: FINDING -- by-id (sidecar) read did not byte-exact " |
| 138 | f"round-trip the stored value. Expected {field_value!r}, got " |
| 139 | f"{by_id[field_name]!r}" |
| 140 | ) |
| 141 | |
| 142 | |
| 143 | def _round_trip_json_wrapped_both_modes( |
| 144 | tmp_path, monkeypatch, collection_prefix: str, field_value: str, *, field_name: str = "note" |
| 145 | ) -> None: |
| 146 | """Same as _round_trip_both_modes, but the stored cell is a compact |
| 147 | json.dumps([field_value]) blob (a realistic shape for how this content |
| 148 | would actually arrive -- e.g. inside a JSON API payload or an embedded |
| 149 | array), read back via json.loads and compared to the original value.""" |
| 150 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 151 | data_dir = tmp_path / "data" |
| 152 | classic_coll = f"{collection_prefix}_classic" |
| 153 | append_coll = f"{collection_prefix}_append" |
| 154 | write_schema(data_dir, classic_coll, [ID_FIELD, TEXT_FIELD]) |
| 155 | write_append_schema(data_dir, append_coll, [ID_FIELD, TEXT_FIELD]) |
| 156 | blob = json.dumps([field_value]) |
| 157 | |
| 158 | for coll in (classic_coll, append_coll): |
| 159 | object_records.create_collection_record( |
| 160 | coll, {"id": "r1", field_name: blob}, base_dir=data_dir, roots=[] |
| 161 | ) |
| 162 | |
| 163 | _clear_caches() |
| 164 | all_records = object_records.read_collection_records(coll, base_dir=data_dir, roots=[]) |
| 165 | assert json.loads(all_records[0][field_name])[0] == field_value, ( |
| 166 | f"{coll}: FINDING -- fold-all read of a json.dumps-wrapped value " |
| 167 | f"did not round-trip. Expected {field_value!r}." |
| 168 | ) |
| 169 | |
| 170 | _clear_caches() |
| 171 | by_id = object_records.get_collection_record(coll, "r1", base_dir=data_dir, roots=[]) |
| 172 | assert json.loads(by_id[field_name])[0] == field_value, ( |
| 173 | f"{coll}: FINDING -- by-id read of a json.dumps-wrapped value " |
| 174 | f"did not round-trip. Expected {field_value!r}." |
| 175 | ) |
| 176 | |
| 177 | |
| 178 | # ============================================================================= |
| 179 | # 1. LONE SURROGATES |
| 180 | # ============================================================================= |
| 181 | # |
| 182 | # A Python str MAY hold an unpaired UTF-16 surrogate codepoint (U+D800- |
| 183 | # U+DFFF) -- json.loads of adversarial/malformed input, or a \uD800-style |
| 184 | # escape typed by a user, can produce one. Such a str has NO valid UTF-8 |
| 185 | # encoding (surrogates are reserved, not "supplementary plane" characters; |
| 186 | # CPython's UTF-8 codec raises UnicodeEncodeError rather than emitting |
| 187 | # CESU-8/WTF-8 bytes). The question: does object_records.create_collection_ |
| 188 | # record catch this and fail cleanly BEFORE writing anything, or does it |
| 189 | # corrupt the file / partially write? |
| 190 | |
| 191 | |
| 192 | LONE_SURROGATE = "\ud800" |
| 193 | |
| 194 | |
| 195 | def test_lone_surrogate_raw_field_classic_mode_raises_cleanly_no_partial_write(tmp_path): |
| 196 | """FINDING (nuance, not a corruption bug): a raw lone surrogate in a |
| 197 | field value raises -- but as a bare stdlib UnicodeEncodeError out of |
| 198 | _check_field_sizes's `str(value).encode("utf-8")` size probe, not as |
| 199 | object_records.InvalidRecordPayloadError. The failure IS clean (occurs |
| 200 | before create_collection_record ever opens records.tsv for write -- see |
| 201 | the assertion below that no file is created), so a caller retains data |
| 202 | integrity; but a caller that only catches InvalidRecordPayloadError to |
| 203 | turn bad input into a clean 4xx response would let this raw |
| 204 | UnicodeEncodeError escape uncaught instead.""" |
| 205 | data_dir = tmp_path / "data" |
| 206 | write_schema(data_dir, "widgets", [ID_FIELD, TEXT_FIELD]) |
| 207 | |
| 208 | with pytest.raises(object_records.InvalidRecordPayloadError): |
| 209 | object_records.create_collection_record( |
| 210 | "widgets", {"id": "w1", "note": LONE_SURROGATE}, base_dir=data_dir, roots=[] |
| 211 | ) |
| 212 | |
| 213 | records_path = data_dir / "collections" / "widgets" / "records.tsv" |
| 214 | assert not records_path.exists(), ( |
| 215 | "FINDING confirmed (worse than expected): a rejected lone-surrogate " |
| 216 | "write left a records.tsv file on disk despite raising." |
| 217 | ) |
| 218 | # Collection must still read back as genuinely empty, not error. |
| 219 | listing = object_records.list_collection_records("widgets", base_dir=data_dir, roots=[]) |
| 220 | assert listing["records"] == [] |
| 221 | assert listing["total"] == 0 |
| 222 | |
| 223 | |
| 224 | def test_lone_surrogate_raw_field_append_mode_raises_cleanly_no_partial_write(tmp_path): |
| 225 | """Same probe, append-mode collection: the failure must be equally clean |
| 226 | -- no `_op`-tagged partial row, no torn-tail artifact left behind that a |
| 227 | later self-heal or fold would have to reckon with.""" |
| 228 | data_dir = tmp_path / "data" |
| 229 | write_append_schema(data_dir, "widgets", [ID_FIELD, TEXT_FIELD]) |
| 230 | |
| 231 | with pytest.raises(object_records.InvalidRecordPayloadError): |
| 232 | object_records.create_collection_record( |
| 233 | "widgets", {"id": "w1", "note": LONE_SURROGATE}, base_dir=data_dir, roots=[] |
| 234 | ) |
| 235 | |
| 236 | records_path = data_dir / "collections" / "widgets" / "records.tsv" |
| 237 | assert not records_path.exists(), ( |
| 238 | "FINDING confirmed (worse than expected): a rejected lone-surrogate " |
| 239 | "write left a records.tsv file on disk despite raising." |
| 240 | ) |
| 241 | listing = object_records.list_collection_records("widgets", base_dir=data_dir, roots=[]) |
| 242 | assert listing["records"] == [] |
| 243 | |
| 244 | |
| 245 | def test_lone_surrogate_via_json_dumps_default_ensure_ascii_is_actually_safe(tmp_path, monkeypatch): |
| 246 | """Counter-intuitive PASS: json.dumps' DEFAULT (ensure_ascii=True) |
| 247 | backslash-escapes a lone surrogate to the literal ASCII text `\\ud800` |
| 248 | rather than embedding the real surrogate character -- so the STORED |
| 249 | blob is plain ASCII and trivially UTF-8-encodable. This is the |
| 250 | "realistic" shape for JSON produced by ordinary application code (no |
| 251 | one passes ensure_ascii=False by default), and it round-trips exactly, |
| 252 | in both modes, both read paths -- json.loads on the way back |
| 253 | reconstructs the very same lone-surrogate string in memory, with no |
| 254 | encode step involved. This is NOT the same as the raw-field or the |
| 255 | ensure_ascii=False case below; see those for where lone surrogates |
| 256 | actually break something.""" |
| 257 | original = "\ud800" |
| 258 | blob = json.dumps({"x": original}) # ensure_ascii=True (default) |
| 259 | assert "\ud800" not in blob # sanity: no raw surrogate byte in the blob itself |
| 260 | blob.encode("utf-8") # sanity: the blob itself is always encodable |
| 261 | |
| 262 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 263 | data_dir = tmp_path / "data" |
| 264 | write_schema(data_dir, "widgets_classic", [ID_FIELD, TEXT_FIELD]) |
| 265 | write_append_schema(data_dir, "widgets_append", [ID_FIELD, TEXT_FIELD]) |
| 266 | |
| 267 | for coll in ("widgets_classic", "widgets_append"): |
| 268 | object_records.create_collection_record( |
| 269 | coll, {"id": "w1", "note": blob}, base_dir=data_dir, roots=[] |
| 270 | ) |
| 271 | _clear_caches() |
| 272 | all_records = object_records.read_collection_records(coll, base_dir=data_dir, roots=[]) |
| 273 | assert json.loads(all_records[0]["note"])["x"] == original |
| 274 | |
| 275 | _clear_caches() |
| 276 | by_id = object_records.get_collection_record(coll, "w1", base_dir=data_dir, roots=[]) |
| 277 | assert json.loads(by_id["note"])["x"] == original |
| 278 | |
| 279 | |
| 280 | @pytest.mark.xfail(reason="substrate: NUL is now cleanly REJECTED (InvalidRecordPayloadError) and lone-CR corruption awaits the QUOTE_ALL-vs-reject-bare-CR format decision (plan/database-test-strategy.md). Assertion refresh bundled with that decision.", strict=False) |
| 281 | def test_lone_surrogate_via_json_dumps_ensure_ascii_false_raises_cleanly(tmp_path): |
| 282 | """FINDING (nuance, matches the raw-field case): json.dumps(..., ensure_ |
| 283 | ascii=False) embeds the REAL surrogate character into the JSON text |
| 284 | (rather than escaping it), producing a blob that itself has no valid |
| 285 | UTF-8 encoding -- exactly the "STORED blob may be unencodable" risk. |
| 286 | This IS a realistic path (ensure_ascii=False is a common choice to keep |
| 287 | non-ASCII text readable in stored JSON). Same clean-failure behavior as |
| 288 | the raw-field case: create_collection_record raises UnicodeEncodeError |
| 289 | before writing, no partial row, collection stays empty and readable.""" |
| 290 | original = "\ud800" |
| 291 | blob = json.dumps({"x": original}, ensure_ascii=False) |
| 292 | with pytest.raises(object_records.InvalidRecordPayloadError): |
| 293 | blob.encode("utf-8") # sanity: confirms the blob itself is unencodable |
| 294 | |
| 295 | data_dir = tmp_path / "data" |
| 296 | write_schema(data_dir, "widgets_classic", [ID_FIELD, TEXT_FIELD]) |
| 297 | write_append_schema(data_dir, "widgets_append", [ID_FIELD, TEXT_FIELD]) |
| 298 | |
| 299 | for coll in ("widgets_classic", "widgets_append"): |
| 300 | with pytest.raises(object_records.InvalidRecordPayloadError): |
| 301 | object_records.create_collection_record( |
| 302 | coll, {"id": "w1", "note": blob}, base_dir=data_dir, roots=[] |
| 303 | ) |
| 304 | records_path = data_dir / "collections" / coll / "records.tsv" |
| 305 | assert not records_path.exists(), ( |
| 306 | f"{coll}: FINDING confirmed (worse than expected): a rejected " |
| 307 | f"unencodable-JSON write left a records.tsv file on disk." |
| 308 | ) |
| 309 | |
| 310 | |
| 311 | def test_lone_surrogate_update_raises_cleanly_and_does_not_corrupt_existing_record(tmp_path): |
| 312 | """The update path runs the same _check_field_sizes probe (after schema |
| 313 | validation, before persist) -- confirm an update attempt with a lone |
| 314 | surrogate raises cleanly and leaves the EXISTING record completely |
| 315 | unchanged, in both storage modes.""" |
| 316 | for storage in ("classic", "append"): |
| 317 | data_dir = tmp_path / f"data_{storage}" |
| 318 | if storage == "classic": |
| 319 | write_schema(data_dir, "widgets", [ID_FIELD, TEXT_FIELD]) |
| 320 | else: |
| 321 | write_append_schema(data_dir, "widgets", [ID_FIELD, TEXT_FIELD]) |
| 322 | |
| 323 | object_records.create_collection_record( |
| 324 | "widgets", {"id": "w1", "note": "original value"}, base_dir=data_dir, roots=[] |
| 325 | ) |
| 326 | |
| 327 | with pytest.raises(object_records.InvalidRecordPayloadError): |
| 328 | object_records.update_collection_record( |
| 329 | "widgets", "w1", {"note": LONE_SURROGATE}, base_dir=data_dir, roots=[] |
| 330 | ) |
| 331 | |
| 332 | _clear_caches() |
| 333 | after = object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[]) |
| 334 | assert after["note"] == "original value", ( |
| 335 | f"{storage}: FINDING confirmed: a rejected update corrupted the " |
| 336 | f"existing record instead of leaving it untouched. Got: {after!r}" |
| 337 | ) |
| 338 | |
| 339 | |
| 340 | # ============================================================================= |
| 341 | # 2. ZERO-WIDTH CHARACTERS |
| 342 | # ============================================================================= |
| 343 | |
| 344 | |
| 345 | @pytest.mark.parametrize( |
| 346 | "label,value", |
| 347 | [ |
| 348 | ("zwsp", f"before{ZWSP}after"), |
| 349 | ("zwnj", f"be{ZWNJ}fore"), |
| 350 | ("zwj", f"a{ZWJ}b{ZWJ}c"), |
| 351 | ("word_joiner", f"x{WORD_JOINER}y"), |
| 352 | ("all_zero_width_mixed", f"a{ZWSP}b{ZWNJ}c{ZWJ}d{WORD_JOINER}e"), |
| 353 | ], |
| 354 | ) |
| 355 | def test_zero_width_chars_mixed_with_text_round_trip(tmp_path, monkeypatch, label, value): |
| 356 | """A zero-width character embedded in otherwise-visible text is a plain |
| 357 | Unicode scalar value with a normal UTF-8 encoding -- no CSV-special |
| 358 | bytes involved. Expected, and the finding target if violated: an exact |
| 359 | round trip with the zero-width character neither stripped nor |
| 360 | collapsed (some text-processing layers treat zero-width chars as |
| 361 | "whitespace-like" and trim them -- this substrate must not).""" |
| 362 | _round_trip_both_modes(tmp_path, monkeypatch, f"zw_{label}", value) |
| 363 | |
| 364 | |
| 365 | def test_zero_width_only_field_round_trips_and_is_not_treated_as_empty(tmp_path, monkeypatch): |
| 366 | """A field that is VISUALLY empty (all zero-width characters, no |
| 367 | visible glyph) but is not the empty string. Two properties checked: |
| 368 | 1. It round-trips byte-exact (doesn't get folded down to "" anywhere |
| 369 | in the write/read pipeline). |
| 370 | 2. object_records' own required/empty check (_is_empty: `value is |
| 371 | None or value == ""`) does NOT treat it as empty -- a `required` |
| 372 | field holding only zero-width characters must be ACCEPTED, since |
| 373 | by that same definition it is not blank. (This documents the |
| 374 | current, narrow definition of "empty" this substrate uses; it does |
| 375 | NOT claim that definition is desirable for every caller -- a |
| 376 | zero-width-only value being accepted as satisfying `required` is |
| 377 | itself worth flagging to product/schema owners, even though it is |
| 378 | not a storage-layer bug.) |
| 379 | """ |
| 380 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 381 | zero_width_only = ZWSP + ZWNJ + ZWJ + WORD_JOINER |
| 382 | data_dir = tmp_path / "data" |
| 383 | write_append_schema(data_dir, "forms", [ID_FIELD, _required_text_field()]) |
| 384 | |
| 385 | # Property 2: must NOT raise "field is required". |
| 386 | object_records.create_collection_record( |
| 387 | "forms", {"id": "f1", "note": zero_width_only}, base_dir=data_dir, roots=[] |
| 388 | ) |
| 389 | |
| 390 | # Property 1: byte-exact round trip, fold + by-id. |
| 391 | _clear_caches() |
| 392 | all_records = object_records.read_collection_records("forms", base_dir=data_dir, roots=[]) |
| 393 | assert all_records[0]["note"] == zero_width_only, ( |
| 394 | "FINDING: a zero-width-only field value was altered (likely " |
| 395 | f"collapsed to empty) on read. Got: {all_records[0]['note']!r}" |
| 396 | ) |
| 397 | |
| 398 | _clear_caches() |
| 399 | by_id = object_records.get_collection_record("forms", "f1", base_dir=data_dir, roots=[]) |
| 400 | assert by_id["note"] == zero_width_only, ( |
| 401 | "FINDING: a zero-width-only field value was altered on by-id read. " |
| 402 | f"Got: {by_id['note']!r}" |
| 403 | ) |
| 404 | |
| 405 | # A genuinely empty ("") required field must still be rejected -- this |
| 406 | # is the contrast case proving the substrate distinguishes the two. |
| 407 | with pytest.raises(object_records.InvalidRecordPayloadError): |
| 408 | object_records.create_collection_record( |
| 409 | "forms", {"id": "f2", "note": ""}, base_dir=data_dir, roots=[] |
| 410 | ) |
| 411 | |
| 412 | |
| 413 | # ============================================================================= |
| 414 | # 3. BIDI CONTROLS |
| 415 | # ============================================================================= |
| 416 | |
| 417 | |
| 418 | @pytest.mark.parametrize( |
| 419 | "label,value", |
| 420 | [ |
| 421 | ("rlo", f"normal {RLO}reversed{RLO}"), |
| 422 | ("lro", f"normal {LRO}forced-ltr{LRO}"), |
| 423 | ("isolates", f"{LRI}left{PDI} middle {RLI}right{PDI}"), |
| 424 | ("flip_direction", f"English {RLO}עברית{RLO} more English"), |
| 425 | ], |
| 426 | ) |
| 427 | def test_bidi_controls_byte_exact_round_trip_no_stripping(tmp_path, monkeypatch, label, value): |
| 428 | """Bidi control characters (RLO/LRO/RLI/LRI/PDI) are the classic |
| 429 | "trojan source" vector -- text that reads differently depending on |
| 430 | whether the control characters are honored or stripped. The storage |
| 431 | layer's job is narrow: hold the bytes exactly, neither stripping the |
| 432 | controls (which would silently change meaning) nor otherwise mangling |
| 433 | them. Probed both as a raw field and wrapped in compact json.dumps.""" |
| 434 | _round_trip_both_modes(tmp_path, monkeypatch, f"bidi_{label}", value) |
| 435 | _round_trip_json_wrapped_both_modes(tmp_path, monkeypatch, f"bidi_json_{label}", value) |
| 436 | |
| 437 | |
| 438 | # ============================================================================= |
| 439 | # 4. UNICODE NONCHARACTERS |
| 440 | # ============================================================================= |
| 441 | |
| 442 | |
| 443 | @pytest.mark.parametrize("codepoint", NONCHARACTERS) |
| 444 | def test_unicode_noncharacter_round_trips(tmp_path, monkeypatch, codepoint): |
| 445 | """U+FFFE/U+FFFF (and their counterparts at the end of every plane) and |
| 446 | U+FDD0-U+FDEF are permanently reserved "noncharacters" -- not illegal, |
| 447 | just never assigned a meaning, and some libraries reject/strip them as |
| 448 | a defensive measure. object_records has no such filtering (it is a |
| 449 | plain UTF-8 TSV substrate), so these should encode/decode/round-trip |
| 450 | exactly like any other scalar value -- included here specifically to |
| 451 | confirm the substrate does NOT do any noncharacter-aware filtering.""" |
| 452 | label = f"nonchar_{ord(codepoint):x}" |
| 453 | value = f"before{codepoint}after" |
| 454 | _round_trip_both_modes(tmp_path, monkeypatch, label, value) |
| 455 | |
| 456 | |
| 457 | def test_unicode_noncharacters_all_together_one_field(tmp_path, monkeypatch): |
| 458 | """All the sampled noncharacters concatenated into a single field, to |
| 459 | also exercise them adjacent to each other (not just individually |
| 460 | padded with ASCII).""" |
| 461 | value = "".join(NONCHARACTERS) |
| 462 | _round_trip_both_modes(tmp_path, monkeypatch, "nonchar_all", value) |
| 463 | |
| 464 | |
| 465 | # ============================================================================= |
| 466 | # 5. NORMALIZATION ATTACK |
| 467 | # ============================================================================= |
| 468 | |
| 469 | |
| 470 | def test_nfc_nfd_nfkc_forms_not_silently_normalized_and_remain_distinguishable(tmp_path, monkeypatch): |
| 471 | """The same VISUAL string ("café") in three different Unicode |
| 472 | normalization forms is, at the byte level, three different strings: |
| 473 | NFC uses the precomposed U+00E9 (é); NFD decomposes it to U+0065 U+0301 |
| 474 | (e + combining acute accent); NFKC (compatibility composition) happens |
| 475 | to coincide with NFC here but is computed independently below rather |
| 476 | than assumed. If the storage layer silently normalized on write (e.g. |
| 477 | via an implicit unicodedata.normalize somewhere in the pipeline), two |
| 478 | logically-different inputs could collide into the same stored bytes -- |
| 479 | a real security concern for anything using stored text for comparison/ |
| 480 | dedup/lookup. Assert: each form's stored bytes exactly match ITS OWN |
| 481 | input (no normalization at all), and the three stored values remain |
| 482 | pairwise distinct from each other, in both storage modes.""" |
| 483 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 484 | nfd = "café" # e + combining acute accent |
| 485 | nfc = unicodedata.normalize("NFC", nfd) # precomposed U+00E9 |
| 486 | nfkc = unicodedata.normalize("NFKC", nfd) |
| 487 | assert nfc != nfd # sanity: genuinely different byte sequences... |
| 488 | assert unicodedata.normalize("NFC", nfd) == nfc # ...that mean the same thing |
| 489 | |
| 490 | data_dir = tmp_path / "data" |
| 491 | write_schema(data_dir, "norm_classic", [ID_FIELD, TEXT_FIELD]) |
| 492 | write_append_schema(data_dir, "norm_append", [ID_FIELD, TEXT_FIELD]) |
| 493 | |
| 494 | forms = {"nfc": nfc, "nfd": nfd, "nfkc": nfkc} |
| 495 | for coll in ("norm_classic", "norm_append"): |
| 496 | for form_id, form_value in forms.items(): |
| 497 | object_records.create_collection_record( |
| 498 | coll, {"id": form_id, "note": form_value}, base_dir=data_dir, roots=[] |
| 499 | ) |
| 500 | |
| 501 | _clear_caches() |
| 502 | stored = { |
| 503 | form_id: object_records.get_collection_record(coll, form_id, base_dir=data_dir, roots=[])["note"] |
| 504 | for form_id in forms |
| 505 | } |
| 506 | for form_id, original in forms.items(): |
| 507 | assert stored[form_id] == original, ( |
| 508 | f"{coll}: FINDING -- silent normalization detected. Stored " |
| 509 | f"'{form_id}' value {stored[form_id]!r} != input {original!r}." |
| 510 | ) |
| 511 | # Pairwise distinctness -- would collapse to fewer than 3 distinct |
| 512 | # values if anything normalized on write. |
| 513 | distinct_values = {stored["nfc"], stored["nfd"], stored["nfkc"]} |
| 514 | assert len(distinct_values) == len({nfc, nfd, nfkc}), ( |
| 515 | f"{coll}: FINDING -- distinct normalization forms collapsed to " |
| 516 | f"fewer distinct stored values than were written: {stored!r}" |
| 517 | ) |
| 518 | |
| 519 | |
| 520 | # ============================================================================= |
| 521 | # 6. VERY LONG SINGLE GRAPHEME ("ZALGO") |
| 522 | # ============================================================================= |
| 523 | |
| 524 | |
| 525 | def test_zalgo_long_combining_grapheme_round_trip(tmp_path, monkeypatch): |
| 526 | """Thousands of combining marks (U+0300-U+036F, cycled) stacked on one |
| 527 | base character -- a single Python "character" position but a very long |
| 528 | string, and the classic Unicode stress-test for anything that assumes |
| 529 | grapheme clusters are short or that iterates by codepoint expecting |
| 530 | bounded work per visible character. The storage layer treats this as |
| 531 | plain text of some length in bytes; the finding target is a byte-exact |
| 532 | round trip at this length, in a single field, both modes.""" |
| 533 | combining_marks = "".join(chr(0x0300 + (i % 0x70)) for i in range(4000)) |
| 534 | zalgo = "e" + combining_marks |
| 535 | assert len(zalgo) == 4001 |
| 536 | byte_len = len(zalgo.encode("utf-8")) |
| 537 | assert byte_len < object_records.MAX_TSV_FIELD_BYTES # sanity: well under the per-field cap |
| 538 | _round_trip_both_modes(tmp_path, monkeypatch, "zalgo", zalgo) |
| 539 | |
| 540 | |
| 541 | # ============================================================================= |
| 542 | # 7. HOMOGLYPH / CONFUSABLE STRINGS |
| 543 | # ============================================================================= |
| 544 | |
| 545 | |
| 546 | def test_homoglyph_confusable_strings_byte_exact_and_stored_distinctly(tmp_path, monkeypatch): |
| 547 | """Cyrillic "а" (U+0430) vs Latin "a" (U+0061) render identically in |
| 548 | most fonts but are different codepoints/bytes. A storage layer that |
| 549 | silently folds/canonicalizes look-alike characters (e.g. via a |
| 550 | case-fold or a confusables-skeleton transform) would let two distinct |
| 551 | identities collide; this substrate should have no such behavior. Store |
| 552 | both as distinct records (distinct ids, since ids themselves are |
| 553 | separately validated/restricted -- the field under test is `note`, an |
| 554 | ordinary text field) and confirm each round-trips exactly and the two |
| 555 | stay byte-distinguishable from each other, both modes.""" |
| 556 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 557 | cyrillic_name = CYRILLIC_A + "dmin" # "аdmin" (Cyrillic а) |
| 558 | latin_name = LATIN_A + "dmin" # "admin" (Latin a) |
| 559 | assert cyrillic_name != latin_name |
| 560 | assert cyrillic_name.encode("utf-8") != latin_name.encode("utf-8") |
| 561 | |
| 562 | data_dir = tmp_path / "data" |
| 563 | write_schema(data_dir, "users_classic", [ID_FIELD, TEXT_FIELD]) |
| 564 | write_append_schema(data_dir, "users_append", [ID_FIELD, TEXT_FIELD]) |
| 565 | |
| 566 | for coll in ("users_classic", "users_append"): |
| 567 | object_records.create_collection_record( |
| 568 | coll, {"id": "u-cyrillic", "note": cyrillic_name}, base_dir=data_dir, roots=[] |
| 569 | ) |
| 570 | object_records.create_collection_record( |
| 571 | coll, {"id": "u-latin", "note": latin_name}, base_dir=data_dir, roots=[] |
| 572 | ) |
| 573 | |
| 574 | _clear_caches() |
| 575 | got_cyrillic = object_records.get_collection_record(coll, "u-cyrillic", base_dir=data_dir, roots=[]) |
| 576 | got_latin = object_records.get_collection_record(coll, "u-latin", base_dir=data_dir, roots=[]) |
| 577 | assert got_cyrillic["note"] == cyrillic_name, f"{coll}: FINDING -- Cyrillic homoglyph altered on read" |
| 578 | assert got_latin["note"] == latin_name, f"{coll}: FINDING -- Latin string altered on read" |
| 579 | assert got_cyrillic["note"] != got_latin["note"], ( |
| 580 | f"{coll}: FINDING -- homoglyph strings collapsed to the same " |
| 581 | f"stored value." |
| 582 | ) |
| 583 | |
| 584 | |
| 585 | # ============================================================================= |
| 586 | # 8. DELIMITER-SAFETY PROPERTY: no multibyte UTF-8 sequence contains 0x09/0x0A |
| 587 | # ============================================================================= |
| 588 | |
| 589 | |
| 590 | def test_no_multibyte_utf8_sequence_contains_tab_or_newline_byte(): |
| 591 | """By construction of UTF-8, every byte in a MULTI-byte sequence (the |
| 592 | lead byte and all continuation bytes) is >= 0x80 -- a multibyte |
| 593 | sequence can never contain a byte that collides with ASCII TAB (0x09) |
| 594 | or LF (0x0A) at the byte level, so a byte-level delimiter scan could |
| 595 | never accidentally split a multibyte character mid-sequence and |
| 596 | mistake a byte inside it for a real field/row delimiter. This is a |
| 597 | property of UTF-8 itself (not of object_records' parser, which uses |
| 598 | real CSV/TSV parsing rather than a byte-level scan regardless) -- |
| 599 | verified here directly, and then reinforced with an over-the-wire |
| 600 | round trip through the real storage path using exactly this file's |
| 601 | adversarial character set, to confirm the property holds for every |
| 602 | codepoint actually exercised in this file (not just a mathematical |
| 603 | argument about UTF-8 in the abstract).""" |
| 604 | probe_chars = ( |
| 605 | [ZWSP, ZWNJ, ZWJ, WORD_JOINER, RLO, LRO, RLI, LRI, PDI, CYRILLIC_A] |
| 606 | + NONCHARACTERS |
| 607 | + [chr(0x0300 + i) for i in range(0x70)] # combining marks used by the zalgo test |
| 608 | + ["é", "\U0001f600", "日本"] # accented latin, emoji, CJK, for good measure |
| 609 | ) |
| 610 | for ch in probe_chars: |
| 611 | encoded = ch.encode("utf-8") |
| 612 | if len(encoded) > 1: |
| 613 | assert 0x09 not in encoded, f"FINDING: U+{ord(ch):04X} encodes to bytes containing a TAB byte: {encoded!r}" |
| 614 | assert 0x0A not in encoded, f"FINDING: U+{ord(ch):04X} encodes to bytes containing a LF byte: {encoded!r}" |
| 615 | |
| 616 | # Systematic sweep across the full codepoint space (skipping surrogates, |
| 617 | # which cannot be UTF-8 encoded standalone -- covered separately in |
| 618 | # section 1), stepping to keep runtime bounded while still covering |
| 619 | # every UTF-8 length class (1/2/3/4-byte encodings). |
| 620 | step = 97 # coprime-ish stride, not aligned to any power-of-two boundary |
| 621 | checked = 0 |
| 622 | for cp in range(0x80, 0x110000, step): |
| 623 | if 0xD800 <= cp <= 0xDFFF: |
| 624 | continue # surrogate range: not independently UTF-8-encodable |
| 625 | encoded = chr(cp).encode("utf-8") |
| 626 | checked += 1 |
| 627 | if len(encoded) > 1: |
| 628 | assert 0x09 not in encoded, f"FINDING: U+{cp:04X} encodes to bytes containing a TAB byte: {encoded!r}" |
| 629 | assert 0x0A not in encoded, f"FINDING: U+{cp:04X} encodes to bytes containing a LF byte: {encoded!r}" |
| 630 | assert checked > 10000 # sanity: the sweep actually covered a meaningful sample |
| 631 | |
| 632 | |
| 633 | def test_delimiter_safety_holds_through_the_real_storage_path(tmp_path, monkeypatch): |
| 634 | """Reinforcement of the property above through the actual create/read |
| 635 | path: a field packed with every adversarial character from this file, |
| 636 | back-to-back with no ASCII padding between them (the adversarial case |
| 637 | for any byte-level scan), must still round-trip exactly -- and, as a |
| 638 | structural check, the physical file must contain exactly one data row |
| 639 | (a naive byte-level split on this content, if the storage layer used |
| 640 | one, would either fragment it across "rows" or corrupt the field |
| 641 | count; the real csv-based parser must not).""" |
| 642 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 643 | packed = "".join( |
| 644 | [ZWSP, ZWNJ, ZWJ, WORD_JOINER, RLO, LRO, RLI, PDI, CYRILLIC_A] |
| 645 | + NONCHARACTERS |
| 646 | + [chr(0x0300 + i) for i in range(20)] |
| 647 | ) |
| 648 | for ch in packed: |
| 649 | assert ch.encode("utf-8").isascii() is False or ord(ch) < 0x80 # sanity, non-fatal |
| 650 | |
| 651 | data_dir = tmp_path / "data" |
| 652 | write_schema(data_dir, "packed_classic", [ID_FIELD, TEXT_FIELD]) |
| 653 | write_append_schema(data_dir, "packed_append", [ID_FIELD, TEXT_FIELD]) |
| 654 | |
| 655 | for coll in ("packed_classic", "packed_append"): |
| 656 | object_records.create_collection_record( |
| 657 | coll, {"id": "p1", "note": packed}, base_dir=data_dir, roots=[] |
| 658 | ) |
| 659 | object_records.create_collection_record( |
| 660 | coll, {"id": "p2", "note": "sibling"}, base_dir=data_dir, roots=[] |
| 661 | ) |
| 662 | |
| 663 | _clear_caches() |
| 664 | all_records = object_records.read_collection_records(coll, base_dir=data_dir, roots=[]) |
| 665 | assert len(all_records) == 2, ( |
| 666 | f"{coll}: FINDING -- packed adversarial content changed the " |
| 667 | f"apparent row count (byte-level mis-split?). Got: {all_records!r}" |
| 668 | ) |
| 669 | by_id = {r["id"]: r["note"] for r in all_records} |
| 670 | assert by_id["p1"] == packed |
| 671 | assert by_id["p2"] == "sibling" |
| 672 | |
| 673 | |
| 674 | # ============================================================================= |
| 675 | # 9. UPDATE PATH -- adversarial content surviving a create -> update sequence |
| 676 | # ============================================================================= |
| 677 | |
| 678 | |
| 679 | def test_update_sequence_through_zero_width_bidi_and_noncharacter_content(tmp_path, monkeypatch): |
| 680 | """A single record's field is created plain, then updated THROUGH a |
| 681 | sequence of adversarial values (zero-width-only, bidi-flipped, |
| 682 | noncharacter-laden, back to plain), asserting an exact round trip after |
| 683 | every step, both storage modes. This is the "and update" axis of the |
| 684 | probe matrix: confirms the update path (not just create) preserves |
| 685 | adversarial content exactly, including the merge-with-existing-record |
| 686 | step every update performs internally.""" |
| 687 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 688 | sequence = [ |
| 689 | "plain start", |
| 690 | ZWSP + ZWNJ + ZWJ, |
| 691 | f"{RLO}reversed text{RLO} trailing", |
| 692 | "".join(NONCHARACTERS), |
| 693 | "plain end", |
| 694 | ] |
| 695 | |
| 696 | for storage in ("classic", "append"): |
| 697 | data_dir = tmp_path / f"data_{storage}" |
| 698 | if storage == "classic": |
| 699 | write_schema(data_dir, "seq", [ID_FIELD, TEXT_FIELD]) |
| 700 | else: |
| 701 | write_append_schema(data_dir, "seq", [ID_FIELD, TEXT_FIELD]) |
| 702 | |
| 703 | object_records.create_collection_record( |
| 704 | "seq", {"id": "s1", "note": sequence[0]}, base_dir=data_dir, roots=[] |
| 705 | ) |
| 706 | _clear_caches() |
| 707 | assert object_records.get_collection_record("seq", "s1", base_dir=data_dir, roots=[])["note"] == sequence[0] |
| 708 | |
| 709 | for step_value in sequence[1:]: |
| 710 | object_records.update_collection_record( |
| 711 | "seq", "s1", {"note": step_value}, base_dir=data_dir, roots=[] |
| 712 | ) |
| 713 | _clear_caches() |
| 714 | current = object_records.get_collection_record("seq", "s1", base_dir=data_dir, roots=[]) |
| 715 | assert current["note"] == step_value, ( |
| 716 | f"{storage}: FINDING -- update to {step_value!r} did not " |
| 717 | f"round-trip. Got: {current['note']!r}" |
| 718 | ) |