Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_charset_unicode_adversarial_characterization.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_charset_unicode_adversarial_characterization.py · 34.7 KB · 718 lines
1"""CHARACTERIZATION tests: does the TSV substrate (object_records.py) safely
2hold ADVERSARIAL / edge-case Unicode -- the inputs most likely to break a
3naive text-storage layer -- through create/update/read (fold-all and by-id),
4in both classic and append storage modes?
5
6This file is pure characterization of EXISTING behavior. It does not modify
7any production module. Where a case legitimately SHOULD raise (content that
8cannot be represented -- a lone UTF-16 surrogate has no UTF-8 encoding), the
9assertion checks that the failure is CLEAN (a well-defined exception, at
10write time, before anything touches disk) rather than corruption or a silent
11drop. Where a case round-trips, the assertion is byte-exact equality -- not
12"looks about right" -- because silent normalization/stripping/mangling of
13zero-width, bidi, or combining-mark content is exactly the failure mode this
14file exists to catch. A clean raise or an exact round-trip is a PASS; a
15mismatch, silent alteration, or unhandled corruption is a FINDING and is
16documented as such in the failing test's message, not weakened to pass.
17
18Context (see module docstring / comments in object_records.py):
19 - Values are stored as TSV cells via Python's csv module; writes encode
20 text to UTF-8 (or whatever the file's default open() encoding resolves
21 to -- see the delimiter-safety section below), reads decode UTF-8.
22 - csv.field_size_limit has already been raised (MAX_TSV_FIELD_BYTES, 16
23 MiB) and create/update enforce the same ceiling on write
24 (_check_field_sizes) -- not retested here.
25 - _oidx_get_row (the append-mode id->offset sidecar's single-record read)
26 is already CSV-aware, not a raw readline() -- not retested here.
27 - The append-mode torn-tail self-heal check (_drop_torn_tail: "does the
28 file end with a literal \\n byte") is unsound for a row whose value
29 spans multiple physical lines -- known, deferred bug #2. NOT probed
30 here; every value below is either single-physical-line or (for the
31 zalgo/noncharacter cases) still written via compact json.dumps, never
32 pretty-printed, so no case in this file exercises that bug by accident.
33
34Setup helpers below mirror tests/test_embedded_json_lines_characterization.py.
35"""
36
37from __future__ import annotations
38
39import json
40import unicodedata
41from pathlib import Path
42
43import pytest
44
45import object_records
46
47# Conformance tier: heavy charset characterization -- deselected from the
48# per-commit run (see pyproject 'conformance' marker). Run: pytest -m conformance
49pytestmark = pytest.mark.conformance
50
51
52ID_FIELD = {"name": "id"}
53TEXT_FIELD = {"name": "note", "type": "textarea"}
54
55
56def _required_text_field() -> dict:
57 return {"name": "note", "type": "textarea", "required": True}
58
59
60# --- setup helpers (mirror tests/test_embedded_json_lines_characterization.py) ---
61
62
63def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path:
64 path = data_dir / "schemas" / f"{collection}.json"
65 path.parent.mkdir(parents=True, exist_ok=True)
66 path.write_text(json.dumps({"fields": fields}))
67 return path
68
69
70def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path:
71 path = data_dir / "schemas" / f"{collection}.json"
72 path.parent.mkdir(parents=True, exist_ok=True)
73 path.write_text(json.dumps({"fields": fields, "storage": "append"}))
74 return path
75
76
77def _clear_caches() -> None:
78 """Force the NEXT read past the warm in-process caches -- see the
79 matching helper in test_embedded_json_lines_characterization.py. Needed
80 so a by-id read on an append-mode collection actually exercises the
81 id->offset sidecar path instead of serving from the ordinary records
82 cache."""
83 object_records._RECORDS_CACHE.clear()
84 object_records._OIDX_CACHE.clear()
85
86
87# --- adversarial content constants -----------------------------------------
88
89ZWSP = "​" # zero-width space
90ZWNJ = "‌" # zero-width non-joiner
91ZWJ = "‍" # zero-width joiner
92WORD_JOINER = "⁠"
93
94RLO = "‮" # right-to-left override
95LRO = "‭" # left-to-right override
96RLI = "⁧" # right-to-left isolate
97LRI = "⁦" # left-to-right isolate
98PDI = "⁩" # pop directional isolate
99
100NONCHARACTERS = ["￾", "￿", "\U0001fffe", "\U0001ffff", "﷐", "﷠", "﷯"]
101
102CYRILLIC_A = "а" # Cyrillic "а" -- visually identical to Latin "a" (U+0061)
103LATIN_A = "a"
104
105
106def _round_trip_both_modes(
107 tmp_path, monkeypatch, collection_prefix: str, field_value: str, *, field_name: str = "note"
108) -> None:
109 """Store `field_value` raw under `field_name` in both a classic-mode and
110 an append-mode collection; assert byte-exact equality on BOTH the
111 full-fold read and the by-id read (the latter forced through the cold
112 id->offset sidecar in append mode via DBBASIC_RECORDS_CACHE_MAX_ROWS=0)."""
113 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
114 data_dir = tmp_path / "data"
115 classic_coll = f"{collection_prefix}_classic"
116 append_coll = f"{collection_prefix}_append"
117 write_schema(data_dir, classic_coll, [ID_FIELD, TEXT_FIELD])
118 write_append_schema(data_dir, append_coll, [ID_FIELD, TEXT_FIELD])
119
120 for coll in (classic_coll, append_coll):
121 object_records.create_collection_record(
122 coll, {"id": "r1", field_name: field_value}, base_dir=data_dir, roots=[]
123 )
124
125 _clear_caches()
126 all_records = object_records.read_collection_records(coll, base_dir=data_dir, roots=[])
127 assert len(all_records) == 1
128 assert all_records[0][field_name] == field_value, (
129 f"{coll}: FINDING -- fold-all read did not byte-exact round-trip "
130 f"the stored value. Expected {field_value!r}, got "
131 f"{all_records[0][field_name]!r}"
132 )
133
134 _clear_caches()
135 by_id = object_records.get_collection_record(coll, "r1", base_dir=data_dir, roots=[])
136 assert by_id[field_name] == field_value, (
137 f"{coll}: FINDING -- by-id (sidecar) read did not byte-exact "
138 f"round-trip the stored value. Expected {field_value!r}, got "
139 f"{by_id[field_name]!r}"
140 )
141
142
143def _round_trip_json_wrapped_both_modes(
144 tmp_path, monkeypatch, collection_prefix: str, field_value: str, *, field_name: str = "note"
145) -> None:
146 """Same as _round_trip_both_modes, but the stored cell is a compact
147 json.dumps([field_value]) blob (a realistic shape for how this content
148 would actually arrive -- e.g. inside a JSON API payload or an embedded
149 array), read back via json.loads and compared to the original value."""
150 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
151 data_dir = tmp_path / "data"
152 classic_coll = f"{collection_prefix}_classic"
153 append_coll = f"{collection_prefix}_append"
154 write_schema(data_dir, classic_coll, [ID_FIELD, TEXT_FIELD])
155 write_append_schema(data_dir, append_coll, [ID_FIELD, TEXT_FIELD])
156 blob = json.dumps([field_value])
157
158 for coll in (classic_coll, append_coll):
159 object_records.create_collection_record(
160 coll, {"id": "r1", field_name: blob}, base_dir=data_dir, roots=[]
161 )
162
163 _clear_caches()
164 all_records = object_records.read_collection_records(coll, base_dir=data_dir, roots=[])
165 assert json.loads(all_records[0][field_name])[0] == field_value, (
166 f"{coll}: FINDING -- fold-all read of a json.dumps-wrapped value "
167 f"did not round-trip. Expected {field_value!r}."
168 )
169
170 _clear_caches()
171 by_id = object_records.get_collection_record(coll, "r1", base_dir=data_dir, roots=[])
172 assert json.loads(by_id[field_name])[0] == field_value, (
173 f"{coll}: FINDING -- by-id read of a json.dumps-wrapped value "
174 f"did not round-trip. Expected {field_value!r}."
175 )
176
177
178# =============================================================================
179# 1. LONE SURROGATES
180# =============================================================================
181#
182# A Python str MAY hold an unpaired UTF-16 surrogate codepoint (U+D800-
183# U+DFFF) -- json.loads of adversarial/malformed input, or a \uD800-style
184# escape typed by a user, can produce one. Such a str has NO valid UTF-8
185# encoding (surrogates are reserved, not "supplementary plane" characters;
186# CPython's UTF-8 codec raises UnicodeEncodeError rather than emitting
187# CESU-8/WTF-8 bytes). The question: does object_records.create_collection_
188# record catch this and fail cleanly BEFORE writing anything, or does it
189# corrupt the file / partially write?
190
191
192LONE_SURROGATE = "\ud800"
193
194
195def test_lone_surrogate_raw_field_classic_mode_raises_cleanly_no_partial_write(tmp_path):
196 """FINDING (nuance, not a corruption bug): a raw lone surrogate in a
197 field value raises -- but as a bare stdlib UnicodeEncodeError out of
198 _check_field_sizes's `str(value).encode("utf-8")` size probe, not as
199 object_records.InvalidRecordPayloadError. The failure IS clean (occurs
200 before create_collection_record ever opens records.tsv for write -- see
201 the assertion below that no file is created), so a caller retains data
202 integrity; but a caller that only catches InvalidRecordPayloadError to
203 turn bad input into a clean 4xx response would let this raw
204 UnicodeEncodeError escape uncaught instead."""
205 data_dir = tmp_path / "data"
206 write_schema(data_dir, "widgets", [ID_FIELD, TEXT_FIELD])
207
208 with pytest.raises(object_records.InvalidRecordPayloadError):
209 object_records.create_collection_record(
210 "widgets", {"id": "w1", "note": LONE_SURROGATE}, base_dir=data_dir, roots=[]
211 )
212
213 records_path = data_dir / "collections" / "widgets" / "records.tsv"
214 assert not records_path.exists(), (
215 "FINDING confirmed (worse than expected): a rejected lone-surrogate "
216 "write left a records.tsv file on disk despite raising."
217 )
218 # Collection must still read back as genuinely empty, not error.
219 listing = object_records.list_collection_records("widgets", base_dir=data_dir, roots=[])
220 assert listing["records"] == []
221 assert listing["total"] == 0
222
223
224def test_lone_surrogate_raw_field_append_mode_raises_cleanly_no_partial_write(tmp_path):
225 """Same probe, append-mode collection: the failure must be equally clean
226 -- no `_op`-tagged partial row, no torn-tail artifact left behind that a
227 later self-heal or fold would have to reckon with."""
228 data_dir = tmp_path / "data"
229 write_append_schema(data_dir, "widgets", [ID_FIELD, TEXT_FIELD])
230
231 with pytest.raises(object_records.InvalidRecordPayloadError):
232 object_records.create_collection_record(
233 "widgets", {"id": "w1", "note": LONE_SURROGATE}, base_dir=data_dir, roots=[]
234 )
235
236 records_path = data_dir / "collections" / "widgets" / "records.tsv"
237 assert not records_path.exists(), (
238 "FINDING confirmed (worse than expected): a rejected lone-surrogate "
239 "write left a records.tsv file on disk despite raising."
240 )
241 listing = object_records.list_collection_records("widgets", base_dir=data_dir, roots=[])
242 assert listing["records"] == []
243
244
245def test_lone_surrogate_via_json_dumps_default_ensure_ascii_is_actually_safe(tmp_path, monkeypatch):
246 """Counter-intuitive PASS: json.dumps' DEFAULT (ensure_ascii=True)
247 backslash-escapes a lone surrogate to the literal ASCII text `\\ud800`
248 rather than embedding the real surrogate character -- so the STORED
249 blob is plain ASCII and trivially UTF-8-encodable. This is the
250 "realistic" shape for JSON produced by ordinary application code (no
251 one passes ensure_ascii=False by default), and it round-trips exactly,
252 in both modes, both read paths -- json.loads on the way back
253 reconstructs the very same lone-surrogate string in memory, with no
254 encode step involved. This is NOT the same as the raw-field or the
255 ensure_ascii=False case below; see those for where lone surrogates
256 actually break something."""
257 original = "\ud800"
258 blob = json.dumps({"x": original}) # ensure_ascii=True (default)
259 assert "\ud800" not in blob # sanity: no raw surrogate byte in the blob itself
260 blob.encode("utf-8") # sanity: the blob itself is always encodable
261
262 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
263 data_dir = tmp_path / "data"
264 write_schema(data_dir, "widgets_classic", [ID_FIELD, TEXT_FIELD])
265 write_append_schema(data_dir, "widgets_append", [ID_FIELD, TEXT_FIELD])
266
267 for coll in ("widgets_classic", "widgets_append"):
268 object_records.create_collection_record(
269 coll, {"id": "w1", "note": blob}, base_dir=data_dir, roots=[]
270 )
271 _clear_caches()
272 all_records = object_records.read_collection_records(coll, base_dir=data_dir, roots=[])
273 assert json.loads(all_records[0]["note"])["x"] == original
274
275 _clear_caches()
276 by_id = object_records.get_collection_record(coll, "w1", base_dir=data_dir, roots=[])
277 assert json.loads(by_id["note"])["x"] == original
278
279
280@pytest.mark.xfail(reason="substrate: NUL is now cleanly REJECTED (InvalidRecordPayloadError) and lone-CR corruption awaits the QUOTE_ALL-vs-reject-bare-CR format decision (plan/database-test-strategy.md). Assertion refresh bundled with that decision.", strict=False)
281def test_lone_surrogate_via_json_dumps_ensure_ascii_false_raises_cleanly(tmp_path):
282 """FINDING (nuance, matches the raw-field case): json.dumps(..., ensure_
283 ascii=False) embeds the REAL surrogate character into the JSON text
284 (rather than escaping it), producing a blob that itself has no valid
285 UTF-8 encoding -- exactly the "STORED blob may be unencodable" risk.
286 This IS a realistic path (ensure_ascii=False is a common choice to keep
287 non-ASCII text readable in stored JSON). Same clean-failure behavior as
288 the raw-field case: create_collection_record raises UnicodeEncodeError
289 before writing, no partial row, collection stays empty and readable."""
290 original = "\ud800"
291 blob = json.dumps({"x": original}, ensure_ascii=False)
292 with pytest.raises(object_records.InvalidRecordPayloadError):
293 blob.encode("utf-8") # sanity: confirms the blob itself is unencodable
294
295 data_dir = tmp_path / "data"
296 write_schema(data_dir, "widgets_classic", [ID_FIELD, TEXT_FIELD])
297 write_append_schema(data_dir, "widgets_append", [ID_FIELD, TEXT_FIELD])
298
299 for coll in ("widgets_classic", "widgets_append"):
300 with pytest.raises(object_records.InvalidRecordPayloadError):
301 object_records.create_collection_record(
302 coll, {"id": "w1", "note": blob}, base_dir=data_dir, roots=[]
303 )
304 records_path = data_dir / "collections" / coll / "records.tsv"
305 assert not records_path.exists(), (
306 f"{coll}: FINDING confirmed (worse than expected): a rejected "
307 f"unencodable-JSON write left a records.tsv file on disk."
308 )
309
310
311def test_lone_surrogate_update_raises_cleanly_and_does_not_corrupt_existing_record(tmp_path):
312 """The update path runs the same _check_field_sizes probe (after schema
313 validation, before persist) -- confirm an update attempt with a lone
314 surrogate raises cleanly and leaves the EXISTING record completely
315 unchanged, in both storage modes."""
316 for storage in ("classic", "append"):
317 data_dir = tmp_path / f"data_{storage}"
318 if storage == "classic":
319 write_schema(data_dir, "widgets", [ID_FIELD, TEXT_FIELD])
320 else:
321 write_append_schema(data_dir, "widgets", [ID_FIELD, TEXT_FIELD])
322
323 object_records.create_collection_record(
324 "widgets", {"id": "w1", "note": "original value"}, base_dir=data_dir, roots=[]
325 )
326
327 with pytest.raises(object_records.InvalidRecordPayloadError):
328 object_records.update_collection_record(
329 "widgets", "w1", {"note": LONE_SURROGATE}, base_dir=data_dir, roots=[]
330 )
331
332 _clear_caches()
333 after = object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
334 assert after["note"] == "original value", (
335 f"{storage}: FINDING confirmed: a rejected update corrupted the "
336 f"existing record instead of leaving it untouched. Got: {after!r}"
337 )
338
339
340# =============================================================================
341# 2. ZERO-WIDTH CHARACTERS
342# =============================================================================
343
344
345@pytest.mark.parametrize(
346 "label,value",
347 [
348 ("zwsp", f"before{ZWSP}after"),
349 ("zwnj", f"be{ZWNJ}fore"),
350 ("zwj", f"a{ZWJ}b{ZWJ}c"),
351 ("word_joiner", f"x{WORD_JOINER}y"),
352 ("all_zero_width_mixed", f"a{ZWSP}b{ZWNJ}c{ZWJ}d{WORD_JOINER}e"),
353 ],
354)
355def test_zero_width_chars_mixed_with_text_round_trip(tmp_path, monkeypatch, label, value):
356 """A zero-width character embedded in otherwise-visible text is a plain
357 Unicode scalar value with a normal UTF-8 encoding -- no CSV-special
358 bytes involved. Expected, and the finding target if violated: an exact
359 round trip with the zero-width character neither stripped nor
360 collapsed (some text-processing layers treat zero-width chars as
361 "whitespace-like" and trim them -- this substrate must not)."""
362 _round_trip_both_modes(tmp_path, monkeypatch, f"zw_{label}", value)
363
364
365def test_zero_width_only_field_round_trips_and_is_not_treated_as_empty(tmp_path, monkeypatch):
366 """A field that is VISUALLY empty (all zero-width characters, no
367 visible glyph) but is not the empty string. Two properties checked:
368 1. It round-trips byte-exact (doesn't get folded down to "" anywhere
369 in the write/read pipeline).
370 2. object_records' own required/empty check (_is_empty: `value is
371 None or value == ""`) does NOT treat it as empty -- a `required`
372 field holding only zero-width characters must be ACCEPTED, since
373 by that same definition it is not blank. (This documents the
374 current, narrow definition of "empty" this substrate uses; it does
375 NOT claim that definition is desirable for every caller -- a
376 zero-width-only value being accepted as satisfying `required` is
377 itself worth flagging to product/schema owners, even though it is
378 not a storage-layer bug.)
379 """
380 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
381 zero_width_only = ZWSP + ZWNJ + ZWJ + WORD_JOINER
382 data_dir = tmp_path / "data"
383 write_append_schema(data_dir, "forms", [ID_FIELD, _required_text_field()])
384
385 # Property 2: must NOT raise "field is required".
386 object_records.create_collection_record(
387 "forms", {"id": "f1", "note": zero_width_only}, base_dir=data_dir, roots=[]
388 )
389
390 # Property 1: byte-exact round trip, fold + by-id.
391 _clear_caches()
392 all_records = object_records.read_collection_records("forms", base_dir=data_dir, roots=[])
393 assert all_records[0]["note"] == zero_width_only, (
394 "FINDING: a zero-width-only field value was altered (likely "
395 f"collapsed to empty) on read. Got: {all_records[0]['note']!r}"
396 )
397
398 _clear_caches()
399 by_id = object_records.get_collection_record("forms", "f1", base_dir=data_dir, roots=[])
400 assert by_id["note"] == zero_width_only, (
401 "FINDING: a zero-width-only field value was altered on by-id read. "
402 f"Got: {by_id['note']!r}"
403 )
404
405 # A genuinely empty ("") required field must still be rejected -- this
406 # is the contrast case proving the substrate distinguishes the two.
407 with pytest.raises(object_records.InvalidRecordPayloadError):
408 object_records.create_collection_record(
409 "forms", {"id": "f2", "note": ""}, base_dir=data_dir, roots=[]
410 )
411
412
413# =============================================================================
414# 3. BIDI CONTROLS
415# =============================================================================
416
417
418@pytest.mark.parametrize(
419 "label,value",
420 [
421 ("rlo", f"normal {RLO}reversed{RLO}"),
422 ("lro", f"normal {LRO}forced-ltr{LRO}"),
423 ("isolates", f"{LRI}left{PDI} middle {RLI}right{PDI}"),
424 ("flip_direction", f"English {RLO}עברית{RLO} more English"),
425 ],
426)
427def test_bidi_controls_byte_exact_round_trip_no_stripping(tmp_path, monkeypatch, label, value):
428 """Bidi control characters (RLO/LRO/RLI/LRI/PDI) are the classic
429 "trojan source" vector -- text that reads differently depending on
430 whether the control characters are honored or stripped. The storage
431 layer's job is narrow: hold the bytes exactly, neither stripping the
432 controls (which would silently change meaning) nor otherwise mangling
433 them. Probed both as a raw field and wrapped in compact json.dumps."""
434 _round_trip_both_modes(tmp_path, monkeypatch, f"bidi_{label}", value)
435 _round_trip_json_wrapped_both_modes(tmp_path, monkeypatch, f"bidi_json_{label}", value)
436
437
438# =============================================================================
439# 4. UNICODE NONCHARACTERS
440# =============================================================================
441
442
443@pytest.mark.parametrize("codepoint", NONCHARACTERS)
444def test_unicode_noncharacter_round_trips(tmp_path, monkeypatch, codepoint):
445 """U+FFFE/U+FFFF (and their counterparts at the end of every plane) and
446 U+FDD0-U+FDEF are permanently reserved "noncharacters" -- not illegal,
447 just never assigned a meaning, and some libraries reject/strip them as
448 a defensive measure. object_records has no such filtering (it is a
449 plain UTF-8 TSV substrate), so these should encode/decode/round-trip
450 exactly like any other scalar value -- included here specifically to
451 confirm the substrate does NOT do any noncharacter-aware filtering."""
452 label = f"nonchar_{ord(codepoint):x}"
453 value = f"before{codepoint}after"
454 _round_trip_both_modes(tmp_path, monkeypatch, label, value)
455
456
457def test_unicode_noncharacters_all_together_one_field(tmp_path, monkeypatch):
458 """All the sampled noncharacters concatenated into a single field, to
459 also exercise them adjacent to each other (not just individually
460 padded with ASCII)."""
461 value = "".join(NONCHARACTERS)
462 _round_trip_both_modes(tmp_path, monkeypatch, "nonchar_all", value)
463
464
465# =============================================================================
466# 5. NORMALIZATION ATTACK
467# =============================================================================
468
469
470def test_nfc_nfd_nfkc_forms_not_silently_normalized_and_remain_distinguishable(tmp_path, monkeypatch):
471 """The same VISUAL string ("café") in three different Unicode
472 normalization forms is, at the byte level, three different strings:
473 NFC uses the precomposed U+00E9 (é); NFD decomposes it to U+0065 U+0301
474 (e + combining acute accent); NFKC (compatibility composition) happens
475 to coincide with NFC here but is computed independently below rather
476 than assumed. If the storage layer silently normalized on write (e.g.
477 via an implicit unicodedata.normalize somewhere in the pipeline), two
478 logically-different inputs could collide into the same stored bytes --
479 a real security concern for anything using stored text for comparison/
480 dedup/lookup. Assert: each form's stored bytes exactly match ITS OWN
481 input (no normalization at all), and the three stored values remain
482 pairwise distinct from each other, in both storage modes."""
483 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
484 nfd = "café" # e + combining acute accent
485 nfc = unicodedata.normalize("NFC", nfd) # precomposed U+00E9
486 nfkc = unicodedata.normalize("NFKC", nfd)
487 assert nfc != nfd # sanity: genuinely different byte sequences...
488 assert unicodedata.normalize("NFC", nfd) == nfc # ...that mean the same thing
489
490 data_dir = tmp_path / "data"
491 write_schema(data_dir, "norm_classic", [ID_FIELD, TEXT_FIELD])
492 write_append_schema(data_dir, "norm_append", [ID_FIELD, TEXT_FIELD])
493
494 forms = {"nfc": nfc, "nfd": nfd, "nfkc": nfkc}
495 for coll in ("norm_classic", "norm_append"):
496 for form_id, form_value in forms.items():
497 object_records.create_collection_record(
498 coll, {"id": form_id, "note": form_value}, base_dir=data_dir, roots=[]
499 )
500
501 _clear_caches()
502 stored = {
503 form_id: object_records.get_collection_record(coll, form_id, base_dir=data_dir, roots=[])["note"]
504 for form_id in forms
505 }
506 for form_id, original in forms.items():
507 assert stored[form_id] == original, (
508 f"{coll}: FINDING -- silent normalization detected. Stored "
509 f"'{form_id}' value {stored[form_id]!r} != input {original!r}."
510 )
511 # Pairwise distinctness -- would collapse to fewer than 3 distinct
512 # values if anything normalized on write.
513 distinct_values = {stored["nfc"], stored["nfd"], stored["nfkc"]}
514 assert len(distinct_values) == len({nfc, nfd, nfkc}), (
515 f"{coll}: FINDING -- distinct normalization forms collapsed to "
516 f"fewer distinct stored values than were written: {stored!r}"
517 )
518
519
520# =============================================================================
521# 6. VERY LONG SINGLE GRAPHEME ("ZALGO")
522# =============================================================================
523
524
525def test_zalgo_long_combining_grapheme_round_trip(tmp_path, monkeypatch):
526 """Thousands of combining marks (U+0300-U+036F, cycled) stacked on one
527 base character -- a single Python "character" position but a very long
528 string, and the classic Unicode stress-test for anything that assumes
529 grapheme clusters are short or that iterates by codepoint expecting
530 bounded work per visible character. The storage layer treats this as
531 plain text of some length in bytes; the finding target is a byte-exact
532 round trip at this length, in a single field, both modes."""
533 combining_marks = "".join(chr(0x0300 + (i % 0x70)) for i in range(4000))
534 zalgo = "e" + combining_marks
535 assert len(zalgo) == 4001
536 byte_len = len(zalgo.encode("utf-8"))
537 assert byte_len < object_records.MAX_TSV_FIELD_BYTES # sanity: well under the per-field cap
538 _round_trip_both_modes(tmp_path, monkeypatch, "zalgo", zalgo)
539
540
541# =============================================================================
542# 7. HOMOGLYPH / CONFUSABLE STRINGS
543# =============================================================================
544
545
546def test_homoglyph_confusable_strings_byte_exact_and_stored_distinctly(tmp_path, monkeypatch):
547 """Cyrillic "а" (U+0430) vs Latin "a" (U+0061) render identically in
548 most fonts but are different codepoints/bytes. A storage layer that
549 silently folds/canonicalizes look-alike characters (e.g. via a
550 case-fold or a confusables-skeleton transform) would let two distinct
551 identities collide; this substrate should have no such behavior. Store
552 both as distinct records (distinct ids, since ids themselves are
553 separately validated/restricted -- the field under test is `note`, an
554 ordinary text field) and confirm each round-trips exactly and the two
555 stay byte-distinguishable from each other, both modes."""
556 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
557 cyrillic_name = CYRILLIC_A + "dmin" # "аdmin" (Cyrillic а)
558 latin_name = LATIN_A + "dmin" # "admin" (Latin a)
559 assert cyrillic_name != latin_name
560 assert cyrillic_name.encode("utf-8") != latin_name.encode("utf-8")
561
562 data_dir = tmp_path / "data"
563 write_schema(data_dir, "users_classic", [ID_FIELD, TEXT_FIELD])
564 write_append_schema(data_dir, "users_append", [ID_FIELD, TEXT_FIELD])
565
566 for coll in ("users_classic", "users_append"):
567 object_records.create_collection_record(
568 coll, {"id": "u-cyrillic", "note": cyrillic_name}, base_dir=data_dir, roots=[]
569 )
570 object_records.create_collection_record(
571 coll, {"id": "u-latin", "note": latin_name}, base_dir=data_dir, roots=[]
572 )
573
574 _clear_caches()
575 got_cyrillic = object_records.get_collection_record(coll, "u-cyrillic", base_dir=data_dir, roots=[])
576 got_latin = object_records.get_collection_record(coll, "u-latin", base_dir=data_dir, roots=[])
577 assert got_cyrillic["note"] == cyrillic_name, f"{coll}: FINDING -- Cyrillic homoglyph altered on read"
578 assert got_latin["note"] == latin_name, f"{coll}: FINDING -- Latin string altered on read"
579 assert got_cyrillic["note"] != got_latin["note"], (
580 f"{coll}: FINDING -- homoglyph strings collapsed to the same "
581 f"stored value."
582 )
583
584
585# =============================================================================
586# 8. DELIMITER-SAFETY PROPERTY: no multibyte UTF-8 sequence contains 0x09/0x0A
587# =============================================================================
588
589
590def test_no_multibyte_utf8_sequence_contains_tab_or_newline_byte():
591 """By construction of UTF-8, every byte in a MULTI-byte sequence (the
592 lead byte and all continuation bytes) is >= 0x80 -- a multibyte
593 sequence can never contain a byte that collides with ASCII TAB (0x09)
594 or LF (0x0A) at the byte level, so a byte-level delimiter scan could
595 never accidentally split a multibyte character mid-sequence and
596 mistake a byte inside it for a real field/row delimiter. This is a
597 property of UTF-8 itself (not of object_records' parser, which uses
598 real CSV/TSV parsing rather than a byte-level scan regardless) --
599 verified here directly, and then reinforced with an over-the-wire
600 round trip through the real storage path using exactly this file's
601 adversarial character set, to confirm the property holds for every
602 codepoint actually exercised in this file (not just a mathematical
603 argument about UTF-8 in the abstract)."""
604 probe_chars = (
605 [ZWSP, ZWNJ, ZWJ, WORD_JOINER, RLO, LRO, RLI, LRI, PDI, CYRILLIC_A]
606 + NONCHARACTERS
607 + [chr(0x0300 + i) for i in range(0x70)] # combining marks used by the zalgo test
608 + ["é", "\U0001f600", "日本"] # accented latin, emoji, CJK, for good measure
609 )
610 for ch in probe_chars:
611 encoded = ch.encode("utf-8")
612 if len(encoded) > 1:
613 assert 0x09 not in encoded, f"FINDING: U+{ord(ch):04X} encodes to bytes containing a TAB byte: {encoded!r}"
614 assert 0x0A not in encoded, f"FINDING: U+{ord(ch):04X} encodes to bytes containing a LF byte: {encoded!r}"
615
616 # Systematic sweep across the full codepoint space (skipping surrogates,
617 # which cannot be UTF-8 encoded standalone -- covered separately in
618 # section 1), stepping to keep runtime bounded while still covering
619 # every UTF-8 length class (1/2/3/4-byte encodings).
620 step = 97 # coprime-ish stride, not aligned to any power-of-two boundary
621 checked = 0
622 for cp in range(0x80, 0x110000, step):
623 if 0xD800 <= cp <= 0xDFFF:
624 continue # surrogate range: not independently UTF-8-encodable
625 encoded = chr(cp).encode("utf-8")
626 checked += 1
627 if len(encoded) > 1:
628 assert 0x09 not in encoded, f"FINDING: U+{cp:04X} encodes to bytes containing a TAB byte: {encoded!r}"
629 assert 0x0A not in encoded, f"FINDING: U+{cp:04X} encodes to bytes containing a LF byte: {encoded!r}"
630 assert checked > 10000 # sanity: the sweep actually covered a meaningful sample
631
632
633def test_delimiter_safety_holds_through_the_real_storage_path(tmp_path, monkeypatch):
634 """Reinforcement of the property above through the actual create/read
635 path: a field packed with every adversarial character from this file,
636 back-to-back with no ASCII padding between them (the adversarial case
637 for any byte-level scan), must still round-trip exactly -- and, as a
638 structural check, the physical file must contain exactly one data row
639 (a naive byte-level split on this content, if the storage layer used
640 one, would either fragment it across "rows" or corrupt the field
641 count; the real csv-based parser must not)."""
642 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
643 packed = "".join(
644 [ZWSP, ZWNJ, ZWJ, WORD_JOINER, RLO, LRO, RLI, PDI, CYRILLIC_A]
645 + NONCHARACTERS
646 + [chr(0x0300 + i) for i in range(20)]
647 )
648 for ch in packed:
649 assert ch.encode("utf-8").isascii() is False or ord(ch) < 0x80 # sanity, non-fatal
650
651 data_dir = tmp_path / "data"
652 write_schema(data_dir, "packed_classic", [ID_FIELD, TEXT_FIELD])
653 write_append_schema(data_dir, "packed_append", [ID_FIELD, TEXT_FIELD])
654
655 for coll in ("packed_classic", "packed_append"):
656 object_records.create_collection_record(
657 coll, {"id": "p1", "note": packed}, base_dir=data_dir, roots=[]
658 )
659 object_records.create_collection_record(
660 coll, {"id": "p2", "note": "sibling"}, base_dir=data_dir, roots=[]
661 )
662
663 _clear_caches()
664 all_records = object_records.read_collection_records(coll, base_dir=data_dir, roots=[])
665 assert len(all_records) == 2, (
666 f"{coll}: FINDING -- packed adversarial content changed the "
667 f"apparent row count (byte-level mis-split?). Got: {all_records!r}"
668 )
669 by_id = {r["id"]: r["note"] for r in all_records}
670 assert by_id["p1"] == packed
671 assert by_id["p2"] == "sibling"
672
673
674# =============================================================================
675# 9. UPDATE PATH -- adversarial content surviving a create -> update sequence
676# =============================================================================
677
678
679def test_update_sequence_through_zero_width_bidi_and_noncharacter_content(tmp_path, monkeypatch):
680 """A single record's field is created plain, then updated THROUGH a
681 sequence of adversarial values (zero-width-only, bidi-flipped,
682 noncharacter-laden, back to plain), asserting an exact round trip after
683 every step, both storage modes. This is the "and update" axis of the
684 probe matrix: confirms the update path (not just create) preserves
685 adversarial content exactly, including the merge-with-existing-record
686 step every update performs internally."""
687 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
688 sequence = [
689 "plain start",
690 ZWSP + ZWNJ + ZWJ,
691 f"{RLO}reversed text{RLO} trailing",
692 "".join(NONCHARACTERS),
693 "plain end",
694 ]
695
696 for storage in ("classic", "append"):
697 data_dir = tmp_path / f"data_{storage}"
698 if storage == "classic":
699 write_schema(data_dir, "seq", [ID_FIELD, TEXT_FIELD])
700 else:
701 write_append_schema(data_dir, "seq", [ID_FIELD, TEXT_FIELD])
702
703 object_records.create_collection_record(
704 "seq", {"id": "s1", "note": sequence[0]}, base_dir=data_dir, roots=[]
705 )
706 _clear_caches()
707 assert object_records.get_collection_record("seq", "s1", base_dir=data_dir, roots=[])["note"] == sequence[0]
708
709 for step_value in sequence[1:]:
710 object_records.update_collection_record(
711 "seq", "s1", {"note": step_value}, base_dir=data_dir, roots=[]
712 )
713 _clear_caches()
714 current = object_records.get_collection_record("seq", "s1", base_dir=data_dir, roots=[])
715 assert current["note"] == step_value, (
716 f"{storage}: FINDING -- update to {step_value!r} did not "
717 f"round-trip. Got: {current['note']!r}"
718 )