Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_charset_utf8_multibyte_characterization.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_charset_utf8_multibyte_characterization.py · 25.5 KB · 546 lines
1"""CHARACTERIZATION tests: does the current TSV substrate (object_records.py)
2safely hold multibyte UTF-8 content through both storage modes, with special
3attention to BYTE-OFFSET accounting?
4
5This file is pure characterization of EXISTING behavior. It does not modify
6any production module. Where a case fails or corrupts data, the assertion is
7written to state the correctness property the substrate SHOULD have, and is
8left to fail -- that failure IS the finding. Do not "fix" a failing assertion
9here by weakening it to match broken behavior.
10
11THE RISK THIS FILE TARGETS: object_records.py stores TSV via csv. In append
12mode, a byte-offset sidecar (`.records.oidx`) is maintained; _scan_append_tail
13derives each row's byte span by decoding text ONCE per scan and measuring
14consumed CHAR-slices back to bytes
15(`len(text[prev_char:cur_char].encode("utf-8"))`), and _oidx_get_row /
16_fast_record_lookup then `seek()` to those byte offsets to read a single row.
17Multibyte characters mean char-count != byte-count -- so any place that
18conflates the two, or seeks to a byte offset landing mid-multibyte-sequence,
19could corrupt a read. Separately, _check_field_sizes caps a field at
20MAX_TSV_FIELD_BYTES measured via `len(str(value).encode("utf-8"))` -- this
21must count BYTES, not characters, or a large-but-under-the-character-cap
22multibyte value could be silently mis-admitted or mis-rejected.
23
24Explicitly NOT probed here (already covered / deferred elsewhere):
25 - field_size_limit / csv oversize-field handling in general (fixed, see
26 tests/test_embedded_json_lines_characterization.py substrate fix #3).
27 - _oidx_get_row's own csv-awareness for multi-physical-line rows (fixed,
28 substrate fix #1, same file).
29 - torn-tail mid-multiline-row resurrection (substrate bug #2, deferred,
30 xfail in the same file) -- not touched here at all.
31
32Two content shapes are probed throughout: a RAW field value, and a value
33wrapped in a compact `json.dumps(...)` blob (a realistic way multibyte
34content actually arrives in a JSON-in-a-cell field) -- json.dumps by default
35does NOT escape non-ASCII (ensure_ascii=True is the default and WOULD
36\\uXXXX-escape everything into pure ASCII, defeating the point of this file),
37so this file passes ensure_ascii=False explicitly to keep raw multibyte UTF-8
38bytes in the cell, matching the module docstring's precedent in the sibling
39embedded-JSON characterization file for what "compact JSON blob" is meant to
40stress.
41"""
42
43from __future__ import annotations
44
45import json
46import unicodedata
47from pathlib import Path
48
49import pytest
50
51import object_records
52
53# Conformance tier: heavy charset characterization -- deselected from the
54# per-commit run (see pyproject 'conformance' marker). Run: pytest -m conformance
55pytestmark = pytest.mark.conformance
56
57
58ID_FIELD = {"name": "id"}
59VALUE_FIELD = {"name": "value", "type": "textarea"}
60
61
62# --- setup helpers (mirrored from test_embedded_json_lines_characterization.py) --
63
64
65def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path:
66 path = data_dir / "schemas" / f"{collection}.json"
67 path.parent.mkdir(parents=True, exist_ok=True)
68 path.write_text(json.dumps({"fields": fields}))
69 return path
70
71
72def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path:
73 path = data_dir / "schemas" / f"{collection}.json"
74 path.parent.mkdir(parents=True, exist_ok=True)
75 path.write_text(json.dumps({"fields": fields, "storage": "append"}))
76 return path
77
78
79def _clear_caches() -> None:
80 """Force the NEXT read past the warm in-process caches -- a cold
81 _RECORDS_CACHE + _OIDX_CACHE is what makes get_collection_record on an
82 append-mode file actually exercise the id->offset sidecar / _oidx_get_row
83 (see _fast_record_lookup) instead of serving from the ordinary records
84 cache, which is what this file needs to test the byte-offset path."""
85 object_records._RECORDS_CACHE.clear()
86 object_records._OIDX_CACHE.clear()
87
88
89# --- multibyte sample strings --------------------------------------------
90
91TWO_BYTE = "café résumé naïve Москва Ελληνικά שלום" # accented Latin + Cyrillic + Greek + Hebrew
92THREE_BYTE = "日本語中文 العربية देवनागरी" # CJK + Arabic + Devanagari
93FOUR_BYTE = "\U00020000\U00020001 astral CJK-B \U0001F600 emoji \U00010000" # astral CJK ext B + emoji
94MIXED = f"{TWO_BYTE} | {THREE_BYTE} | {FOUR_BYTE}"
95
96NFC_STRING = unicodedata.normalize("NFC", "é café") # precomposed é where possible
97NFD_STRING = unicodedata.normalize("NFD", "é café") # fully decomposed "e" + combining acute
98
99
100def _roundtrip_raw(data_dir: Path, collection: str, record_id: str, value: str) -> None:
101 """create -> fold read -> by-id read (cold), assert exact string equality
102 at every hop, for a RAW (non-JSON) field value."""
103 object_records.create_collection_record(
104 collection, {"id": record_id, "value": value}, base_dir=data_dir, roots=[]
105 )
106
107 _clear_caches()
108 all_records = object_records.read_collection_records(collection, base_dir=data_dir, roots=[])
109 match = next(r for r in all_records if r["id"] == record_id)
110 assert match["value"] == value, (
111 f"{collection}/{record_id}: fold read did not round-trip raw value exactly. "
112 f"Expected {value!r}, got {match['value']!r}"
113 )
114
115 _clear_caches()
116 by_id = object_records.get_collection_record(collection, record_id, base_dir=data_dir, roots=[])
117 assert by_id["value"] == value, (
118 f"{collection}/{record_id}: by-id (cold sidecar) read did not round-trip raw "
119 f"value exactly. Expected {value!r}, got {by_id['value']!r}"
120 )
121
122
123def _roundtrip_json(data_dir: Path, collection: str, record_id: str, payload) -> None:
124 """create -> fold read -> by-id read (cold), for a value that is a
125 compact json.dumps(..., ensure_ascii=False) blob containing multibyte
126 content -- json.loads back and compare structurally."""
127 blob = json.dumps(payload, ensure_ascii=False)
128 object_records.create_collection_record(
129 collection, {"id": record_id, "value": blob}, base_dir=data_dir, roots=[]
130 )
131
132 _clear_caches()
133 all_records = object_records.read_collection_records(collection, base_dir=data_dir, roots=[])
134 match = next(r for r in all_records if r["id"] == record_id)
135 assert json.loads(match["value"]) == payload, (
136 f"{collection}/{record_id}: fold read did not round-trip JSON blob. "
137 f"Expected {payload!r}, got {match['value']!r}"
138 )
139
140 _clear_caches()
141 by_id = object_records.get_collection_record(collection, record_id, base_dir=data_dir, roots=[])
142 assert json.loads(by_id["value"]) == payload, (
143 f"{collection}/{record_id}: by-id (cold sidecar) read did not round-trip "
144 f"JSON blob. Expected {payload!r}, got {by_id['value']!r}"
145 )
146
147
148# =============================================================================
149# 1. Basic multibyte width classes -- RAW field, classic AND append mode
150# =============================================================================
151
152
153@pytest.mark.parametrize(
154 "label,value",
155 [
156 ("2byte", TWO_BYTE),
157 ("3byte", THREE_BYTE),
158 ("4byte_astral", FOUR_BYTE),
159 ("mixed", MIXED),
160 ],
161)
162def test_raw_field_multibyte_round_trip_classic_and_append(tmp_path, monkeypatch, label, value):
163 """Each width class, RAW (non-JSON) field, both storage modes, cold-cache
164 by-id read to force the byte-offset sidecar path in append mode."""
165 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
166 data_dir = tmp_path / "data"
167 write_schema(data_dir, "classic_coll", [ID_FIELD, VALUE_FIELD])
168 write_append_schema(data_dir, "append_coll", [ID_FIELD, VALUE_FIELD])
169
170 _roundtrip_raw(data_dir, "classic_coll", f"rec-{label}", value)
171 _roundtrip_raw(data_dir, "append_coll", f"rec-{label}", value)
172
173
174@pytest.mark.parametrize(
175 "label,value",
176 [
177 ("2byte", TWO_BYTE),
178 ("3byte", THREE_BYTE),
179 ("4byte_astral", FOUR_BYTE),
180 ("mixed", MIXED),
181 ],
182)
183def test_json_blob_multibyte_round_trip_classic_and_append(tmp_path, monkeypatch, label, value):
184 """Same width classes, but the value is embedded inside a compact
185 json.dumps(..., ensure_ascii=False) blob -- the realistic shape for a
186 JSON-in-a-cell field carrying multibyte user content."""
187 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
188 data_dir = tmp_path / "data"
189 write_schema(data_dir, "classic_coll", [ID_FIELD, VALUE_FIELD])
190 write_append_schema(data_dir, "append_coll", [ID_FIELD, VALUE_FIELD])
191
192 payload = {"sku": f"SKU-{label}", "note": value, "nested": [value, value]}
193 _roundtrip_json(data_dir, "classic_coll", f"rec-{label}", payload)
194 _roundtrip_json(data_dir, "append_coll", f"rec-{label}", payload)
195
196
197def test_update_preserves_multibyte_raw_and_json(tmp_path, monkeypatch):
198 """update_collection_record specifically: create with ASCII, then update
199 to multibyte content, both raw and JSON-wrapped, append mode, cold-cache
200 by-id read after the update."""
201 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
202 data_dir = tmp_path / "data"
203 write_append_schema(data_dir, "append_coll", [ID_FIELD, VALUE_FIELD])
204
205 object_records.create_collection_record(
206 "append_coll", {"id": "u1", "value": "plain ascii"}, base_dir=data_dir, roots=[]
207 )
208 object_records.update_collection_record(
209 "append_coll", "u1", {"value": MIXED}, base_dir=data_dir, roots=[]
210 )
211 _clear_caches()
212 got = object_records.get_collection_record("append_coll", "u1", base_dir=data_dir, roots=[])
213 assert got["value"] == MIXED, (
214 f"update -> cold by-id read did not preserve raw multibyte value exactly. "
215 f"Expected {MIXED!r}, got {got['value']!r}"
216 )
217
218 payload = {"note": MIXED}
219 blob = json.dumps(payload, ensure_ascii=False)
220 object_records.update_collection_record(
221 "append_coll", "u1", {"value": blob}, base_dir=data_dir, roots=[]
222 )
223 _clear_caches()
224 got2 = object_records.get_collection_record("append_coll", "u1", base_dir=data_dir, roots=[])
225 assert json.loads(got2["value"]) == payload, (
226 f"update -> cold by-id read did not preserve JSON-wrapped multibyte value. "
227 f"Expected {payload!r}, got {got2['value']!r}"
228 )
229
230
231# =============================================================================
232# 2. CRITICAL: append byte-offset sidecar correctness when EARLIER rows are
233# heavily multibyte (char-offset != byte-offset for every row after them)
234# =============================================================================
235
236
237def test_offset_sidecar_survives_heavy_multibyte_content_in_earlier_rows(tmp_path, monkeypatch):
238 """THE central offset-accounting question this file exists to answer:
239 _scan_append_tail computes each row's byte span from a CHARACTER-offset
240 delta (`stream.tell()`, a csv.reader-over-io.StringIO character
241 position) re-encoded to bytes. If an EARLY record's field is packed with
242 multibyte characters, every row AFTER it starts at a byte offset that is
243 numerically far from its character offset. If the byte-offset sidecar
244 (or _oidx_get_row's seek) ever conflated the two, or mis-tracked the
245 cumulative byte cursor, later records' by-id reads (forced through the
246 cold sidecar path) would come back garbled, truncated, or attributed to
247 the wrong row.
248
249 Layout: A (heavy 3-byte CJK, ~2000 chars => ~6000 bytes), B (heavy 4-byte
250 astral, plenty of surrogate-pair-relevant code points), C/D/E (small,
251 ordinary ASCII-ish records) -- then C, D, E are each read BY ID with the
252 cache forced cold, and must come back byte-exact.
253 """
254 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
255 data_dir = tmp_path / "data"
256 write_append_schema(data_dir, "offsets", [ID_FIELD, VALUE_FIELD])
257
258 heavy_cjk = "日本語" * 700 # 2100 chars, 3 bytes/char => 6300 bytes
259 heavy_astral = "\U00020000\U0001F600" * 500 # 1000 chars, 4 bytes/char => 4000 bytes
260
261 c_value = "record C, plain and short"
262 d_value = f"record D with some multibyte: {TWO_BYTE}"
263 e_value = f"record E with more: {THREE_BYTE} {FOUR_BYTE}"
264
265 object_records.create_collection_record(
266 "offsets", {"id": "A", "value": heavy_cjk}, base_dir=data_dir, roots=[]
267 )
268 object_records.create_collection_record(
269 "offsets", {"id": "B", "value": heavy_astral}, base_dir=data_dir, roots=[]
270 )
271 object_records.create_collection_record(
272 "offsets", {"id": "C", "value": c_value}, base_dir=data_dir, roots=[]
273 )
274 object_records.create_collection_record(
275 "offsets", {"id": "D", "value": d_value}, base_dir=data_dir, roots=[]
276 )
277 object_records.create_collection_record(
278 "offsets", {"id": "E", "value": e_value}, base_dir=data_dir, roots=[]
279 )
280
281 for record_id, expected in (("C", c_value), ("D", d_value), ("E", e_value)):
282 _clear_caches()
283 got = object_records.get_collection_record("offsets", record_id, base_dir=data_dir, roots=[])
284 assert got["value"] == expected, (
285 f"FINDING: cold by-id read of '{record_id}' (after multibyte-heavy "
286 f"earlier rows A/B) did not round-trip exactly -- the byte-offset "
287 f"sidecar's accounting broke under multibyte content in preceding "
288 f"rows. Expected {expected!r}, got {got['value']!r}"
289 )
290
291 # Also confirm A and B (the heavy rows themselves) round-trip.
292 for record_id, expected in (("A", heavy_cjk), ("B", heavy_astral)):
293 _clear_caches()
294 got = object_records.get_collection_record("offsets", record_id, base_dir=data_dir, roots=[])
295 assert got["value"] == expected, (
296 f"FINDING: cold by-id read of the heavy-multibyte record '{record_id}' "
297 f"itself did not round-trip exactly."
298 )
299
300 # Full fold read too, for good measure.
301 _clear_caches()
302 all_records = {r["id"]: r["value"] for r in object_records.read_collection_records(
303 "offsets", base_dir=data_dir, roots=[]
304 )}
305 assert all_records == {
306 "A": heavy_cjk, "B": heavy_astral, "C": c_value, "D": d_value, "E": e_value,
307 }
308
309
310def test_offset_sidecar_survives_after_update_of_early_multibyte_row(tmp_path, monkeypatch):
311 """Same crux as above, but the multibyte-heavy row is UPDATED (superseded
312 by a second, different-length physical row) after later records already
313 exist -- confirming the sidecar's catch-up scan correctly advances past
314 the new physical row's byte span, not the old one's, before it reaches
315 later rows' offsets again."""
316 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
317 data_dir = tmp_path / "data"
318 write_append_schema(data_dir, "offsets2", [ID_FIELD, VALUE_FIELD])
319
320 a_value_v1 = "short ascii to start"
321 c_value = "record C"
322 d_value = "record D"
323
324 object_records.create_collection_record(
325 "offsets2", {"id": "A", "value": a_value_v1}, base_dir=data_dir, roots=[]
326 )
327 object_records.create_collection_record(
328 "offsets2", {"id": "C", "value": c_value}, base_dir=data_dir, roots=[]
329 )
330 object_records.create_collection_record(
331 "offsets2", {"id": "D", "value": d_value}, base_dir=data_dir, roots=[]
332 )
333
334 # Now append a NEW physical row for A, much larger and heavily multibyte,
335 # AFTER C and D's rows already exist on disk.
336 a_value_v2 = "日本語中文" * 900 + FOUR_BYTE # large, multibyte-heavy
337 object_records.update_collection_record(
338 "offsets2", "A", {"value": a_value_v2}, base_dir=data_dir, roots=[]
339 )
340
341 for record_id, expected in (("C", c_value), ("D", d_value), ("A", a_value_v2)):
342 _clear_caches()
343 got = object_records.get_collection_record("offsets2", record_id, base_dir=data_dir, roots=[])
344 assert got["value"] == expected, (
345 f"FINDING: cold by-id read of '{record_id}' after updating A to a "
346 f"large multibyte value did not round-trip. Expected {expected!r}, "
347 f"got {got['value']!r}"
348 )
349
350
351# =============================================================================
352# 3. LARGE multibyte field -- _check_field_sizes must count BYTES not chars
353# =============================================================================
354
355
356def test_large_multibyte_field_counts_bytes_not_chars(tmp_path):
357 """100,000 CJK characters (3 bytes each => ~300 KB) is comfortably under
358 MAX_TSV_FIELD_BYTES (16 MiB) whether measured in chars or bytes, so this
359 alone can't distinguish char-counting from byte-counting bugs in
360 _check_field_sizes. The real probe: a value whose CHARACTER count is
361 small enough that a char-counting bug would wrongly ADMIT it past the
362 byte cap, but whose BYTE count is actually over MAX_TSV_FIELD_BYTES --
363 if _check_field_sizes measures bytes correctly (as its source says:
364 `len(str(value).encode("utf-8"))`), this must be REJECTED.
365 """
366 data_dir = tmp_path / "data"
367 write_schema(data_dir, "classic_coll", [ID_FIELD, VALUE_FIELD])
368 write_append_schema(data_dir, "append_coll", [ID_FIELD, VALUE_FIELD])
369
370 # Comfortably-under-cap large multibyte value: round-trip check.
371 big_cjk = "日本語" * 33334 # 100,002 chars, 3 bytes/char = 300,006 bytes
372 big_bytes = len(big_cjk.encode("utf-8"))
373 assert big_bytes < object_records.MAX_TSV_FIELD_BYTES
374 for collection in ("classic_coll", "append_coll"):
375 object_records.create_collection_record(
376 collection, {"id": "big-cjk", "value": big_cjk}, base_dir=data_dir, roots=[]
377 )
378 _clear_caches()
379 got = object_records.get_collection_record(collection, "big-cjk", base_dir=data_dir, roots=[])
380 assert got["value"] == big_cjk, (
381 f"{collection}: large (~{big_bytes} byte) multibyte field did not round-trip"
382 )
383
384 # Char count UNDER MAX_TSV_FIELD_BYTES, but byte count OVER it: 4-byte
385 # astral characters, ~4.1M of them => ~4.1M chars (well under the
386 # 16,777,216 char/byte cap numerically) but ~16.4M bytes (over the cap).
387 # If _check_field_sizes ever counted `len(value)` (chars) instead of
388 # `len(value.encode("utf-8"))` (bytes), this value's admit/reject
389 # decision would flip.
390 astral_char = "\U00020000" # 4 bytes in UTF-8, 1 Python char
391 n_chars = (object_records.MAX_TSV_FIELD_BYTES // 4) + 100 # chars: just over cap/4
392 over_by_bytes_under_by_chars = astral_char * n_chars
393 char_len = len(over_by_bytes_under_by_chars)
394 byte_len = len(over_by_bytes_under_by_chars.encode("utf-8"))
395 assert char_len < object_records.MAX_TSV_FIELD_BYTES, "sanity: char count must be under the cap"
396 assert byte_len > object_records.MAX_TSV_FIELD_BYTES, "sanity: byte count must be over the cap"
397
398 for collection in ("classic_coll", "append_coll"):
399 with pytest.raises(object_records.InvalidRecordPayloadError):
400 object_records.create_collection_record(
401 collection,
402 {"id": "byte-over-char-under", "value": over_by_bytes_under_by_chars},
403 base_dir=data_dir,
404 roots=[],
405 )
406
407
408# =============================================================================
409# 4. BOM (U+FEFF)
410# =============================================================================
411
412
413@pytest.mark.parametrize(
414 "label,value",
415 [
416 ("start", "hello world"),
417 ("middle", "hello  world"),
418 ("end", "hello world"),
419 ("only", ""),
420 ],
421)
422def test_bom_survives_raw_and_json_append_and_classic(tmp_path, monkeypatch, label, value):
423 """U+FEFF (BOM / zero-width no-break space) at start, middle, end of a
424 value, and a value that IS only a BOM -- must not be stripped, moved, or
425 misinterpreted as a stream-level encoding marker by the csv/text layer
426 (object_records opens files with explicit encoding="utf-8", not
427 "utf-8-sig", so a leading U+FEFF in a CELL VALUE should be indistinguishable
428 from any other codepoint to this layer)."""
429 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
430 data_dir = tmp_path / "data"
431 write_schema(data_dir, "classic_coll", [ID_FIELD, VALUE_FIELD])
432 write_append_schema(data_dir, "append_coll", [ID_FIELD, VALUE_FIELD])
433
434 _roundtrip_raw(data_dir, "classic_coll", f"bom-{label}", value)
435 _roundtrip_raw(data_dir, "append_coll", f"bom-{label}", value)
436
437 payload = {"note": value}
438 _roundtrip_json(data_dir, "classic_coll", f"bom-json-{label}", payload)
439 _roundtrip_json(data_dir, "append_coll", f"bom-json-{label}", payload)
440
441
442# =============================================================================
443# 5. Combining sequences -- NFC vs NFD must round-trip byte-exact (no
444# silent normalization by the store)
445# =============================================================================
446
447
448def test_nfc_vs_nfd_are_distinct_and_survive_byte_exact(tmp_path, monkeypatch):
449 """"e" + U+0301 (combining acute accent, NFD) and precomposed "é"
450 (U+00E9, NFC) are canonically equivalent under Unicode normalization but
451 are DIFFERENT byte sequences. A storage layer must never silently
452 normalize one to the other -- this test asserts the two forms remain
453 distinguishable (their raw bytes differ) AND each survives its own
454 round trip unchanged, both raw and JSON-wrapped, both storage modes,
455 cold by-id read."""
456 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
457 data_dir = tmp_path / "data"
458 write_schema(data_dir, "classic_coll", [ID_FIELD, VALUE_FIELD])
459 write_append_schema(data_dir, "append_coll", [ID_FIELD, VALUE_FIELD])
460
461 # Sanity: these two Python strings really are different byte sequences,
462 # not accidentally identical (confirms this test is exercising what it
463 # claims to).
464 assert NFC_STRING != NFD_STRING
465 assert NFC_STRING.encode("utf-8") != NFD_STRING.encode("utf-8")
466 assert unicodedata.normalize("NFC", NFD_STRING) == NFC_STRING # canonically equivalent
467
468 for collection in ("classic_coll", "append_coll"):
469 _roundtrip_raw(data_dir, collection, f"nfc-{collection}", NFC_STRING)
470 _roundtrip_raw(data_dir, collection, f"nfd-{collection}", NFD_STRING)
471
472 # The store must not have coalesced the two into the same bytes.
473 _clear_caches()
474 nfc_got = object_records.get_collection_record(
475 collection, f"nfc-{collection}", base_dir=data_dir, roots=[]
476 )
477 _clear_caches()
478 nfd_got = object_records.get_collection_record(
479 collection, f"nfd-{collection}", base_dir=data_dir, roots=[]
480 )
481 assert nfc_got["value"] != nfd_got["value"] or NFC_STRING == NFD_STRING, (
482 f"{collection}: NFC and NFD forms were silently normalized to the "
483 f"same stored value -- store must preserve the exact form given. "
484 f"nfc={nfc_got['value']!r} nfd={nfd_got['value']!r}"
485 )
486 assert nfc_got["value"] == NFC_STRING
487 assert nfd_got["value"] == NFD_STRING
488
489 payload_nfc = {"note": NFC_STRING}
490 payload_nfd = {"note": NFD_STRING}
491 _roundtrip_json(data_dir, collection, f"nfc-json-{collection}", payload_nfc)
492 _roundtrip_json(data_dir, collection, f"nfd-json-{collection}", payload_nfd)
493
494
495# =============================================================================
496# 6. A multibyte character's UTF-8 bytes straddling a naive row-boundary split
497# =============================================================================
498
499
500def test_multibyte_char_straddling_row_boundary_bytes(tmp_path, monkeypatch):
501 """Construct rows so that a multibyte character's byte span sits right
502 where a naive fixed-width or byte-count-based row splitter (as opposed
503 to the real csv/text-based row delimiting object_records.py actually
504 uses) would be tempted to cut -- e.g. a value engineered so the
505 cumulative byte offset right before/after it lands on suspicious
506 boundaries (mod 2, mod 3, mod 4 relative to the row start), stressing
507 that ROW delimiting is happening on '\\n'/csv structure and never on a
508 raw byte count that could bisect a multibyte sequence.
509
510 Layout: several small rows, each ending in a different multibyte
511 character (2/3/4-byte), each row's total byte length deliberately
512 NOT a clean multiple of the multibyte char's own width, so the trailing
513 character's bytes don't line up neatly with the row's total byte count
514 -- if row-splitting ever degraded to a byte-oriented (not char/csv-
515 oriented) scan, misalignment here would be likely to surface as
516 corruption on the NEXT row rather than silently working by accident."""
517 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
518 data_dir = tmp_path / "data"
519 write_append_schema(data_dir, "straddle", [ID_FIELD, VALUE_FIELD])
520
521 rows = {
522 "R1": "x" * 1 + "café", # ends in 2-byte char (é), odd prefix length
523 "R2": "xx" * 1 + "日本語", # ends in 3-byte chars, even prefix length
524 "R3": "x" * 3 + "\U00020000", # ends in 4-byte astral char, odd prefix length
525 "R4": "xxxx" + "Ω" + "x", # 2-byte char in the MIDDLE, ascii after
526 "R5": "plain ascii tail",
527 }
528 for record_id, value in rows.items():
529 object_records.create_collection_record(
530 "straddle", {"id": record_id, "value": value}, base_dir=data_dir, roots=[]
531 )
532
533 for record_id, expected in rows.items():
534 _clear_caches()
535 got = object_records.get_collection_record("straddle", record_id, base_dir=data_dir, roots=[])
536 assert got["value"] == expected, (
537 f"FINDING: '{record_id}' did not round-trip byte-exact -- possible "
538 f"row-boundary/byte-offset misalignment around a multibyte "
539 f"character. Expected {expected!r}, got {got['value']!r}"
540 )
541
542 _clear_caches()
543 all_records = {r["id"]: r["value"] for r in object_records.read_collection_records(
544 "straddle", base_dir=data_dir, roots=[]
545 )}
546 assert all_records == rows