askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_charset_utf8_multibyte_characterization.py · 25.5 KB · 546 lines
| 1 | """CHARACTERIZATION tests: does the current TSV substrate (object_records.py) |
| 2 | safely hold multibyte UTF-8 content through both storage modes, with special |
| 3 | attention to BYTE-OFFSET accounting? |
| 4 | |
| 5 | This file is pure characterization of EXISTING behavior. It does not modify |
| 6 | any production module. Where a case fails or corrupts data, the assertion is |
| 7 | written to state the correctness property the substrate SHOULD have, and is |
| 8 | left to fail -- that failure IS the finding. Do not "fix" a failing assertion |
| 9 | here by weakening it to match broken behavior. |
| 10 | |
| 11 | THE RISK THIS FILE TARGETS: object_records.py stores TSV via csv. In append |
| 12 | mode, a byte-offset sidecar (`.records.oidx`) is maintained; _scan_append_tail |
| 13 | derives each row's byte span by decoding text ONCE per scan and measuring |
| 14 | consumed CHAR-slices back to bytes |
| 15 | (`len(text[prev_char:cur_char].encode("utf-8"))`), and _oidx_get_row / |
| 16 | _fast_record_lookup then `seek()` to those byte offsets to read a single row. |
| 17 | Multibyte characters mean char-count != byte-count -- so any place that |
| 18 | conflates the two, or seeks to a byte offset landing mid-multibyte-sequence, |
| 19 | could corrupt a read. Separately, _check_field_sizes caps a field at |
| 20 | MAX_TSV_FIELD_BYTES measured via `len(str(value).encode("utf-8"))` -- this |
| 21 | must count BYTES, not characters, or a large-but-under-the-character-cap |
| 22 | multibyte value could be silently mis-admitted or mis-rejected. |
| 23 | |
| 24 | Explicitly NOT probed here (already covered / deferred elsewhere): |
| 25 | - field_size_limit / csv oversize-field handling in general (fixed, see |
| 26 | tests/test_embedded_json_lines_characterization.py substrate fix #3). |
| 27 | - _oidx_get_row's own csv-awareness for multi-physical-line rows (fixed, |
| 28 | substrate fix #1, same file). |
| 29 | - torn-tail mid-multiline-row resurrection (substrate bug #2, deferred, |
| 30 | xfail in the same file) -- not touched here at all. |
| 31 | |
| 32 | Two content shapes are probed throughout: a RAW field value, and a value |
| 33 | wrapped in a compact `json.dumps(...)` blob (a realistic way multibyte |
| 34 | content actually arrives in a JSON-in-a-cell field) -- json.dumps by default |
| 35 | does NOT escape non-ASCII (ensure_ascii=True is the default and WOULD |
| 36 | \\uXXXX-escape everything into pure ASCII, defeating the point of this file), |
| 37 | so this file passes ensure_ascii=False explicitly to keep raw multibyte UTF-8 |
| 38 | bytes in the cell, matching the module docstring's precedent in the sibling |
| 39 | embedded-JSON characterization file for what "compact JSON blob" is meant to |
| 40 | stress. |
| 41 | """ |
| 42 | |
| 43 | from __future__ import annotations |
| 44 | |
| 45 | import json |
| 46 | import unicodedata |
| 47 | from pathlib import Path |
| 48 | |
| 49 | import pytest |
| 50 | |
| 51 | import object_records |
| 52 | |
| 53 | # Conformance tier: heavy charset characterization -- deselected from the |
| 54 | # per-commit run (see pyproject 'conformance' marker). Run: pytest -m conformance |
| 55 | pytestmark = pytest.mark.conformance |
| 56 | |
| 57 | |
| 58 | ID_FIELD = {"name": "id"} |
| 59 | VALUE_FIELD = {"name": "value", "type": "textarea"} |
| 60 | |
| 61 | |
| 62 | # --- setup helpers (mirrored from test_embedded_json_lines_characterization.py) -- |
| 63 | |
| 64 | |
| 65 | def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 66 | path = data_dir / "schemas" / f"{collection}.json" |
| 67 | path.parent.mkdir(parents=True, exist_ok=True) |
| 68 | path.write_text(json.dumps({"fields": fields})) |
| 69 | return path |
| 70 | |
| 71 | |
| 72 | def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 73 | path = data_dir / "schemas" / f"{collection}.json" |
| 74 | path.parent.mkdir(parents=True, exist_ok=True) |
| 75 | path.write_text(json.dumps({"fields": fields, "storage": "append"})) |
| 76 | return path |
| 77 | |
| 78 | |
| 79 | def _clear_caches() -> None: |
| 80 | """Force the NEXT read past the warm in-process caches -- a cold |
| 81 | _RECORDS_CACHE + _OIDX_CACHE is what makes get_collection_record on an |
| 82 | append-mode file actually exercise the id->offset sidecar / _oidx_get_row |
| 83 | (see _fast_record_lookup) instead of serving from the ordinary records |
| 84 | cache, which is what this file needs to test the byte-offset path.""" |
| 85 | object_records._RECORDS_CACHE.clear() |
| 86 | object_records._OIDX_CACHE.clear() |
| 87 | |
| 88 | |
| 89 | # --- multibyte sample strings -------------------------------------------- |
| 90 | |
| 91 | TWO_BYTE = "café résumé naïve Москва Ελληνικά שלום" # accented Latin + Cyrillic + Greek + Hebrew |
| 92 | THREE_BYTE = "日本語中文 العربية देवनागरी" # CJK + Arabic + Devanagari |
| 93 | FOUR_BYTE = "\U00020000\U00020001 astral CJK-B \U0001F600 emoji \U00010000" # astral CJK ext B + emoji |
| 94 | MIXED = f"{TWO_BYTE} | {THREE_BYTE} | {FOUR_BYTE}" |
| 95 | |
| 96 | NFC_STRING = unicodedata.normalize("NFC", "é café") # precomposed é where possible |
| 97 | NFD_STRING = unicodedata.normalize("NFD", "é café") # fully decomposed "e" + combining acute |
| 98 | |
| 99 | |
| 100 | def _roundtrip_raw(data_dir: Path, collection: str, record_id: str, value: str) -> None: |
| 101 | """create -> fold read -> by-id read (cold), assert exact string equality |
| 102 | at every hop, for a RAW (non-JSON) field value.""" |
| 103 | object_records.create_collection_record( |
| 104 | collection, {"id": record_id, "value": value}, base_dir=data_dir, roots=[] |
| 105 | ) |
| 106 | |
| 107 | _clear_caches() |
| 108 | all_records = object_records.read_collection_records(collection, base_dir=data_dir, roots=[]) |
| 109 | match = next(r for r in all_records if r["id"] == record_id) |
| 110 | assert match["value"] == value, ( |
| 111 | f"{collection}/{record_id}: fold read did not round-trip raw value exactly. " |
| 112 | f"Expected {value!r}, got {match['value']!r}" |
| 113 | ) |
| 114 | |
| 115 | _clear_caches() |
| 116 | by_id = object_records.get_collection_record(collection, record_id, base_dir=data_dir, roots=[]) |
| 117 | assert by_id["value"] == value, ( |
| 118 | f"{collection}/{record_id}: by-id (cold sidecar) read did not round-trip raw " |
| 119 | f"value exactly. Expected {value!r}, got {by_id['value']!r}" |
| 120 | ) |
| 121 | |
| 122 | |
| 123 | def _roundtrip_json(data_dir: Path, collection: str, record_id: str, payload) -> None: |
| 124 | """create -> fold read -> by-id read (cold), for a value that is a |
| 125 | compact json.dumps(..., ensure_ascii=False) blob containing multibyte |
| 126 | content -- json.loads back and compare structurally.""" |
| 127 | blob = json.dumps(payload, ensure_ascii=False) |
| 128 | object_records.create_collection_record( |
| 129 | collection, {"id": record_id, "value": blob}, base_dir=data_dir, roots=[] |
| 130 | ) |
| 131 | |
| 132 | _clear_caches() |
| 133 | all_records = object_records.read_collection_records(collection, base_dir=data_dir, roots=[]) |
| 134 | match = next(r for r in all_records if r["id"] == record_id) |
| 135 | assert json.loads(match["value"]) == payload, ( |
| 136 | f"{collection}/{record_id}: fold read did not round-trip JSON blob. " |
| 137 | f"Expected {payload!r}, got {match['value']!r}" |
| 138 | ) |
| 139 | |
| 140 | _clear_caches() |
| 141 | by_id = object_records.get_collection_record(collection, record_id, base_dir=data_dir, roots=[]) |
| 142 | assert json.loads(by_id["value"]) == payload, ( |
| 143 | f"{collection}/{record_id}: by-id (cold sidecar) read did not round-trip " |
| 144 | f"JSON blob. Expected {payload!r}, got {by_id['value']!r}" |
| 145 | ) |
| 146 | |
| 147 | |
| 148 | # ============================================================================= |
| 149 | # 1. Basic multibyte width classes -- RAW field, classic AND append mode |
| 150 | # ============================================================================= |
| 151 | |
| 152 | |
| 153 | @pytest.mark.parametrize( |
| 154 | "label,value", |
| 155 | [ |
| 156 | ("2byte", TWO_BYTE), |
| 157 | ("3byte", THREE_BYTE), |
| 158 | ("4byte_astral", FOUR_BYTE), |
| 159 | ("mixed", MIXED), |
| 160 | ], |
| 161 | ) |
| 162 | def test_raw_field_multibyte_round_trip_classic_and_append(tmp_path, monkeypatch, label, value): |
| 163 | """Each width class, RAW (non-JSON) field, both storage modes, cold-cache |
| 164 | by-id read to force the byte-offset sidecar path in append mode.""" |
| 165 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 166 | data_dir = tmp_path / "data" |
| 167 | write_schema(data_dir, "classic_coll", [ID_FIELD, VALUE_FIELD]) |
| 168 | write_append_schema(data_dir, "append_coll", [ID_FIELD, VALUE_FIELD]) |
| 169 | |
| 170 | _roundtrip_raw(data_dir, "classic_coll", f"rec-{label}", value) |
| 171 | _roundtrip_raw(data_dir, "append_coll", f"rec-{label}", value) |
| 172 | |
| 173 | |
| 174 | @pytest.mark.parametrize( |
| 175 | "label,value", |
| 176 | [ |
| 177 | ("2byte", TWO_BYTE), |
| 178 | ("3byte", THREE_BYTE), |
| 179 | ("4byte_astral", FOUR_BYTE), |
| 180 | ("mixed", MIXED), |
| 181 | ], |
| 182 | ) |
| 183 | def test_json_blob_multibyte_round_trip_classic_and_append(tmp_path, monkeypatch, label, value): |
| 184 | """Same width classes, but the value is embedded inside a compact |
| 185 | json.dumps(..., ensure_ascii=False) blob -- the realistic shape for a |
| 186 | JSON-in-a-cell field carrying multibyte user content.""" |
| 187 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 188 | data_dir = tmp_path / "data" |
| 189 | write_schema(data_dir, "classic_coll", [ID_FIELD, VALUE_FIELD]) |
| 190 | write_append_schema(data_dir, "append_coll", [ID_FIELD, VALUE_FIELD]) |
| 191 | |
| 192 | payload = {"sku": f"SKU-{label}", "note": value, "nested": [value, value]} |
| 193 | _roundtrip_json(data_dir, "classic_coll", f"rec-{label}", payload) |
| 194 | _roundtrip_json(data_dir, "append_coll", f"rec-{label}", payload) |
| 195 | |
| 196 | |
| 197 | def test_update_preserves_multibyte_raw_and_json(tmp_path, monkeypatch): |
| 198 | """update_collection_record specifically: create with ASCII, then update |
| 199 | to multibyte content, both raw and JSON-wrapped, append mode, cold-cache |
| 200 | by-id read after the update.""" |
| 201 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 202 | data_dir = tmp_path / "data" |
| 203 | write_append_schema(data_dir, "append_coll", [ID_FIELD, VALUE_FIELD]) |
| 204 | |
| 205 | object_records.create_collection_record( |
| 206 | "append_coll", {"id": "u1", "value": "plain ascii"}, base_dir=data_dir, roots=[] |
| 207 | ) |
| 208 | object_records.update_collection_record( |
| 209 | "append_coll", "u1", {"value": MIXED}, base_dir=data_dir, roots=[] |
| 210 | ) |
| 211 | _clear_caches() |
| 212 | got = object_records.get_collection_record("append_coll", "u1", base_dir=data_dir, roots=[]) |
| 213 | assert got["value"] == MIXED, ( |
| 214 | f"update -> cold by-id read did not preserve raw multibyte value exactly. " |
| 215 | f"Expected {MIXED!r}, got {got['value']!r}" |
| 216 | ) |
| 217 | |
| 218 | payload = {"note": MIXED} |
| 219 | blob = json.dumps(payload, ensure_ascii=False) |
| 220 | object_records.update_collection_record( |
| 221 | "append_coll", "u1", {"value": blob}, base_dir=data_dir, roots=[] |
| 222 | ) |
| 223 | _clear_caches() |
| 224 | got2 = object_records.get_collection_record("append_coll", "u1", base_dir=data_dir, roots=[]) |
| 225 | assert json.loads(got2["value"]) == payload, ( |
| 226 | f"update -> cold by-id read did not preserve JSON-wrapped multibyte value. " |
| 227 | f"Expected {payload!r}, got {got2['value']!r}" |
| 228 | ) |
| 229 | |
| 230 | |
| 231 | # ============================================================================= |
| 232 | # 2. CRITICAL: append byte-offset sidecar correctness when EARLIER rows are |
| 233 | # heavily multibyte (char-offset != byte-offset for every row after them) |
| 234 | # ============================================================================= |
| 235 | |
| 236 | |
| 237 | def test_offset_sidecar_survives_heavy_multibyte_content_in_earlier_rows(tmp_path, monkeypatch): |
| 238 | """THE central offset-accounting question this file exists to answer: |
| 239 | _scan_append_tail computes each row's byte span from a CHARACTER-offset |
| 240 | delta (`stream.tell()`, a csv.reader-over-io.StringIO character |
| 241 | position) re-encoded to bytes. If an EARLY record's field is packed with |
| 242 | multibyte characters, every row AFTER it starts at a byte offset that is |
| 243 | numerically far from its character offset. If the byte-offset sidecar |
| 244 | (or _oidx_get_row's seek) ever conflated the two, or mis-tracked the |
| 245 | cumulative byte cursor, later records' by-id reads (forced through the |
| 246 | cold sidecar path) would come back garbled, truncated, or attributed to |
| 247 | the wrong row. |
| 248 | |
| 249 | Layout: A (heavy 3-byte CJK, ~2000 chars => ~6000 bytes), B (heavy 4-byte |
| 250 | astral, plenty of surrogate-pair-relevant code points), C/D/E (small, |
| 251 | ordinary ASCII-ish records) -- then C, D, E are each read BY ID with the |
| 252 | cache forced cold, and must come back byte-exact. |
| 253 | """ |
| 254 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 255 | data_dir = tmp_path / "data" |
| 256 | write_append_schema(data_dir, "offsets", [ID_FIELD, VALUE_FIELD]) |
| 257 | |
| 258 | heavy_cjk = "日本語" * 700 # 2100 chars, 3 bytes/char => 6300 bytes |
| 259 | heavy_astral = "\U00020000\U0001F600" * 500 # 1000 chars, 4 bytes/char => 4000 bytes |
| 260 | |
| 261 | c_value = "record C, plain and short" |
| 262 | d_value = f"record D with some multibyte: {TWO_BYTE}" |
| 263 | e_value = f"record E with more: {THREE_BYTE} {FOUR_BYTE}" |
| 264 | |
| 265 | object_records.create_collection_record( |
| 266 | "offsets", {"id": "A", "value": heavy_cjk}, base_dir=data_dir, roots=[] |
| 267 | ) |
| 268 | object_records.create_collection_record( |
| 269 | "offsets", {"id": "B", "value": heavy_astral}, base_dir=data_dir, roots=[] |
| 270 | ) |
| 271 | object_records.create_collection_record( |
| 272 | "offsets", {"id": "C", "value": c_value}, base_dir=data_dir, roots=[] |
| 273 | ) |
| 274 | object_records.create_collection_record( |
| 275 | "offsets", {"id": "D", "value": d_value}, base_dir=data_dir, roots=[] |
| 276 | ) |
| 277 | object_records.create_collection_record( |
| 278 | "offsets", {"id": "E", "value": e_value}, base_dir=data_dir, roots=[] |
| 279 | ) |
| 280 | |
| 281 | for record_id, expected in (("C", c_value), ("D", d_value), ("E", e_value)): |
| 282 | _clear_caches() |
| 283 | got = object_records.get_collection_record("offsets", record_id, base_dir=data_dir, roots=[]) |
| 284 | assert got["value"] == expected, ( |
| 285 | f"FINDING: cold by-id read of '{record_id}' (after multibyte-heavy " |
| 286 | f"earlier rows A/B) did not round-trip exactly -- the byte-offset " |
| 287 | f"sidecar's accounting broke under multibyte content in preceding " |
| 288 | f"rows. Expected {expected!r}, got {got['value']!r}" |
| 289 | ) |
| 290 | |
| 291 | # Also confirm A and B (the heavy rows themselves) round-trip. |
| 292 | for record_id, expected in (("A", heavy_cjk), ("B", heavy_astral)): |
| 293 | _clear_caches() |
| 294 | got = object_records.get_collection_record("offsets", record_id, base_dir=data_dir, roots=[]) |
| 295 | assert got["value"] == expected, ( |
| 296 | f"FINDING: cold by-id read of the heavy-multibyte record '{record_id}' " |
| 297 | f"itself did not round-trip exactly." |
| 298 | ) |
| 299 | |
| 300 | # Full fold read too, for good measure. |
| 301 | _clear_caches() |
| 302 | all_records = {r["id"]: r["value"] for r in object_records.read_collection_records( |
| 303 | "offsets", base_dir=data_dir, roots=[] |
| 304 | )} |
| 305 | assert all_records == { |
| 306 | "A": heavy_cjk, "B": heavy_astral, "C": c_value, "D": d_value, "E": e_value, |
| 307 | } |
| 308 | |
| 309 | |
| 310 | def test_offset_sidecar_survives_after_update_of_early_multibyte_row(tmp_path, monkeypatch): |
| 311 | """Same crux as above, but the multibyte-heavy row is UPDATED (superseded |
| 312 | by a second, different-length physical row) after later records already |
| 313 | exist -- confirming the sidecar's catch-up scan correctly advances past |
| 314 | the new physical row's byte span, not the old one's, before it reaches |
| 315 | later rows' offsets again.""" |
| 316 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 317 | data_dir = tmp_path / "data" |
| 318 | write_append_schema(data_dir, "offsets2", [ID_FIELD, VALUE_FIELD]) |
| 319 | |
| 320 | a_value_v1 = "short ascii to start" |
| 321 | c_value = "record C" |
| 322 | d_value = "record D" |
| 323 | |
| 324 | object_records.create_collection_record( |
| 325 | "offsets2", {"id": "A", "value": a_value_v1}, base_dir=data_dir, roots=[] |
| 326 | ) |
| 327 | object_records.create_collection_record( |
| 328 | "offsets2", {"id": "C", "value": c_value}, base_dir=data_dir, roots=[] |
| 329 | ) |
| 330 | object_records.create_collection_record( |
| 331 | "offsets2", {"id": "D", "value": d_value}, base_dir=data_dir, roots=[] |
| 332 | ) |
| 333 | |
| 334 | # Now append a NEW physical row for A, much larger and heavily multibyte, |
| 335 | # AFTER C and D's rows already exist on disk. |
| 336 | a_value_v2 = "日本語中文" * 900 + FOUR_BYTE # large, multibyte-heavy |
| 337 | object_records.update_collection_record( |
| 338 | "offsets2", "A", {"value": a_value_v2}, base_dir=data_dir, roots=[] |
| 339 | ) |
| 340 | |
| 341 | for record_id, expected in (("C", c_value), ("D", d_value), ("A", a_value_v2)): |
| 342 | _clear_caches() |
| 343 | got = object_records.get_collection_record("offsets2", record_id, base_dir=data_dir, roots=[]) |
| 344 | assert got["value"] == expected, ( |
| 345 | f"FINDING: cold by-id read of '{record_id}' after updating A to a " |
| 346 | f"large multibyte value did not round-trip. Expected {expected!r}, " |
| 347 | f"got {got['value']!r}" |
| 348 | ) |
| 349 | |
| 350 | |
| 351 | # ============================================================================= |
| 352 | # 3. LARGE multibyte field -- _check_field_sizes must count BYTES not chars |
| 353 | # ============================================================================= |
| 354 | |
| 355 | |
| 356 | def test_large_multibyte_field_counts_bytes_not_chars(tmp_path): |
| 357 | """100,000 CJK characters (3 bytes each => ~300 KB) is comfortably under |
| 358 | MAX_TSV_FIELD_BYTES (16 MiB) whether measured in chars or bytes, so this |
| 359 | alone can't distinguish char-counting from byte-counting bugs in |
| 360 | _check_field_sizes. The real probe: a value whose CHARACTER count is |
| 361 | small enough that a char-counting bug would wrongly ADMIT it past the |
| 362 | byte cap, but whose BYTE count is actually over MAX_TSV_FIELD_BYTES -- |
| 363 | if _check_field_sizes measures bytes correctly (as its source says: |
| 364 | `len(str(value).encode("utf-8"))`), this must be REJECTED. |
| 365 | """ |
| 366 | data_dir = tmp_path / "data" |
| 367 | write_schema(data_dir, "classic_coll", [ID_FIELD, VALUE_FIELD]) |
| 368 | write_append_schema(data_dir, "append_coll", [ID_FIELD, VALUE_FIELD]) |
| 369 | |
| 370 | # Comfortably-under-cap large multibyte value: round-trip check. |
| 371 | big_cjk = "日本語" * 33334 # 100,002 chars, 3 bytes/char = 300,006 bytes |
| 372 | big_bytes = len(big_cjk.encode("utf-8")) |
| 373 | assert big_bytes < object_records.MAX_TSV_FIELD_BYTES |
| 374 | for collection in ("classic_coll", "append_coll"): |
| 375 | object_records.create_collection_record( |
| 376 | collection, {"id": "big-cjk", "value": big_cjk}, base_dir=data_dir, roots=[] |
| 377 | ) |
| 378 | _clear_caches() |
| 379 | got = object_records.get_collection_record(collection, "big-cjk", base_dir=data_dir, roots=[]) |
| 380 | assert got["value"] == big_cjk, ( |
| 381 | f"{collection}: large (~{big_bytes} byte) multibyte field did not round-trip" |
| 382 | ) |
| 383 | |
| 384 | # Char count UNDER MAX_TSV_FIELD_BYTES, but byte count OVER it: 4-byte |
| 385 | # astral characters, ~4.1M of them => ~4.1M chars (well under the |
| 386 | # 16,777,216 char/byte cap numerically) but ~16.4M bytes (over the cap). |
| 387 | # If _check_field_sizes ever counted `len(value)` (chars) instead of |
| 388 | # `len(value.encode("utf-8"))` (bytes), this value's admit/reject |
| 389 | # decision would flip. |
| 390 | astral_char = "\U00020000" # 4 bytes in UTF-8, 1 Python char |
| 391 | n_chars = (object_records.MAX_TSV_FIELD_BYTES // 4) + 100 # chars: just over cap/4 |
| 392 | over_by_bytes_under_by_chars = astral_char * n_chars |
| 393 | char_len = len(over_by_bytes_under_by_chars) |
| 394 | byte_len = len(over_by_bytes_under_by_chars.encode("utf-8")) |
| 395 | assert char_len < object_records.MAX_TSV_FIELD_BYTES, "sanity: char count must be under the cap" |
| 396 | assert byte_len > object_records.MAX_TSV_FIELD_BYTES, "sanity: byte count must be over the cap" |
| 397 | |
| 398 | for collection in ("classic_coll", "append_coll"): |
| 399 | with pytest.raises(object_records.InvalidRecordPayloadError): |
| 400 | object_records.create_collection_record( |
| 401 | collection, |
| 402 | {"id": "byte-over-char-under", "value": over_by_bytes_under_by_chars}, |
| 403 | base_dir=data_dir, |
| 404 | roots=[], |
| 405 | ) |
| 406 | |
| 407 | |
| 408 | # ============================================================================= |
| 409 | # 4. BOM (U+FEFF) |
| 410 | # ============================================================================= |
| 411 | |
| 412 | |
| 413 | @pytest.mark.parametrize( |
| 414 | "label,value", |
| 415 | [ |
| 416 | ("start", "hello world"), |
| 417 | ("middle", "hello world"), |
| 418 | ("end", "hello world"), |
| 419 | ("only", ""), |
| 420 | ], |
| 421 | ) |
| 422 | def test_bom_survives_raw_and_json_append_and_classic(tmp_path, monkeypatch, label, value): |
| 423 | """U+FEFF (BOM / zero-width no-break space) at start, middle, end of a |
| 424 | value, and a value that IS only a BOM -- must not be stripped, moved, or |
| 425 | misinterpreted as a stream-level encoding marker by the csv/text layer |
| 426 | (object_records opens files with explicit encoding="utf-8", not |
| 427 | "utf-8-sig", so a leading U+FEFF in a CELL VALUE should be indistinguishable |
| 428 | from any other codepoint to this layer).""" |
| 429 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 430 | data_dir = tmp_path / "data" |
| 431 | write_schema(data_dir, "classic_coll", [ID_FIELD, VALUE_FIELD]) |
| 432 | write_append_schema(data_dir, "append_coll", [ID_FIELD, VALUE_FIELD]) |
| 433 | |
| 434 | _roundtrip_raw(data_dir, "classic_coll", f"bom-{label}", value) |
| 435 | _roundtrip_raw(data_dir, "append_coll", f"bom-{label}", value) |
| 436 | |
| 437 | payload = {"note": value} |
| 438 | _roundtrip_json(data_dir, "classic_coll", f"bom-json-{label}", payload) |
| 439 | _roundtrip_json(data_dir, "append_coll", f"bom-json-{label}", payload) |
| 440 | |
| 441 | |
| 442 | # ============================================================================= |
| 443 | # 5. Combining sequences -- NFC vs NFD must round-trip byte-exact (no |
| 444 | # silent normalization by the store) |
| 445 | # ============================================================================= |
| 446 | |
| 447 | |
| 448 | def test_nfc_vs_nfd_are_distinct_and_survive_byte_exact(tmp_path, monkeypatch): |
| 449 | """"e" + U+0301 (combining acute accent, NFD) and precomposed "é" |
| 450 | (U+00E9, NFC) are canonically equivalent under Unicode normalization but |
| 451 | are DIFFERENT byte sequences. A storage layer must never silently |
| 452 | normalize one to the other -- this test asserts the two forms remain |
| 453 | distinguishable (their raw bytes differ) AND each survives its own |
| 454 | round trip unchanged, both raw and JSON-wrapped, both storage modes, |
| 455 | cold by-id read.""" |
| 456 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 457 | data_dir = tmp_path / "data" |
| 458 | write_schema(data_dir, "classic_coll", [ID_FIELD, VALUE_FIELD]) |
| 459 | write_append_schema(data_dir, "append_coll", [ID_FIELD, VALUE_FIELD]) |
| 460 | |
| 461 | # Sanity: these two Python strings really are different byte sequences, |
| 462 | # not accidentally identical (confirms this test is exercising what it |
| 463 | # claims to). |
| 464 | assert NFC_STRING != NFD_STRING |
| 465 | assert NFC_STRING.encode("utf-8") != NFD_STRING.encode("utf-8") |
| 466 | assert unicodedata.normalize("NFC", NFD_STRING) == NFC_STRING # canonically equivalent |
| 467 | |
| 468 | for collection in ("classic_coll", "append_coll"): |
| 469 | _roundtrip_raw(data_dir, collection, f"nfc-{collection}", NFC_STRING) |
| 470 | _roundtrip_raw(data_dir, collection, f"nfd-{collection}", NFD_STRING) |
| 471 | |
| 472 | # The store must not have coalesced the two into the same bytes. |
| 473 | _clear_caches() |
| 474 | nfc_got = object_records.get_collection_record( |
| 475 | collection, f"nfc-{collection}", base_dir=data_dir, roots=[] |
| 476 | ) |
| 477 | _clear_caches() |
| 478 | nfd_got = object_records.get_collection_record( |
| 479 | collection, f"nfd-{collection}", base_dir=data_dir, roots=[] |
| 480 | ) |
| 481 | assert nfc_got["value"] != nfd_got["value"] or NFC_STRING == NFD_STRING, ( |
| 482 | f"{collection}: NFC and NFD forms were silently normalized to the " |
| 483 | f"same stored value -- store must preserve the exact form given. " |
| 484 | f"nfc={nfc_got['value']!r} nfd={nfd_got['value']!r}" |
| 485 | ) |
| 486 | assert nfc_got["value"] == NFC_STRING |
| 487 | assert nfd_got["value"] == NFD_STRING |
| 488 | |
| 489 | payload_nfc = {"note": NFC_STRING} |
| 490 | payload_nfd = {"note": NFD_STRING} |
| 491 | _roundtrip_json(data_dir, collection, f"nfc-json-{collection}", payload_nfc) |
| 492 | _roundtrip_json(data_dir, collection, f"nfd-json-{collection}", payload_nfd) |
| 493 | |
| 494 | |
| 495 | # ============================================================================= |
| 496 | # 6. A multibyte character's UTF-8 bytes straddling a naive row-boundary split |
| 497 | # ============================================================================= |
| 498 | |
| 499 | |
| 500 | def test_multibyte_char_straddling_row_boundary_bytes(tmp_path, monkeypatch): |
| 501 | """Construct rows so that a multibyte character's byte span sits right |
| 502 | where a naive fixed-width or byte-count-based row splitter (as opposed |
| 503 | to the real csv/text-based row delimiting object_records.py actually |
| 504 | uses) would be tempted to cut -- e.g. a value engineered so the |
| 505 | cumulative byte offset right before/after it lands on suspicious |
| 506 | boundaries (mod 2, mod 3, mod 4 relative to the row start), stressing |
| 507 | that ROW delimiting is happening on '\\n'/csv structure and never on a |
| 508 | raw byte count that could bisect a multibyte sequence. |
| 509 | |
| 510 | Layout: several small rows, each ending in a different multibyte |
| 511 | character (2/3/4-byte), each row's total byte length deliberately |
| 512 | NOT a clean multiple of the multibyte char's own width, so the trailing |
| 513 | character's bytes don't line up neatly with the row's total byte count |
| 514 | -- if row-splitting ever degraded to a byte-oriented (not char/csv- |
| 515 | oriented) scan, misalignment here would be likely to surface as |
| 516 | corruption on the NEXT row rather than silently working by accident.""" |
| 517 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 518 | data_dir = tmp_path / "data" |
| 519 | write_append_schema(data_dir, "straddle", [ID_FIELD, VALUE_FIELD]) |
| 520 | |
| 521 | rows = { |
| 522 | "R1": "x" * 1 + "café", # ends in 2-byte char (é), odd prefix length |
| 523 | "R2": "xx" * 1 + "日本語", # ends in 3-byte chars, even prefix length |
| 524 | "R3": "x" * 3 + "\U00020000", # ends in 4-byte astral char, odd prefix length |
| 525 | "R4": "xxxx" + "Ω" + "x", # 2-byte char in the MIDDLE, ascii after |
| 526 | "R5": "plain ascii tail", |
| 527 | } |
| 528 | for record_id, value in rows.items(): |
| 529 | object_records.create_collection_record( |
| 530 | "straddle", {"id": record_id, "value": value}, base_dir=data_dir, roots=[] |
| 531 | ) |
| 532 | |
| 533 | for record_id, expected in rows.items(): |
| 534 | _clear_caches() |
| 535 | got = object_records.get_collection_record("straddle", record_id, base_dir=data_dir, roots=[]) |
| 536 | assert got["value"] == expected, ( |
| 537 | f"FINDING: '{record_id}' did not round-trip byte-exact -- possible " |
| 538 | f"row-boundary/byte-offset misalignment around a multibyte " |
| 539 | f"character. Expected {expected!r}, got {got['value']!r}" |
| 540 | ) |
| 541 | |
| 542 | _clear_caches() |
| 543 | all_records = {r["id"]: r["value"] for r in object_records.read_collection_records( |
| 544 | "straddle", base_dir=data_dir, roots=[] |
| 545 | )} |
| 546 | assert all_records == rows |