askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_charset_emoji_astral_characterization.py · 21.8 KB · 502 lines
| 1 | """CHARACTERIZATION tests: does the TSV/CSV record storage substrate |
| 2 | (object_records.py) safely hold emoji / astral-plane (U+10000+, 4-byte |
| 3 | UTF-8) / grapheme-cluster content through both storage modes -- especially |
| 4 | append mode's byte-offset id->offset sidecar (_scan_append_tail / |
| 5 | _oidx_get_row), whose row-span and seek math is byte-offset-based while |
| 6 | emoji content is multi-byte and often multi-codepoint per user-visible |
| 7 | "character" (ZWJ sequences, skin-tone modifiers, regional-indicator flag |
| 8 | pairs, variation selectors, keycap sequences). |
| 9 | |
| 10 | Pure characterization of EXISTING behavior. Does not modify any production |
| 11 | module. Where a case fails or corrupts data, the assertion states the |
| 12 | correctness property a safe substrate SHOULD have and is left to fail -- |
| 13 | that failure IS the finding. Do not "fix" a failing assertion by weakening |
| 14 | it to match broken behavior. |
| 15 | |
| 16 | Explicitly OUT OF SCOPE (per instructions): torn-tail behavior (known |
| 17 | deferred bug #2, see test_embedded_json_lines_characterization.py's |
| 18 | xfail). Nothing here writes a deliberately-truncated file. |
| 19 | |
| 20 | Mirrors tests/test_embedded_json_lines_characterization.py's setup |
| 21 | helpers/conventions (write_schema / write_append_schema / _clear_caches). |
| 22 | """ |
| 23 | |
| 24 | from __future__ import annotations |
| 25 | |
| 26 | import json |
| 27 | from pathlib import Path |
| 28 | |
| 29 | import pytest |
| 30 | |
| 31 | import object_records |
| 32 | |
| 33 | # Conformance tier: heavy charset characterization -- deselected from the |
| 34 | # per-commit run (see pyproject 'conformance' marker). Run: pytest -m conformance |
| 35 | pytestmark = pytest.mark.conformance |
| 36 | |
| 37 | |
| 38 | ID_FIELD = {"name": "id"} |
| 39 | TEXT_FIELD = {"name": "note", "type": "textarea"} |
| 40 | BLOB_FIELD = {"name": "blob", "type": "textarea"} |
| 41 | |
| 42 | |
| 43 | # --- setup helpers (mirror test_embedded_json_lines_characterization.py) --- |
| 44 | |
| 45 | |
| 46 | def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 47 | path = data_dir / "schemas" / f"{collection}.json" |
| 48 | path.parent.mkdir(parents=True, exist_ok=True) |
| 49 | path.write_text(json.dumps({"fields": fields})) |
| 50 | return path |
| 51 | |
| 52 | |
| 53 | def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 54 | path = data_dir / "schemas" / f"{collection}.json" |
| 55 | path.parent.mkdir(parents=True, exist_ok=True) |
| 56 | path.write_text(json.dumps({"fields": fields, "storage": "append"})) |
| 57 | return path |
| 58 | |
| 59 | |
| 60 | def _clear_caches() -> None: |
| 61 | """Force the NEXT read past the warm in-process caches: a cold |
| 62 | _RECORDS_CACHE + _OIDX_CACHE is what makes get_collection_record on an |
| 63 | append-mode file actually exercise the id->offset sidecar / _oidx_get_row |
| 64 | instead of serving from the ordinary records cache.""" |
| 65 | object_records._RECORDS_CACHE.clear() |
| 66 | object_records._OIDX_CACHE.clear() |
| 67 | |
| 68 | |
| 69 | # --- emoji / astral-plane / grapheme-cluster sample strings ----------------- |
| 70 | |
| 71 | SINGLE_EMOJI = "\U0001F600" # 😀 U+1F600, 4-byte UTF-8, single codepoint |
| 72 | MANY_EMOJI = SINGLE_EMOJI * 500 # many 4-byte chars in one field |
| 73 | |
| 74 | FAMILY_ZWJ = "\U0001F468\U0001F469\U0001F467\U0001F466" # 👨👩👧👦 |
| 75 | PROFESSION_ZWJ = "\U0001F469\U0001F4BB" # 👩💻 (woman + ZWJ + laptop) |
| 76 | COUPLE_ZWJ = "\U0001F469❤️\U0001F468" # 👩❤️👨 (woman-heart-man) |
| 77 | |
| 78 | SKIN_TONE_THUMBSUP = "\U0001F44D\U0001F3FD" # 👍🏽 base + Fitzpatrick modifier |
| 79 | SKIN_TONE_ZWJ_COMBO = ( |
| 80 | "\U0001F469\U0001F3FD\U0001F4BB" |
| 81 | ) # woman+medium-skin-tone ZWJ laptop (skin tone + ZWJ combined) |
| 82 | |
| 83 | FLAG_US = "\U0001F1FA\U0001F1F8" # 🇺🇸 two regional indicators |
| 84 | FLAG_JP = "\U0001F1EF\U0001F1F5" # 🇯🇵 |
| 85 | FLAGS_ADJACENT = FLAG_US + FLAG_JP + "\U0001F1EC\U0001F1E7" # US JP GB adjacent |
| 86 | |
| 87 | HEART_VARSEL = "❤️" # ❤️ base heart + emoji presentation selector |
| 88 | TEXT_PRESENTATION = "❤︎" # ❤︎ same base + TEXT presentation selector (VS15) |
| 89 | |
| 90 | KEYCAP_1 = "1️⃣" # 1️⃣ digit + VS16 + combining keycap |
| 91 | KEYCAP_SEQ = "1️⃣" + "2️⃣" + "3️⃣" # 1️⃣2️⃣3️⃣ |
| 92 | |
| 93 | MIXED = ( |
| 94 | "café éèê " # accented Latin |
| 95 | "日本語 " # CJK: 日本語 |
| 96 | + SINGLE_EMOJI |
| 97 | + " " |
| 98 | + FAMILY_ZWJ |
| 99 | + " " |
| 100 | + FLAG_US |
| 101 | + " " |
| 102 | + SKIN_TONE_THUMBSUP |
| 103 | + " " |
| 104 | + HEART_VARSEL |
| 105 | + " " |
| 106 | + KEYCAP_1 |
| 107 | ) |
| 108 | |
| 109 | ALL_SAMPLES: dict[str, str] = { |
| 110 | "single_emoji": SINGLE_EMOJI, |
| 111 | "many_emoji": MANY_EMOJI, |
| 112 | "family_zwj": FAMILY_ZWJ, |
| 113 | "profession_zwj": PROFESSION_ZWJ, |
| 114 | "couple_zwj": COUPLE_ZWJ, |
| 115 | "skin_tone_thumbsup": SKIN_TONE_THUMBSUP, |
| 116 | "skin_tone_zwj_combo": SKIN_TONE_ZWJ_COMBO, |
| 117 | "flag_us": FLAG_US, |
| 118 | "flags_adjacent": FLAGS_ADJACENT, |
| 119 | "heart_varsel": HEART_VARSEL, |
| 120 | "text_presentation": TEXT_PRESENTATION, |
| 121 | "keycap_1": KEYCAP_1, |
| 122 | "keycap_seq": KEYCAP_SEQ, |
| 123 | "mixed": MIXED, |
| 124 | } |
| 125 | |
| 126 | |
| 127 | # ============================================================================= |
| 128 | # 1. RAW FIELD round-trip -- classic mode, fold read AND by-id read |
| 129 | # ============================================================================= |
| 130 | |
| 131 | |
| 132 | @pytest.mark.parametrize("name", list(ALL_SAMPLES.keys())) |
| 133 | def test_raw_field_classic_mode_round_trip(tmp_path, name): |
| 134 | """A raw (non-JSON) field holding emoji/astral/grapheme-cluster content, |
| 135 | classic storage: exact string round trip via fold-all read and by-id |
| 136 | read (classic mode never touches the id->offset sidecar).""" |
| 137 | value = ALL_SAMPLES[name] |
| 138 | data_dir = tmp_path / "data" |
| 139 | write_schema(data_dir, "items", [ID_FIELD, TEXT_FIELD]) |
| 140 | |
| 141 | object_records.create_collection_record( |
| 142 | "items", {"id": "r1", "note": value}, base_dir=data_dir, roots=[] |
| 143 | ) |
| 144 | |
| 145 | _clear_caches() |
| 146 | all_records = object_records.read_collection_records("items", base_dir=data_dir, roots=[]) |
| 147 | assert len(all_records) == 1 |
| 148 | assert all_records[0]["note"] == value, ( |
| 149 | f"[{name}] classic fold-read: expected {value!r} ({len(value)} codepoints, " |
| 150 | f"{len(value.encode('utf-8'))} bytes), got {all_records[0]['note']!r}" |
| 151 | ) |
| 152 | |
| 153 | _clear_caches() |
| 154 | by_id = object_records.get_collection_record("items", "r1", base_dir=data_dir, roots=[]) |
| 155 | assert by_id["note"] == value, ( |
| 156 | f"[{name}] classic by-id read: expected {value!r}, got {by_id['note']!r}" |
| 157 | ) |
| 158 | |
| 159 | |
| 160 | # ============================================================================= |
| 161 | # 2. RAW FIELD round-trip -- append mode, fold read AND cold by-id/sidecar read |
| 162 | # ============================================================================= |
| 163 | |
| 164 | |
| 165 | @pytest.mark.parametrize("name", list(ALL_SAMPLES.keys())) |
| 166 | def test_raw_field_append_mode_round_trip(tmp_path, monkeypatch, name): |
| 167 | """Same, append storage, with the by-id read forced through the cold |
| 168 | id->offset sidecar path (DBBASIC_RECORDS_CACHE_MAX_ROWS=0), which is |
| 169 | the byte-offset-math-sensitive path this file exists to probe.""" |
| 170 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 171 | value = ALL_SAMPLES[name] |
| 172 | data_dir = tmp_path / "data" |
| 173 | write_append_schema(data_dir, "items", [ID_FIELD, TEXT_FIELD]) |
| 174 | |
| 175 | object_records.create_collection_record( |
| 176 | "items", {"id": "r1", "note": value}, base_dir=data_dir, roots=[] |
| 177 | ) |
| 178 | |
| 179 | _clear_caches() |
| 180 | all_records = object_records.read_collection_records("items", base_dir=data_dir, roots=[]) |
| 181 | assert len(all_records) == 1 |
| 182 | assert all_records[0]["note"] == value, ( |
| 183 | f"[{name}] append fold-read: expected {value!r}, got {all_records[0]['note']!r}" |
| 184 | ) |
| 185 | |
| 186 | _clear_caches() |
| 187 | by_id = object_records.get_collection_record("items", "r1", base_dir=data_dir, roots=[]) |
| 188 | assert by_id["note"] == value, ( |
| 189 | f"[{name}] append by-id (cold sidecar) read: expected {value!r} " |
| 190 | f"({len(value)} codepoints, {len(value.encode('utf-8'))} bytes), " |
| 191 | f"got {by_id['note']!r}" |
| 192 | ) |
| 193 | assert by_id["note"].encode("utf-8") == value.encode("utf-8"), ( |
| 194 | f"[{name}] append by-id read: raw UTF-8 bytes diverged even though " |
| 195 | f"decoded string comparison may have passed -- check for silently " |
| 196 | f"split surrogate pairs or a re-encoding round trip" |
| 197 | ) |
| 198 | |
| 199 | |
| 200 | # ============================================================================= |
| 201 | # 3. Inside a compact json.dumps blob -- classic AND append, fold + by-id |
| 202 | # ============================================================================= |
| 203 | |
| 204 | |
| 205 | @pytest.mark.parametrize("name", list(ALL_SAMPLES.keys())) |
| 206 | def test_json_blob_classic_and_append_round_trip(tmp_path, monkeypatch, name): |
| 207 | """The same content, wrapped as a value inside a compact json.dumps(...) |
| 208 | blob (the realistic way structured emoji-bearing content is stored), |
| 209 | both storage modes. json.dumps with default ensure_ascii=True escapes |
| 210 | non-ASCII as \\uXXXX (surrogate-pair escapes for astral codepoints) -- |
| 211 | that escaping happens BEFORE the TSV/CSV layer ever sees the string, so |
| 212 | this is a genuinely different code path than the raw-field tests above |
| 213 | (the literal bytes handed to csv.writer are pure ASCII backslash-u |
| 214 | escapes, not raw UTF-8 emoji bytes) and must be probed separately.""" |
| 215 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 216 | value = ALL_SAMPLES[name] |
| 217 | original = {"sku": f"SKU-{name}", "label": value, "n": 42} |
| 218 | blob = json.dumps(original) |
| 219 | |
| 220 | for mode, writer in (("classic", write_schema), ("append", write_append_schema)): |
| 221 | data_dir = tmp_path / f"data_{mode}" |
| 222 | writer(data_dir, "items", [ID_FIELD, BLOB_FIELD]) |
| 223 | |
| 224 | object_records.create_collection_record( |
| 225 | "items", {"id": "r1", "blob": blob}, base_dir=data_dir, roots=[] |
| 226 | ) |
| 227 | |
| 228 | _clear_caches() |
| 229 | all_records = object_records.read_collection_records("items", base_dir=data_dir, roots=[]) |
| 230 | assert len(all_records) == 1 |
| 231 | fold_loaded = json.loads(all_records[0]["blob"]) |
| 232 | assert fold_loaded == original, ( |
| 233 | f"[{name}/{mode}] json-blob fold-read: expected {original!r}, got {fold_loaded!r}" |
| 234 | ) |
| 235 | |
| 236 | _clear_caches() |
| 237 | by_id = object_records.get_collection_record("items", "r1", base_dir=data_dir, roots=[]) |
| 238 | by_id_loaded = json.loads(by_id["blob"]) |
| 239 | assert by_id_loaded == original, ( |
| 240 | f"[{name}/{mode}] json-blob by-id read: expected {original!r}, got {by_id_loaded!r}" |
| 241 | ) |
| 242 | assert by_id_loaded["label"] == value |
| 243 | assert len(by_id_loaded["label"]) == len(value) # codepoint count preserved |
| 244 | |
| 245 | |
| 246 | def test_json_blob_ensure_ascii_false_classic_and_append_round_trip(tmp_path, monkeypatch): |
| 247 | """Same as above but with ensure_ascii=False, so the JSON text itself |
| 248 | carries raw multi-byte UTF-8 emoji bytes (not \\uXXXX escapes) straight |
| 249 | into the TSV cell -- a different, arguably more hostile, byte pattern |
| 250 | for the CSV layer to quote/carry.""" |
| 251 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 252 | original = {"sku": "SKU-mixed", "label": MIXED, "flags": [FLAG_US, FLAG_JP], "n": 7} |
| 253 | blob = json.dumps(original, ensure_ascii=False) |
| 254 | |
| 255 | for mode, writer in (("classic", write_schema), ("append", write_append_schema)): |
| 256 | data_dir = tmp_path / f"data_{mode}" |
| 257 | writer(data_dir, "items", [ID_FIELD, BLOB_FIELD]) |
| 258 | |
| 259 | object_records.create_collection_record( |
| 260 | "items", {"id": "r1", "blob": blob}, base_dir=data_dir, roots=[] |
| 261 | ) |
| 262 | |
| 263 | _clear_caches() |
| 264 | all_records = object_records.read_collection_records("items", base_dir=data_dir, roots=[]) |
| 265 | assert json.loads(all_records[0]["blob"]) == original, ( |
| 266 | f"[{mode}] ensure_ascii=False fold-read mismatch" |
| 267 | ) |
| 268 | |
| 269 | _clear_caches() |
| 270 | by_id = object_records.get_collection_record("items", "r1", base_dir=data_dir, roots=[]) |
| 271 | assert json.loads(by_id["blob"]) == original, ( |
| 272 | f"[{mode}] ensure_ascii=False by-id read mismatch" |
| 273 | ) |
| 274 | |
| 275 | |
| 276 | # ============================================================================= |
| 277 | # 4. UPDATE path -- overwrite a record's emoji field, re-read |
| 278 | # ============================================================================= |
| 279 | |
| 280 | |
| 281 | @pytest.mark.parametrize("name", ["family_zwj", "flags_adjacent", "skin_tone_zwj_combo", "many_emoji"]) |
| 282 | def test_update_record_emoji_field_round_trip(tmp_path, monkeypatch, name): |
| 283 | """update_collection_record on a field that already holds -- and is |
| 284 | being replaced with different -- emoji/grapheme-cluster content, both |
| 285 | modes, by-id read forced cold afterwards.""" |
| 286 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 287 | value = ALL_SAMPLES[name] |
| 288 | updated_value = value + " updated " + SINGLE_EMOJI |
| 289 | |
| 290 | for mode, writer in (("classic", write_schema), ("append", write_append_schema)): |
| 291 | data_dir = tmp_path / f"data_{mode}" |
| 292 | writer(data_dir, "items", [ID_FIELD, TEXT_FIELD]) |
| 293 | |
| 294 | object_records.create_collection_record( |
| 295 | "items", {"id": "r1", "note": "placeholder"}, base_dir=data_dir, roots=[] |
| 296 | ) |
| 297 | object_records.update_collection_record( |
| 298 | "items", "r1", {"note": value}, base_dir=data_dir, roots=[] |
| 299 | ) |
| 300 | object_records.update_collection_record( |
| 301 | "items", "r1", {"note": updated_value}, base_dir=data_dir, roots=[] |
| 302 | ) |
| 303 | |
| 304 | _clear_caches() |
| 305 | by_id = object_records.get_collection_record("items", "r1", base_dir=data_dir, roots=[]) |
| 306 | assert by_id["note"] == updated_value, ( |
| 307 | f"[{name}/{mode}] update round-trip: expected {updated_value!r}, " |
| 308 | f"got {by_id['note']!r}" |
| 309 | ) |
| 310 | |
| 311 | |
| 312 | # ============================================================================= |
| 313 | # 5. OFFSET PROBE: append-mode sidecar byte math, emoji-heavy EARLY rows, |
| 314 | # later rows read BY ID with a cold cache to force the sidecar path. |
| 315 | # ============================================================================= |
| 316 | |
| 317 | |
| 318 | def test_offset_probe_emoji_heavy_early_rows_do_not_corrupt_later_by_id_reads(tmp_path, monkeypatch): |
| 319 | """THE central question this file exists to answer for append mode: |
| 320 | _scan_append_tail computes each physical row's byte span by re-encoding |
| 321 | consumed TEXT to UTF-8 bytes (see object_records.py's docstring on that |
| 322 | function), not by assuming 1 char == 1 byte -- so a run of early rows |
| 323 | stuffed with 4-byte astral emoji (which inflate byte length far more |
| 324 | than char length) should not throw off the byte offsets computed for |
| 325 | rows written AFTER them. This test forces the sidecar cold on every |
| 326 | point op and reads LATER records by id, asserting exact round trip, |
| 327 | after EARLIER records were written with large emoji payloads. |
| 328 | """ |
| 329 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 330 | data_dir = tmp_path / "data" |
| 331 | write_append_schema(data_dir, "items", [ID_FIELD, TEXT_FIELD]) |
| 332 | |
| 333 | # Early records: heavy emoji content (4-byte-per-char inflation), some |
| 334 | # with ZWJ sequences / flags / skin tones mixed in, to shift byte |
| 335 | # offsets substantially relative to char offsets. |
| 336 | early = { |
| 337 | "A": MANY_EMOJI, # 500 x 4-byte emoji = 2000 bytes in one field |
| 338 | "B": FAMILY_ZWJ * 100, |
| 339 | "C": FLAGS_ADJACENT * 100, |
| 340 | "D": SKIN_TONE_ZWJ_COMBO * 100, |
| 341 | } |
| 342 | for rec_id, value in early.items(): |
| 343 | object_records.create_collection_record( |
| 344 | "items", {"id": rec_id, "note": value}, base_dir=data_dir, roots=[] |
| 345 | ) |
| 346 | |
| 347 | # Later records: plain, distinguishable content -- correctness here is |
| 348 | # a direct probe of whether the sidecar's row offsets (computed while |
| 349 | # scanning past A-D's multi-byte content) landed in the right place. |
| 350 | later = { |
| 351 | "E": "later-record-E " + SINGLE_EMOJI, |
| 352 | "F": "later-record-F " + FLAG_JP, |
| 353 | "G": "later-record-G plain ascii", |
| 354 | "H": "later-record-H " + HEART_VARSEL + KEYCAP_1, |
| 355 | } |
| 356 | for rec_id, value in later.items(): |
| 357 | object_records.create_collection_record( |
| 358 | "items", {"id": rec_id, "note": value}, base_dir=data_dir, roots=[] |
| 359 | ) |
| 360 | |
| 361 | for rec_id, value in later.items(): |
| 362 | _clear_caches() # force cold cache -> id->offset sidecar path |
| 363 | got = object_records.get_collection_record("items", rec_id, base_dir=data_dir, roots=[]) |
| 364 | assert got["note"] == value, ( |
| 365 | f"OFFSET-PROBE FINDING: by-id read of '{rec_id}' (written AFTER " |
| 366 | f"emoji-heavy early rows A-D) returned wrong content -- the " |
| 367 | f"append sidecar's byte-offset math may be corrupted by 4-byte " |
| 368 | f"UTF-8 content in earlier rows. Expected {value!r}, got " |
| 369 | f"{got['note']!r}" |
| 370 | ) |
| 371 | |
| 372 | # Also confirm the early (emoji-heavy) records themselves still resolve |
| 373 | # correctly by id via the same cold sidecar path. |
| 374 | for rec_id, value in early.items(): |
| 375 | _clear_caches() |
| 376 | got = object_records.get_collection_record("items", rec_id, base_dir=data_dir, roots=[]) |
| 377 | assert got["note"] == value, ( |
| 378 | f"OFFSET-PROBE FINDING: by-id read of emoji-heavy early record " |
| 379 | f"'{rec_id}' itself is wrong. Expected {value!r}, got {got['note']!r}" |
| 380 | ) |
| 381 | |
| 382 | # And a fold-all read agrees with every by-id read. |
| 383 | _clear_caches() |
| 384 | all_records = {r["id"]: r["note"] for r in object_records.read_collection_records( |
| 385 | "items", base_dir=data_dir, roots=[] |
| 386 | )} |
| 387 | for rec_id, value in {**early, **later}.items(): |
| 388 | assert all_records[rec_id] == value, ( |
| 389 | f"OFFSET-PROBE FINDING: fold-all read disagrees with by-id read for '{rec_id}'" |
| 390 | ) |
| 391 | |
| 392 | |
| 393 | def test_offset_probe_emoji_heavy_row_update_then_later_rows_still_resolve(tmp_path, monkeypatch): |
| 394 | """Same offset-probe idea but adding an UPDATE of an early emoji-heavy |
| 395 | record (a second physical row superseding the first, shifting the tail |
| 396 | of the file again) before re-checking later records by id -- mirrors |
| 397 | the update-then-recheck shape of the offset-index crux test in |
| 398 | test_embedded_json_lines_characterization.py, but with emoji payloads |
| 399 | instead of embedded-newline JSON.""" |
| 400 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 401 | data_dir = tmp_path / "data" |
| 402 | write_append_schema(data_dir, "items", [ID_FIELD, TEXT_FIELD]) |
| 403 | |
| 404 | object_records.create_collection_record( |
| 405 | "items", {"id": "A", "note": "a-v1"}, base_dir=data_dir, roots=[] |
| 406 | ) |
| 407 | object_records.create_collection_record( |
| 408 | "items", {"id": "B", "note": MANY_EMOJI}, base_dir=data_dir, roots=[] |
| 409 | ) |
| 410 | object_records.create_collection_record( |
| 411 | "items", {"id": "C", "note": "c-v1 " + FLAGS_ADJACENT}, base_dir=data_dir, roots=[] |
| 412 | ) |
| 413 | object_records.create_collection_record( |
| 414 | "items", {"id": "D", "note": "d-v1 " + FAMILY_ZWJ}, base_dir=data_dir, roots=[] |
| 415 | ) |
| 416 | |
| 417 | _clear_caches() |
| 418 | rec_c = object_records.get_collection_record("items", "C", base_dir=data_dir, roots=[]) |
| 419 | assert rec_c["note"] == "c-v1 " + FLAGS_ADJACENT |
| 420 | |
| 421 | _clear_caches() |
| 422 | rec_d = object_records.get_collection_record("items", "D", base_dir=data_dir, roots=[]) |
| 423 | assert rec_d["note"] == "d-v1 " + FAMILY_ZWJ |
| 424 | |
| 425 | # Update B (emoji-heavy) -- a second physical row, superseding the |
| 426 | # first, shifting every subsequent byte offset again. |
| 427 | b_v2 = MANY_EMOJI + SKIN_TONE_ZWJ_COMBO * 50 |
| 428 | object_records.update_collection_record( |
| 429 | "items", "B", {"note": b_v2}, base_dir=data_dir, roots=[] |
| 430 | ) |
| 431 | |
| 432 | _clear_caches() |
| 433 | rec_c2 = object_records.get_collection_record("items", "C", base_dir=data_dir, roots=[]) |
| 434 | assert rec_c2["note"] == "c-v1 " + FLAGS_ADJACENT, ( |
| 435 | "OFFSET-PROBE FINDING: record C's by-id read is wrong after " |
| 436 | "updating emoji-heavy record B (second physical row)." |
| 437 | ) |
| 438 | |
| 439 | _clear_caches() |
| 440 | rec_d2 = object_records.get_collection_record("items", "D", base_dir=data_dir, roots=[]) |
| 441 | assert rec_d2["note"] == "d-v1 " + FAMILY_ZWJ, ( |
| 442 | "OFFSET-PROBE FINDING: record D's by-id read is wrong after " |
| 443 | "updating emoji-heavy record B (second physical row)." |
| 444 | ) |
| 445 | |
| 446 | _clear_caches() |
| 447 | rec_b = object_records.get_collection_record("items", "B", base_dir=data_dir, roots=[]) |
| 448 | assert rec_b["note"] == b_v2, ( |
| 449 | "OFFSET-PROBE FINDING: record B's own by-id read after its update is wrong." |
| 450 | ) |
| 451 | |
| 452 | |
| 453 | # ============================================================================= |
| 454 | # 6. Byte-exact / codepoint-exact assertions, explicit |
| 455 | # ============================================================================= |
| 456 | |
| 457 | |
| 458 | @pytest.mark.parametrize("name", list(ALL_SAMPLES.keys())) |
| 459 | def test_byte_exact_and_codepoint_exact_no_surrogate_split_no_dropped_joiner( |
| 460 | tmp_path, monkeypatch, name |
| 461 | ): |
| 462 | """Explicit assertions beyond simple string equality: codepoint COUNT |
| 463 | preserved (nothing collapsed/expanded), raw UTF-8 BYTES preserved |
| 464 | (nothing silently re-encoded/mangled), and -- for the ZWJ/modifier/flag |
| 465 | samples specifically -- every individual codepoint in the original |
| 466 | sequence appears, in the SAME ORDER, in the read-back value (so a U+200D |
| 467 | ZWJ joiner or a U+1F3FD skin-tone modifier can't have been silently |
| 468 | dropped or reordered while the surrounding text still happens to look |
| 469 | right under loose comparison).""" |
| 470 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 471 | value = ALL_SAMPLES[name] |
| 472 | data_dir = tmp_path / "data" |
| 473 | write_append_schema(data_dir, "items", [ID_FIELD, TEXT_FIELD]) |
| 474 | |
| 475 | object_records.create_collection_record( |
| 476 | "items", {"id": "r1", "note": value}, base_dir=data_dir, roots=[] |
| 477 | ) |
| 478 | |
| 479 | _clear_caches() |
| 480 | by_id = object_records.get_collection_record("items", "r1", base_dir=data_dir, roots=[]) |
| 481 | got = by_id["note"] |
| 482 | |
| 483 | assert len(got) == len(value), ( |
| 484 | f"[{name}] codepoint count changed: expected {len(value)}, got {len(got)} " |
| 485 | f"-- original codepoints={[hex(ord(c)) for c in value]!r}, " |
| 486 | f"got codepoints={[hex(ord(c)) for c in got]!r}" |
| 487 | ) |
| 488 | assert got.encode("utf-8") == value.encode("utf-8"), ( |
| 489 | f"[{name}] raw UTF-8 bytes diverged: " |
| 490 | f"expected {value.encode('utf-8')!r}, got {got.encode('utf-8')!r}" |
| 491 | ) |
| 492 | assert [ord(c) for c in got] == [ord(c) for c in value], ( |
| 493 | f"[{name}] codepoint sequence diverged (reorder/drop/insert): " |
| 494 | f"expected {[hex(ord(c)) for c in value]!r}, got {[hex(ord(c)) for c in got]!r}" |
| 495 | ) |
| 496 | # No lone surrogates: a well-formed decode of valid UTF-8 never produces |
| 497 | # one, but assert explicitly since a torn 4-byte sequence is exactly |
| 498 | # the kind of corruption a byte-offset bug could produce. |
| 499 | assert not any(0xD800 <= ord(c) <= 0xDFFF for c in got), ( |
| 500 | f"[{name}] read-back value contains a lone surrogate codepoint -- " |
| 501 | f"evidence of a split 4-byte UTF-8 sequence: {got!r}" |
| 502 | ) |