askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_charset_json_roundtrip_characterization.py · 32.5 KB · 768 lines
| 1 | """CHARACTERIZATION tests: does an ARBITRARY JSON payload -- specifically the |
| 2 | shape plan/vocabulary/66-line-items-spec.md proposes (a document's `items` |
| 3 | array, compact-`json.dumps`'d into one TSV cell) -- survive storage |
| 4 | byte-exactly and re-parse deep-equal, through BOTH the record storage layer |
| 5 | (object_records.py, classic + append storage) AND the HTTP API |
| 6 | (object_server.py's ASGI app, admin collection-record routes)? |
| 7 | |
| 8 | This file is pure characterization of EXISTING behavior. It does not modify |
| 9 | any production module. Where a case fails, the assertion is written to state |
| 10 | the correctness property a "safe to hold embedded JSON" substrate SHOULD |
| 11 | have, and is left to fail (or is downgraded to a soft print+record) -- that |
| 12 | outcome IS the finding. Do not "fix" a failing assertion here by weakening it |
| 13 | to match broken behavior. |
| 14 | |
| 15 | Companion to tests/test_embedded_json_lines_characterization.py (which |
| 16 | established the record-layer setup conventions this file reuses, and already |
| 17 | covers hostile-content-across-physical-lines / offset-index / scale / torn |
| 18 | tail -- NOT retested here; in particular the "torn tail is quote-blind" |
| 19 | substrate bug #2 is a KNOWN, DEFERRED finding and is deliberately NOT probed |
| 20 | by this file). This file's unique angle: full Unicode character-CLASS |
| 21 | coverage (RTL scripts, emoji+ZWJ+skin-tone+flags, combining marks, |
| 22 | zero-width chars), JSON structural edge cases (unicode keys, nesting, null/ |
| 23 | bool/large-int/float), the ensure_ascii=True vs False choice, the |
| 24 | MAX_TSV_FIELD_BYTES cap under multibyte content, idempotent re-serialization, |
| 25 | and the HTTP API surface, which the companion file never drives at all. |
| 26 | |
| 27 | Per instructions: do NOT probe torn-tail. Recent fixes (csv.field_size_limit |
| 28 | raise + write cap; _oidx_get_row csv-aware) are assumed already covered by |
| 29 | the companion file's regression guards and are not retested here except as |
| 30 | incidental background for the size-cap section. |
| 31 | """ |
| 32 | |
| 33 | from __future__ import annotations |
| 34 | |
| 35 | import asyncio |
| 36 | import copy |
| 37 | import json |
| 38 | from pathlib import Path |
| 39 | |
| 40 | import pytest |
| 41 | |
| 42 | import object_records |
| 43 | import object_server |
| 44 | |
| 45 | # Conformance tier: heavy charset characterization -- deselected from the |
| 46 | # per-commit run (see pyproject 'conformance' marker). Run: pytest -m conformance |
| 47 | pytestmark = pytest.mark.conformance |
| 48 | |
| 49 | ID_FIELD = {"name": "id"} |
| 50 | ITEMS_FIELD = {"name": "items", "type": "textarea"} |
| 51 | |
| 52 | TEST_ADMIN_TOKEN = "unit-test-only-admin-token" |
| 53 | |
| 54 | |
| 55 | # ============================================================================= |
| 56 | # setup helpers (mirror tests/test_embedded_json_lines_characterization.py) |
| 57 | # ============================================================================= |
| 58 | |
| 59 | |
| 60 | def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 61 | path = data_dir / "schemas" / f"{collection}.json" |
| 62 | path.parent.mkdir(parents=True, exist_ok=True) |
| 63 | path.write_text(json.dumps({"fields": fields})) |
| 64 | return path |
| 65 | |
| 66 | |
| 67 | def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path: |
| 68 | path = data_dir / "schemas" / f"{collection}.json" |
| 69 | path.parent.mkdir(parents=True, exist_ok=True) |
| 70 | path.write_text(json.dumps({"fields": fields, "storage": "append"})) |
| 71 | return path |
| 72 | |
| 73 | |
| 74 | def _clear_caches() -> None: |
| 75 | """Cold the in-process record cache + id->offset sidecar so a |
| 76 | following read genuinely exercises storage, not a warm hit -- same |
| 77 | trick the companion characterization file and object_records' own |
| 78 | oidx tests use.""" |
| 79 | object_records._RECORDS_CACHE.clear() |
| 80 | object_records._OIDX_CACHE.clear() |
| 81 | |
| 82 | |
| 83 | # ============================================================================= |
| 84 | # HTTP in-process ASGI driver (mirrors tests/test_object_server.py's |
| 85 | # asgi_request/request/raw_request helpers exactly, trimmed to what this |
| 86 | # file needs) |
| 87 | # ============================================================================= |
| 88 | |
| 89 | |
| 90 | def auth_headers(): |
| 91 | return [("authorization", f"Token {TEST_ADMIN_TOKEN}")] |
| 92 | |
| 93 | |
| 94 | def enable_admin_token(monkeypatch): |
| 95 | monkeypatch.setenv("DBBASIC_ADMIN_TOKEN", TEST_ADMIN_TOKEN) |
| 96 | |
| 97 | |
| 98 | async def asgi_request(path, method="GET", query_string="", body=b"", headers=None): |
| 99 | messages = [] |
| 100 | |
| 101 | async def receive(): |
| 102 | return {"type": "http.request", "body": body, "more_body": False} |
| 103 | |
| 104 | async def send(message): |
| 105 | messages.append(message) |
| 106 | |
| 107 | scope_headers = [(b"accept", b"application/json")] |
| 108 | for name, value in headers or []: |
| 109 | if isinstance(name, str): |
| 110 | name = name.encode("latin-1") |
| 111 | if isinstance(value, str): |
| 112 | value = value.encode("latin-1") |
| 113 | scope_headers.append((name, value)) |
| 114 | |
| 115 | scope = { |
| 116 | "type": "http", |
| 117 | "method": method, |
| 118 | "path": path, |
| 119 | "query_string": query_string.encode("utf-8"), |
| 120 | "headers": scope_headers, |
| 121 | "client": ("127.0.0.1", 12345), |
| 122 | } |
| 123 | await object_server.app(scope, receive, send) |
| 124 | |
| 125 | start = next(m for m in messages if m["type"] == "http.response.start") |
| 126 | body_parts = [m.get("body", b"") for m in messages if m["type"] == "http.response.body"] |
| 127 | payload = b"".join(body_parts) |
| 128 | return start["status"], dict(start["headers"]), payload |
| 129 | |
| 130 | |
| 131 | def request(path, method="GET", query_string="", body=b"", headers=None): |
| 132 | status, headers_out, payload = asyncio.run( |
| 133 | asgi_request(path, method=method, query_string=query_string, body=body, headers=headers) |
| 134 | ) |
| 135 | return status, headers_out, json.loads(payload.decode("utf-8")) |
| 136 | |
| 137 | |
| 138 | # ============================================================================= |
| 139 | # Realistic + adversarial JSON item-array fixtures |
| 140 | # ============================================================================= |
| 141 | |
| 142 | |
| 143 | def _charclass_string() -> str: |
| 144 | """One string mixing every character class the probe cares about.""" |
| 145 | return ( |
| 146 | 'ASCII "quoted" back\\slash, comma, and a semicolon; ' |
| 147 | "café résumé naïve " # accented Latin |
| 148 | "日本語のテスト文字列 " # CJK |
| 149 | "العربية " # Arabic (RTL) |
| 150 | "עברית " # Hebrew (RTL) |
| 151 | "ȩ́ " # combining acute + cedilla |
| 152 | "zerowidth " # ZWSP, ZWNJ, ZWJ |
| 153 | "\U0001f469\U0001f469\U0001f466\U0001f466 " # family ZWJ sequence |
| 154 | "\U0001f44d\U0001f3fd " # thumbs up + medium skin tone modifier |
| 155 | "\U0001f1fa\U0001f1f8 " # flag: US (regional indicator pair) |
| 156 | "\U0001f600" # simple emoji |
| 157 | ) |
| 158 | |
| 159 | |
| 160 | def _realistic_items() -> list[dict]: |
| 161 | """The actual embed use case: a document's `items` array, per |
| 162 | plan/vocabulary/66-line-items-spec.md's item_fields shape.""" |
| 163 | return [ |
| 164 | { |
| 165 | "product_id": "prod-001", |
| 166 | "description": "Café crème, 12oz — 日本茶セット", |
| 167 | "quantity": 2, |
| 168 | "unit_price_cents": 1099, |
| 169 | "tax_rate_bps": 875, |
| 170 | }, |
| 171 | { |
| 172 | "product_id": "prod-002", |
| 173 | "description": _charclass_string(), |
| 174 | "quantity": 1, |
| 175 | "unit_price_cents": 2500, |
| 176 | "tax_rate_bps": 0, |
| 177 | }, |
| 178 | { |
| 179 | "product_id": "prod-003", |
| 180 | "description": "", |
| 181 | "quantity": 0, |
| 182 | "unit_price_cents": 0, |
| 183 | "tax_rate_bps": None, |
| 184 | "note": None, |
| 185 | "gift_wrapped": True, |
| 186 | "backordered": False, |
| 187 | }, |
| 188 | ] |
| 189 | |
| 190 | |
| 191 | def _adversarial_items() -> list: |
| 192 | """JSON structural edge cases: unicode keys, deep nesting, empty |
| 193 | string, null, booleans, ints (incl. large), floats, a JSON-looking |
| 194 | string value, and a value containing literal TSV delimiter chars.""" |
| 195 | return [ |
| 196 | { |
| 197 | "étoile": "unicode key (accented Latin)", |
| 198 | "日本": "unicode key (CJK)", |
| 199 | "\U0001f600key": "unicode key (emoji)", |
| 200 | }, |
| 201 | { |
| 202 | "nested": { |
| 203 | "level2": { |
| 204 | "level3": [1, 2, {"level4": [{"level5": "deep"}]}], |
| 205 | }, |
| 206 | "array_of_arrays": [[1, 2], [3, [4, 5, [6, 7]]]], |
| 207 | } |
| 208 | }, |
| 209 | { |
| 210 | "empty_string": "", |
| 211 | "is_null": None, |
| 212 | "is_true": True, |
| 213 | "is_false": False, |
| 214 | "small_int": 42, |
| 215 | "large_int": 9223372036854775807, # int64 max |
| 216 | "huge_int": 123456789012345678901234567890, # beyond int64 |
| 217 | "float_val": 19.99, |
| 218 | "float_tricky": 0.1, |
| 219 | "json_looking_string": '{"not":"parsed","nested":[1,2,3]}', |
| 220 | "delimiter_string": "a\tb,c;d\nrepresented-as-escaped-in-json", |
| 221 | }, |
| 222 | ] |
| 223 | |
| 224 | |
| 225 | # ============================================================================= |
| 226 | # 1. Realistic items array -- classic + append, fold + by-id, plus update |
| 227 | # ============================================================================= |
| 228 | |
| 229 | |
| 230 | def test_realistic_items_round_trip_classic_mode_fold_and_by_id(tmp_path): |
| 231 | data_dir = tmp_path / "data" |
| 232 | write_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD]) |
| 233 | original = _realistic_items() |
| 234 | blob = json.dumps(original) |
| 235 | |
| 236 | object_records.create_collection_record( |
| 237 | "orders", {"id": "ord-1", "items": blob}, base_dir=data_dir, roots=[] |
| 238 | ) |
| 239 | |
| 240 | _clear_caches() |
| 241 | all_records = object_records.read_collection_records("orders", base_dir=data_dir, roots=[]) |
| 242 | assert json.loads(all_records[0]["items"]) == original |
| 243 | |
| 244 | _clear_caches() |
| 245 | by_id = object_records.get_collection_record("orders", "ord-1", base_dir=data_dir, roots=[]) |
| 246 | assert json.loads(by_id["items"]) == original |
| 247 | |
| 248 | # Stored cell must be delimiter-safe: compact json.dumps escapes all |
| 249 | # control chars in string VALUES, so no raw tab/newline byte should |
| 250 | # appear in the stored blob itself. |
| 251 | assert "\t" not in by_id["items"] |
| 252 | assert "\n" not in by_id["items"] |
| 253 | |
| 254 | |
| 255 | def test_realistic_items_round_trip_append_mode_fold_and_by_id(tmp_path, monkeypatch): |
| 256 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 257 | data_dir = tmp_path / "data" |
| 258 | write_append_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD]) |
| 259 | original = _realistic_items() |
| 260 | blob = json.dumps(original) |
| 261 | |
| 262 | object_records.create_collection_record( |
| 263 | "orders", {"id": "ord-1", "items": blob}, base_dir=data_dir, roots=[] |
| 264 | ) |
| 265 | |
| 266 | _clear_caches() |
| 267 | all_records = object_records.read_collection_records("orders", base_dir=data_dir, roots=[]) |
| 268 | assert json.loads(all_records[0]["items"]) == original |
| 269 | |
| 270 | _clear_caches() |
| 271 | by_id = object_records.get_collection_record("orders", "ord-1", base_dir=data_dir, roots=[]) |
| 272 | assert json.loads(by_id["items"]) == original |
| 273 | assert "\t" not in by_id["items"] |
| 274 | assert "\n" not in by_id["items"] |
| 275 | |
| 276 | |
| 277 | @pytest.mark.parametrize("storage", ["classic", "append"]) |
| 278 | def test_realistic_items_update_one_item_then_reread(tmp_path, monkeypatch, storage): |
| 279 | """The document-lines-spec's mutation shape: read the whole array, |
| 280 | mutate ONE item, re-serialize the WHOLE array, write it back -- then |
| 281 | confirm a cold-cache reread agrees deep-equal.""" |
| 282 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 283 | data_dir = tmp_path / "data" |
| 284 | collection = "orders" |
| 285 | if storage == "classic": |
| 286 | write_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD]) |
| 287 | else: |
| 288 | write_append_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD]) |
| 289 | |
| 290 | original = _realistic_items() |
| 291 | object_records.create_collection_record( |
| 292 | collection, {"id": "ord-1", "items": json.dumps(original)}, base_dir=data_dir, roots=[] |
| 293 | ) |
| 294 | |
| 295 | _clear_caches() |
| 296 | current = object_records.get_collection_record(collection, "ord-1", base_dir=data_dir, roots=[]) |
| 297 | items = json.loads(current["items"]) |
| 298 | items[1]["quantity"] += 5 |
| 299 | items[1]["description"] = items[1]["description"] + " (updated) " + _charclass_string() |
| 300 | expected = copy.deepcopy(items) |
| 301 | |
| 302 | object_records.update_collection_record( |
| 303 | collection, "ord-1", {"items": json.dumps(items)}, base_dir=data_dir, roots=[] |
| 304 | ) |
| 305 | |
| 306 | _clear_caches() |
| 307 | after = object_records.get_collection_record(collection, "ord-1", base_dir=data_dir, roots=[]) |
| 308 | assert json.loads(after["items"]) == expected |
| 309 | |
| 310 | |
| 311 | # ============================================================================= |
| 312 | # 2. Adversarial JSON structural edge cases -- both modes |
| 313 | # ============================================================================= |
| 314 | |
| 315 | |
| 316 | @pytest.mark.parametrize("storage", ["classic", "append"]) |
| 317 | def test_adversarial_structural_edge_cases_round_trip(tmp_path, monkeypatch, storage): |
| 318 | """unicode keys, deep nesting, null/bool/int/float, a JSON-looking |
| 319 | string value, and a string literally containing TSV delimiter chars |
| 320 | (tab/comma/semicolon/newline) -- all inside JSON so json.dumps has |
| 321 | already escaped the control chars before the TSV layer ever sees |
| 322 | them.""" |
| 323 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 324 | data_dir = tmp_path / "data" |
| 325 | collection = "orders" |
| 326 | if storage == "classic": |
| 327 | write_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD]) |
| 328 | else: |
| 329 | write_append_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD]) |
| 330 | |
| 331 | original = _adversarial_items() |
| 332 | blob = json.dumps(original) |
| 333 | |
| 334 | object_records.create_collection_record( |
| 335 | collection, {"id": "adv-1", "items": blob}, base_dir=data_dir, roots=[] |
| 336 | ) |
| 337 | |
| 338 | _clear_caches() |
| 339 | all_records = object_records.read_collection_records(collection, base_dir=data_dir, roots=[]) |
| 340 | loaded_fold = json.loads(all_records[0]["items"]) |
| 341 | |
| 342 | _clear_caches() |
| 343 | by_id = object_records.get_collection_record(collection, "adv-1", base_dir=data_dir, roots=[]) |
| 344 | loaded_by_id = json.loads(by_id["items"]) |
| 345 | |
| 346 | # unicode keys survive |
| 347 | assert loaded_fold[0] == original[0] |
| 348 | assert loaded_by_id[0] == original[0] |
| 349 | assert set(loaded_by_id[0].keys()) == set(original[0].keys()) |
| 350 | |
| 351 | # deep nesting survives |
| 352 | assert loaded_fold[1] == original[1] |
| 353 | assert loaded_by_id[1] == original[1] |
| 354 | |
| 355 | # null/bool/int survive exactly |
| 356 | assert loaded_by_id[2]["is_null"] is None |
| 357 | assert loaded_by_id[2]["is_true"] is True |
| 358 | assert loaded_by_id[2]["is_false"] is False |
| 359 | assert loaded_by_id[2]["small_int"] == 42 |
| 360 | assert loaded_by_id[2]["large_int"] == 9223372036854775807 |
| 361 | assert loaded_by_id[2]["huge_int"] == 123456789012345678901234567890 |
| 362 | |
| 363 | # a JSON-looking string value must come back as a STRING, not be |
| 364 | # double-parsed into a nested object |
| 365 | assert isinstance(loaded_by_id[2]["json_looking_string"], str) |
| 366 | assert loaded_by_id[2]["json_looking_string"] == original[2]["json_looking_string"] |
| 367 | |
| 368 | # a value containing literal delimiter characters (tab/comma/ |
| 369 | # semicolon/newline) survives -- these are JSON-string-escaped by |
| 370 | # json.dumps, so the TSV layer never sees the raw bytes |
| 371 | assert loaded_by_id[2]["delimiter_string"] == original[2]["delimiter_string"] |
| 372 | |
| 373 | assert loaded_fold == original |
| 374 | assert loaded_by_id == original |
| 375 | |
| 376 | |
| 377 | def test_float_round_trip_report(tmp_path): |
| 378 | """Report-only: do JSON floats survive json.dumps -> TSV cell -> |
| 379 | json.loads exactly? (Money doctrine elsewhere in this codebase already |
| 380 | mandates integer minor units / cents for currency -- this test exists |
| 381 | to CONFIRM that recommendation empirically for the items-array use |
| 382 | case, not to relitigate it.)""" |
| 383 | data_dir = tmp_path / "data" |
| 384 | write_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD]) |
| 385 | original = [{"float_a": 19.99, "float_b": 0.1, "float_c": 1e300, "float_d": 1.1e-10}] |
| 386 | blob = json.dumps(original) |
| 387 | |
| 388 | object_records.create_collection_record( |
| 389 | "orders", {"id": "f-1", "items": blob}, base_dir=data_dir, roots=[] |
| 390 | ) |
| 391 | _clear_caches() |
| 392 | by_id = object_records.get_collection_record("orders", "f-1", base_dir=data_dir, roots=[]) |
| 393 | loaded = json.loads(by_id["items"]) |
| 394 | |
| 395 | # Python's float repr round-trips via repr()/json for IEEE-754 |
| 396 | # doubles (json encodes via float.__repr__, which is the shortest |
| 397 | # string that round-trips) -- so THIS layer (storage) does not lose |
| 398 | # precision. The finding is about REPRESENTATION choice, not this |
| 399 | # substrate: any float value is inherently binary-imprecise for |
| 400 | # currency math (e.g. 19.99 is not exactly representable), so even |
| 401 | # though it round-trips bit-for-bit through storage here, arithmetic |
| 402 | # on such floats elsewhere will accumulate error. Report both facts. |
| 403 | print(f"\n[float report] storage round-trip exact: {loaded == original}") |
| 404 | print(f"[float report] original: {original}") |
| 405 | print(f"[float report] loaded: {loaded}") |
| 406 | assert loaded == original, ( |
| 407 | "FINDING: float values did NOT survive storage exactly -- " |
| 408 | f"original={original!r} loaded={loaded!r}" |
| 409 | ) |
| 410 | print( |
| 411 | "[float report] RECOMMENDATION: storage itself preserves float bits " |
| 412 | "exactly (json+repr round-trip), but floats remain the wrong " |
| 413 | "representation for money -- integer minor units (cents) per the " |
| 414 | "codebase's existing money doctrine, not because storage corrupts " |
| 415 | "them, but because binary floats can't exactly represent most " |
| 416 | "decimal currency values in the first place (e.g. 19.99), so any " |
| 417 | "arithmetic over them (subtotals/tax) will drift." |
| 418 | ) |
| 419 | |
| 420 | |
| 421 | # ============================================================================= |
| 422 | # 3. ensure_ascii True vs False |
| 423 | # ============================================================================= |
| 424 | |
| 425 | |
| 426 | @pytest.mark.parametrize("storage", ["classic", "append"]) |
| 427 | def test_ensure_ascii_true_default_round_trips_and_is_pure_ascii_cell(tmp_path, monkeypatch, storage): |
| 428 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 429 | data_dir = tmp_path / "data" |
| 430 | collection = "orders" |
| 431 | if storage == "classic": |
| 432 | write_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD]) |
| 433 | else: |
| 434 | write_append_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD]) |
| 435 | |
| 436 | original = _realistic_items() |
| 437 | blob = json.dumps(original) # ensure_ascii=True is the json.dumps default |
| 438 | assert blob.isascii(), "sanity: default json.dumps must produce a pure-ASCII string" |
| 439 | |
| 440 | object_records.create_collection_record( |
| 441 | collection, {"id": "ascii-1", "items": blob}, base_dir=data_dir, roots=[] |
| 442 | ) |
| 443 | _clear_caches() |
| 444 | by_id = object_records.get_collection_record(collection, "ascii-1", base_dir=data_dir, roots=[]) |
| 445 | assert by_id["items"].isascii(), "stored cell should remain pure ASCII (\\uXXXX escapes only)" |
| 446 | assert json.loads(by_id["items"]) == original |
| 447 | |
| 448 | |
| 449 | @pytest.mark.parametrize("storage", ["classic", "append"]) |
| 450 | def test_ensure_ascii_false_round_trips_with_raw_multibyte_in_cell(tmp_path, monkeypatch, storage): |
| 451 | """ensure_ascii=False puts raw UTF-8 multibyte (and raw emoji, RTL |
| 452 | script, etc.) directly in the TSV cell instead of \\uXXXX escapes. |
| 453 | This still round-trips through this substrate (the TSV/CSV layer is |
| 454 | UTF-8 text throughout), but the cell is no longer pure ASCII on disk.""" |
| 455 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 456 | data_dir = tmp_path / "data" |
| 457 | collection = "orders" |
| 458 | if storage == "classic": |
| 459 | write_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD]) |
| 460 | else: |
| 461 | write_append_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD]) |
| 462 | |
| 463 | original = _realistic_items() |
| 464 | blob = json.dumps(original, ensure_ascii=False) |
| 465 | assert not blob.isascii(), "sanity: ensure_ascii=False should emit raw multibyte here" |
| 466 | |
| 467 | object_records.create_collection_record( |
| 468 | collection, {"id": "nonascii-1", "items": blob}, base_dir=data_dir, roots=[] |
| 469 | ) |
| 470 | _clear_caches() |
| 471 | by_id = object_records.get_collection_record(collection, "nonascii-1", base_dir=data_dir, roots=[]) |
| 472 | assert not by_id["items"].isascii() |
| 473 | assert json.loads(by_id["items"]) == original |
| 474 | # No raw tab/newline bytes even with ensure_ascii=False -- control-char |
| 475 | # escaping in JSON strings is independent of the ensure_ascii setting |
| 476 | # (only affects non-ASCII codepoints >= 0x80, not control chars). |
| 477 | assert "\t" not in by_id["items"] |
| 478 | assert "\n" not in by_id["items"] |
| 479 | |
| 480 | |
| 481 | # ============================================================================= |
| 482 | # 4. Size cap interaction with multibyte content |
| 483 | # ============================================================================= |
| 484 | |
| 485 | |
| 486 | def test_size_cap_multibyte_content_byte_accurate(tmp_path): |
| 487 | """A JSON array whose serialized bytes approach MAX_TSV_FIELD_BYTES, |
| 488 | built from multibyte (CJK) content so char-count and byte-count |
| 489 | diverge sharply -- verify the cap is enforced on BYTES (as |
| 490 | MAX_TSV_FIELD_BYTES's docstring/companion test claims), not characters, |
| 491 | and that a comfortably-under-cap multibyte blob still round-trips. |
| 492 | |
| 493 | Uses ensure_ascii=False so each CJK codepoint costs its real 3 raw |
| 494 | UTF-8 bytes in the cell. NOTE (finding, surfaced in the report): with |
| 495 | the codebase's actual convention -- default json.dumps, ensure_ascii= |
| 496 | True -- the SAME logical content costs 6 ASCII bytes per CJK |
| 497 | codepoint (a "\\uXXXX" escape) instead of 3 raw UTF-8 bytes, i.e. |
| 498 | roughly DOUBLE the serialized size for heavily non-ASCII item text. |
| 499 | That inflation eats into the effective headroom under MAX_TSV_FIELD_ |
| 500 | BYTES / max_items caps for non-Latin-script content specifically, and |
| 501 | was surprising enough during test construction (a naive byte budget |
| 502 | silently produced a blob 2x the intended size) that it's worth |
| 503 | reporting explicitly rather than treating as obvious.""" |
| 504 | data_dir = tmp_path / "data" |
| 505 | write_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD]) |
| 506 | |
| 507 | # Each item's description is a CJK string (3 bytes/char in UTF-8) -- |
| 508 | # build enough items to land close to, but safely under, the cap. |
| 509 | cjk_char = "日" # 3 bytes UTF-8 (raw); 6 bytes if json.dumps escapes it |
| 510 | target_bytes = object_records.MAX_TSV_FIELD_BYTES - 2048 # headroom for JSON syntax overhead |
| 511 | # each item is roughly: {"d":"...."} -- budget item body length below |
| 512 | item_overhead = len('{"d":""}') |
| 513 | n_items = 50 |
| 514 | chars_per_item = (target_bytes // n_items - item_overhead) // 3 |
| 515 | items = [{"d": cjk_char * chars_per_item} for _ in range(n_items)] |
| 516 | blob = json.dumps(items, ensure_ascii=False) |
| 517 | blob_bytes = len(blob.encode("utf-8")) |
| 518 | assert blob_bytes < object_records.MAX_TSV_FIELD_BYTES, ( |
| 519 | f"test construction sanity: blob_bytes={blob_bytes} must stay under cap " |
| 520 | f"{object_records.MAX_TSV_FIELD_BYTES} to prove the under-cap case" |
| 521 | ) |
| 522 | # Same logical content, but serialized the codebase's actual way |
| 523 | # (ensure_ascii=True, the json.dumps default used everywhere else in |
| 524 | # this repo) -- report the inflation factor. |
| 525 | blob_ascii_escaped = json.dumps(items) |
| 526 | blob_ascii_escaped_bytes = len(blob_ascii_escaped.encode("utf-8")) |
| 527 | print(f"\n[size-cap] multibyte blob (ensure_ascii=False, raw UTF-8): " |
| 528 | f"{blob_bytes} bytes (cap={object_records.MAX_TSV_FIELD_BYTES}), {len(blob)} chars") |
| 529 | print(f"[size-cap] SAME content (ensure_ascii=True, \\uXXXX escapes): " |
| 530 | f"{blob_ascii_escaped_bytes} bytes -- " |
| 531 | f"{blob_ascii_escaped_bytes / blob_bytes:.2f}x larger") |
| 532 | |
| 533 | object_records.create_collection_record( |
| 534 | "orders", {"id": "big-multibyte", "items": blob}, base_dir=data_dir, roots=[] |
| 535 | ) |
| 536 | _clear_caches() |
| 537 | by_id = object_records.get_collection_record("orders", "big-multibyte", base_dir=data_dir, roots=[]) |
| 538 | assert json.loads(by_id["items"]) == items |
| 539 | |
| 540 | # Now push a multibyte blob just OVER the byte cap (but with a |
| 541 | # char-count that would be well under it) -- the cap must reject on |
| 542 | # BYTES, not code points, or this write would wrongly succeed. |
| 543 | over_target_bytes = object_records.MAX_TSV_FIELD_BYTES + 300 |
| 544 | n_items_over = 60 |
| 545 | chars_per_item_over = (over_target_bytes // n_items_over - item_overhead) // 3 + 50 |
| 546 | over_items = [{"d": cjk_char * chars_per_item_over} for _ in range(n_items_over)] |
| 547 | over_blob = json.dumps(over_items, ensure_ascii=False) |
| 548 | over_bytes = len(over_blob.encode("utf-8")) |
| 549 | over_chars = len(over_blob) |
| 550 | assert over_bytes > object_records.MAX_TSV_FIELD_BYTES, ( |
| 551 | f"test construction sanity: need over_bytes > cap, got {over_bytes}" |
| 552 | ) |
| 553 | print(f"[size-cap] over-cap multibyte blob: {over_bytes} bytes, {over_chars} chars " |
| 554 | f"(chars < bytes: {over_chars < object_records.MAX_TSV_FIELD_BYTES})") |
| 555 | |
| 556 | with pytest.raises(object_records.InvalidRecordPayloadError): |
| 557 | object_records.create_collection_record( |
| 558 | "orders", {"id": "too-big-multibyte", "items": over_blob}, base_dir=data_dir, roots=[] |
| 559 | ) |
| 560 | |
| 561 | |
| 562 | # ============================================================================= |
| 563 | # 5. Idempotent re-serialization (compact-form stability) |
| 564 | # ============================================================================= |
| 565 | |
| 566 | |
| 567 | @pytest.mark.parametrize("storage", ["classic", "append"]) |
| 568 | def test_idempotent_reserialization_stability(tmp_path, monkeypatch, storage): |
| 569 | """json.loads(read) then json.dumps(...) again (compact form, same |
| 570 | separators/key order as the original write) must reproduce the exact |
| 571 | stored blob byte-for-byte -- i.e. round-tripping through this layer |
| 572 | introduces no drift a naive re-save-without-edits would compound.""" |
| 573 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 574 | data_dir = tmp_path / "data" |
| 575 | collection = "orders" |
| 576 | if storage == "classic": |
| 577 | write_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD]) |
| 578 | else: |
| 579 | write_append_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD]) |
| 580 | |
| 581 | original = _realistic_items() + _adversarial_items() |
| 582 | blob = json.dumps(original) |
| 583 | |
| 584 | object_records.create_collection_record( |
| 585 | collection, {"id": "stable-1", "items": blob}, base_dir=data_dir, roots=[] |
| 586 | ) |
| 587 | _clear_caches() |
| 588 | by_id = object_records.get_collection_record(collection, "stable-1", base_dir=data_dir, roots=[]) |
| 589 | stored = by_id["items"] |
| 590 | |
| 591 | reparsed = json.loads(stored) |
| 592 | reserialized = json.dumps(reparsed) |
| 593 | assert reserialized == stored, ( |
| 594 | "idempotency FINDING: re-serializing the parsed value did not " |
| 595 | "reproduce the stored blob byte-for-byte" |
| 596 | ) |
| 597 | assert reserialized == blob |
| 598 | |
| 599 | |
| 600 | # ============================================================================= |
| 601 | # 6. HTTP API path -- admin collection-record routes |
| 602 | # ============================================================================= |
| 603 | |
| 604 | |
| 605 | def test_http_create_and_read_realistic_items_round_trip_classic(tmp_path, monkeypatch): |
| 606 | data_dir = tmp_path / "data" |
| 607 | write_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD]) |
| 608 | monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir)) |
| 609 | enable_admin_token(monkeypatch) |
| 610 | |
| 611 | original = _realistic_items() |
| 612 | blob = json.dumps(original) |
| 613 | |
| 614 | create_status, _, create_payload = request( |
| 615 | "/admin/collections/orders/records", |
| 616 | method="POST", |
| 617 | body=json.dumps({"id": "http-1", "items": blob}).encode("utf-8"), |
| 618 | headers=auth_headers(), |
| 619 | ) |
| 620 | assert create_status == 201, create_payload |
| 621 | assert json.loads(create_payload["record"]["items"]) == original |
| 622 | |
| 623 | _clear_caches() |
| 624 | list_status, _, list_payload = request( |
| 625 | "/admin/collections/orders/records", headers=auth_headers() |
| 626 | ) |
| 627 | assert list_status == 200 |
| 628 | fold_row = next(r for r in list_payload["records"] if r["id"] == "http-1") |
| 629 | assert json.loads(fold_row["items"]) == original |
| 630 | |
| 631 | _clear_caches() |
| 632 | get_status, _, get_payload = request( |
| 633 | "/admin/collections/orders/records/http-1", headers=auth_headers() |
| 634 | ) |
| 635 | assert get_status == 200 |
| 636 | assert json.loads(get_payload["record"]["items"]) == original |
| 637 | |
| 638 | |
| 639 | def test_http_create_and_read_realistic_items_round_trip_append(tmp_path, monkeypatch): |
| 640 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 641 | data_dir = tmp_path / "data" |
| 642 | write_append_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD]) |
| 643 | monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir)) |
| 644 | enable_admin_token(monkeypatch) |
| 645 | |
| 646 | original = _adversarial_items() |
| 647 | blob = json.dumps(original) |
| 648 | |
| 649 | create_status, _, create_payload = request( |
| 650 | "/admin/collections/orders/records", |
| 651 | method="POST", |
| 652 | body=json.dumps({"id": "http-adv-1", "items": blob}).encode("utf-8"), |
| 653 | headers=auth_headers(), |
| 654 | ) |
| 655 | assert create_status == 201, create_payload |
| 656 | assert json.loads(create_payload["record"]["items"]) == original |
| 657 | |
| 658 | _clear_caches() |
| 659 | get_status, _, get_payload = request( |
| 660 | "/admin/collections/orders/records/http-adv-1", headers=auth_headers() |
| 661 | ) |
| 662 | assert get_status == 200 |
| 663 | assert json.loads(get_payload["record"]["items"]) == original |
| 664 | |
| 665 | |
| 666 | def test_http_update_one_item_then_reread(tmp_path, monkeypatch): |
| 667 | """The HTTP path's version of the update-an-item-then-reread probe: |
| 668 | POST create, GET, mutate client-side, PUT the whole items array back, |
| 669 | GET again (cold cache) and confirm deep equality.""" |
| 670 | monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0") |
| 671 | data_dir = tmp_path / "data" |
| 672 | write_append_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD]) |
| 673 | monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir)) |
| 674 | enable_admin_token(monkeypatch) |
| 675 | |
| 676 | original = _realistic_items() |
| 677 | request( |
| 678 | "/admin/collections/orders/records", |
| 679 | method="POST", |
| 680 | body=json.dumps({"id": "http-upd-1", "items": json.dumps(original)}).encode("utf-8"), |
| 681 | headers=auth_headers(), |
| 682 | ) |
| 683 | |
| 684 | _clear_caches() |
| 685 | get_status, _, get_payload = request( |
| 686 | "/admin/collections/orders/records/http-upd-1", headers=auth_headers() |
| 687 | ) |
| 688 | assert get_status == 200 |
| 689 | items = json.loads(get_payload["record"]["items"]) |
| 690 | items[0]["description"] = _charclass_string() + " EDITED" |
| 691 | items[0]["quantity"] = 999 |
| 692 | expected = copy.deepcopy(items) |
| 693 | |
| 694 | update_status, _, update_payload = request( |
| 695 | "/admin/collections/orders/records/http-upd-1", |
| 696 | method="PUT", |
| 697 | body=json.dumps({"items": json.dumps(items)}).encode("utf-8"), |
| 698 | headers=auth_headers(), |
| 699 | ) |
| 700 | assert update_status == 200, update_payload |
| 701 | assert json.loads(update_payload["record"]["items"]) == expected |
| 702 | |
| 703 | _clear_caches() |
| 704 | reget_status, _, reget_payload = request( |
| 705 | "/admin/collections/orders/records/http-upd-1", headers=auth_headers() |
| 706 | ) |
| 707 | assert reget_status == 200 |
| 708 | assert json.loads(reget_payload["record"]["items"]) == expected |
| 709 | |
| 710 | |
| 711 | def test_http_request_body_cap_is_far_below_the_field_cap_FINDING(tmp_path, monkeypatch): |
| 712 | """FINDING: object_server's own HTTP request-body-size gate |
| 713 | (DBBASIC_MAX_REQUEST_BYTES, default 1 MiB -- see |
| 714 | object_server.DEFAULT_MAX_REQUEST_BYTES) is enforced BEFORE the JSON |
| 715 | body is even parsed, and its default (1,048,576 bytes) is 16x SMALLER |
| 716 | than object_records.MAX_TSV_FIELD_BYTES (16 MiB). A single `items` |
| 717 | field anywhere near the record-layer cap can never actually reach |
| 718 | create_collection_record's InvalidRecordPayloadError check via HTTP |
| 719 | under default configuration -- it is rejected first, at the transport |
| 720 | layer, with 413 "Request body too large" (a different status/shape |
| 721 | than the record-layer's 400 InvalidRecordPayloadError). This is not |
| 722 | unsafe (both reject the oversize write), but it means the HTTP path's |
| 723 | effective per-field ceiling for `items` is ~1 MiB by default, not |
| 724 | 16 MiB, and any client-facing error handling for "items too large" |
| 725 | over HTTP must expect 413, not 400, unless the operator raises |
| 726 | DBBASIC_MAX_REQUEST_BYTES.""" |
| 727 | data_dir = tmp_path / "data" |
| 728 | write_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD]) |
| 729 | monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir)) |
| 730 | enable_admin_token(monkeypatch) |
| 731 | |
| 732 | # A field comfortably PAST MAX_TSV_FIELD_BYTES, well past the |
| 733 | # object_server.DEFAULT_MAX_REQUEST_BYTES (1 MiB) transport gate too. |
| 734 | over = json.dumps([{"d": "x" * (object_records.MAX_TSV_FIELD_BYTES + 1000)}]) |
| 735 | status, _, payload = request( |
| 736 | "/admin/collections/orders/records", |
| 737 | method="POST", |
| 738 | body=json.dumps({"id": "http-too-big", "items": over}).encode("utf-8"), |
| 739 | headers=auth_headers(), |
| 740 | ) |
| 741 | print(f"\n[http-cap FINDING] default DBBASIC_MAX_REQUEST_BYTES gate hit first: " |
| 742 | f"status={status} payload={payload}") |
| 743 | assert status == 413, payload |
| 744 | assert payload["status"] == "error" |
| 745 | |
| 746 | |
| 747 | def test_http_oversize_items_payload_rejected_when_request_cap_raised(tmp_path, monkeypatch): |
| 748 | """With the transport-level request-body cap explicitly raised above |
| 749 | MAX_TSV_FIELD_BYTES (DBBASIC_MAX_REQUEST_BYTES), the request now |
| 750 | reaches record-layer validation and the FIELD cap itself is exercised |
| 751 | over HTTP: InvalidRecordPayloadError -> structured 400 (see |
| 752 | object_server.py's exception-to-status mapping around |
| 753 | InvalidRecordPayloadError).""" |
| 754 | data_dir = tmp_path / "data" |
| 755 | write_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD]) |
| 756 | monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir)) |
| 757 | monkeypatch.setenv("DBBASIC_MAX_REQUEST_BYTES", str(object_records.MAX_TSV_FIELD_BYTES + 1_000_000)) |
| 758 | enable_admin_token(monkeypatch) |
| 759 | |
| 760 | over = json.dumps([{"d": "x" * (object_records.MAX_TSV_FIELD_BYTES + 1000)}]) |
| 761 | status, _, payload = request( |
| 762 | "/admin/collections/orders/records", |
| 763 | method="POST", |
| 764 | body=json.dumps({"id": "http-too-big", "items": over}).encode("utf-8"), |
| 765 | headers=auth_headers(), |
| 766 | ) |
| 767 | assert status == 400, payload |
| 768 | assert payload["status"] == "error" |