Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_charset_json_roundtrip_characterization.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_charset_json_roundtrip_characterization.py · 32.5 KB · 768 lines
1"""CHARACTERIZATION tests: does an ARBITRARY JSON payload -- specifically the
2shape plan/vocabulary/66-line-items-spec.md proposes (a document's `items`
3array, compact-`json.dumps`'d into one TSV cell) -- survive storage
4byte-exactly and re-parse deep-equal, through BOTH the record storage layer
5(object_records.py, classic + append storage) AND the HTTP API
6(object_server.py's ASGI app, admin collection-record routes)?
7
8This file is pure characterization of EXISTING behavior. It does not modify
9any production module. Where a case fails, the assertion is written to state
10the correctness property a "safe to hold embedded JSON" substrate SHOULD
11have, and is left to fail (or is downgraded to a soft print+record) -- that
12outcome IS the finding. Do not "fix" a failing assertion here by weakening it
13to match broken behavior.
14
15Companion to tests/test_embedded_json_lines_characterization.py (which
16established the record-layer setup conventions this file reuses, and already
17covers hostile-content-across-physical-lines / offset-index / scale / torn
18tail -- NOT retested here; in particular the "torn tail is quote-blind"
19substrate bug #2 is a KNOWN, DEFERRED finding and is deliberately NOT probed
20by this file). This file's unique angle: full Unicode character-CLASS
21coverage (RTL scripts, emoji+ZWJ+skin-tone+flags, combining marks,
22zero-width chars), JSON structural edge cases (unicode keys, nesting, null/
23bool/large-int/float), the ensure_ascii=True vs False choice, the
24MAX_TSV_FIELD_BYTES cap under multibyte content, idempotent re-serialization,
25and the HTTP API surface, which the companion file never drives at all.
26
27Per instructions: do NOT probe torn-tail. Recent fixes (csv.field_size_limit
28raise + write cap; _oidx_get_row csv-aware) are assumed already covered by
29the companion file's regression guards and are not retested here except as
30incidental background for the size-cap section.
31"""
32
33from __future__ import annotations
34
35import asyncio
36import copy
37import json
38from pathlib import Path
39
40import pytest
41
42import object_records
43import object_server
44
45# Conformance tier: heavy charset characterization -- deselected from the
46# per-commit run (see pyproject 'conformance' marker). Run: pytest -m conformance
47pytestmark = pytest.mark.conformance
48
49ID_FIELD = {"name": "id"}
50ITEMS_FIELD = {"name": "items", "type": "textarea"}
51
52TEST_ADMIN_TOKEN = "unit-test-only-admin-token"
53
54
55# =============================================================================
56# setup helpers (mirror tests/test_embedded_json_lines_characterization.py)
57# =============================================================================
58
59
60def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path:
61 path = data_dir / "schemas" / f"{collection}.json"
62 path.parent.mkdir(parents=True, exist_ok=True)
63 path.write_text(json.dumps({"fields": fields}))
64 return path
65
66
67def write_append_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path:
68 path = data_dir / "schemas" / f"{collection}.json"
69 path.parent.mkdir(parents=True, exist_ok=True)
70 path.write_text(json.dumps({"fields": fields, "storage": "append"}))
71 return path
72
73
74def _clear_caches() -> None:
75 """Cold the in-process record cache + id->offset sidecar so a
76 following read genuinely exercises storage, not a warm hit -- same
77 trick the companion characterization file and object_records' own
78 oidx tests use."""
79 object_records._RECORDS_CACHE.clear()
80 object_records._OIDX_CACHE.clear()
81
82
83# =============================================================================
84# HTTP in-process ASGI driver (mirrors tests/test_object_server.py's
85# asgi_request/request/raw_request helpers exactly, trimmed to what this
86# file needs)
87# =============================================================================
88
89
90def auth_headers():
91 return [("authorization", f"Token {TEST_ADMIN_TOKEN}")]
92
93
94def enable_admin_token(monkeypatch):
95 monkeypatch.setenv("DBBASIC_ADMIN_TOKEN", TEST_ADMIN_TOKEN)
96
97
98async def asgi_request(path, method="GET", query_string="", body=b"", headers=None):
99 messages = []
100
101 async def receive():
102 return {"type": "http.request", "body": body, "more_body": False}
103
104 async def send(message):
105 messages.append(message)
106
107 scope_headers = [(b"accept", b"application/json")]
108 for name, value in headers or []:
109 if isinstance(name, str):
110 name = name.encode("latin-1")
111 if isinstance(value, str):
112 value = value.encode("latin-1")
113 scope_headers.append((name, value))
114
115 scope = {
116 "type": "http",
117 "method": method,
118 "path": path,
119 "query_string": query_string.encode("utf-8"),
120 "headers": scope_headers,
121 "client": ("127.0.0.1", 12345),
122 }
123 await object_server.app(scope, receive, send)
124
125 start = next(m for m in messages if m["type"] == "http.response.start")
126 body_parts = [m.get("body", b"") for m in messages if m["type"] == "http.response.body"]
127 payload = b"".join(body_parts)
128 return start["status"], dict(start["headers"]), payload
129
130
131def request(path, method="GET", query_string="", body=b"", headers=None):
132 status, headers_out, payload = asyncio.run(
133 asgi_request(path, method=method, query_string=query_string, body=body, headers=headers)
134 )
135 return status, headers_out, json.loads(payload.decode("utf-8"))
136
137
138# =============================================================================
139# Realistic + adversarial JSON item-array fixtures
140# =============================================================================
141
142
143def _charclass_string() -> str:
144 """One string mixing every character class the probe cares about."""
145 return (
146 'ASCII "quoted" back\\slash, comma, and a semicolon; '
147 "café résumé naïve " # accented Latin
148 "日本語のテスト文字列 " # CJK
149 "العربية " # Arabic (RTL)
150 "עברית " # Hebrew (RTL)
151 "ȩ́ " # combining acute + cedilla
152 "​‌zero‍width " # ZWSP, ZWNJ, ZWJ
153 "\U0001f469‍\U0001f469‍\U0001f466‍\U0001f466 " # family ZWJ sequence
154 "\U0001f44d\U0001f3fd " # thumbs up + medium skin tone modifier
155 "\U0001f1fa\U0001f1f8 " # flag: US (regional indicator pair)
156 "\U0001f600" # simple emoji
157 )
158
159
160def _realistic_items() -> list[dict]:
161 """The actual embed use case: a document's `items` array, per
162 plan/vocabulary/66-line-items-spec.md's item_fields shape."""
163 return [
164 {
165 "product_id": "prod-001",
166 "description": "Café crème, 12oz — 日本茶セット",
167 "quantity": 2,
168 "unit_price_cents": 1099,
169 "tax_rate_bps": 875,
170 },
171 {
172 "product_id": "prod-002",
173 "description": _charclass_string(),
174 "quantity": 1,
175 "unit_price_cents": 2500,
176 "tax_rate_bps": 0,
177 },
178 {
179 "product_id": "prod-003",
180 "description": "",
181 "quantity": 0,
182 "unit_price_cents": 0,
183 "tax_rate_bps": None,
184 "note": None,
185 "gift_wrapped": True,
186 "backordered": False,
187 },
188 ]
189
190
191def _adversarial_items() -> list:
192 """JSON structural edge cases: unicode keys, deep nesting, empty
193 string, null, booleans, ints (incl. large), floats, a JSON-looking
194 string value, and a value containing literal TSV delimiter chars."""
195 return [
196 {
197 "étoile": "unicode key (accented Latin)",
198 "日本": "unicode key (CJK)",
199 "\U0001f600key": "unicode key (emoji)",
200 },
201 {
202 "nested": {
203 "level2": {
204 "level3": [1, 2, {"level4": [{"level5": "deep"}]}],
205 },
206 "array_of_arrays": [[1, 2], [3, [4, 5, [6, 7]]]],
207 }
208 },
209 {
210 "empty_string": "",
211 "is_null": None,
212 "is_true": True,
213 "is_false": False,
214 "small_int": 42,
215 "large_int": 9223372036854775807, # int64 max
216 "huge_int": 123456789012345678901234567890, # beyond int64
217 "float_val": 19.99,
218 "float_tricky": 0.1,
219 "json_looking_string": '{"not":"parsed","nested":[1,2,3]}',
220 "delimiter_string": "a\tb,c;d\nrepresented-as-escaped-in-json",
221 },
222 ]
223
224
225# =============================================================================
226# 1. Realistic items array -- classic + append, fold + by-id, plus update
227# =============================================================================
228
229
230def test_realistic_items_round_trip_classic_mode_fold_and_by_id(tmp_path):
231 data_dir = tmp_path / "data"
232 write_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD])
233 original = _realistic_items()
234 blob = json.dumps(original)
235
236 object_records.create_collection_record(
237 "orders", {"id": "ord-1", "items": blob}, base_dir=data_dir, roots=[]
238 )
239
240 _clear_caches()
241 all_records = object_records.read_collection_records("orders", base_dir=data_dir, roots=[])
242 assert json.loads(all_records[0]["items"]) == original
243
244 _clear_caches()
245 by_id = object_records.get_collection_record("orders", "ord-1", base_dir=data_dir, roots=[])
246 assert json.loads(by_id["items"]) == original
247
248 # Stored cell must be delimiter-safe: compact json.dumps escapes all
249 # control chars in string VALUES, so no raw tab/newline byte should
250 # appear in the stored blob itself.
251 assert "\t" not in by_id["items"]
252 assert "\n" not in by_id["items"]
253
254
255def test_realistic_items_round_trip_append_mode_fold_and_by_id(tmp_path, monkeypatch):
256 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
257 data_dir = tmp_path / "data"
258 write_append_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD])
259 original = _realistic_items()
260 blob = json.dumps(original)
261
262 object_records.create_collection_record(
263 "orders", {"id": "ord-1", "items": blob}, base_dir=data_dir, roots=[]
264 )
265
266 _clear_caches()
267 all_records = object_records.read_collection_records("orders", base_dir=data_dir, roots=[])
268 assert json.loads(all_records[0]["items"]) == original
269
270 _clear_caches()
271 by_id = object_records.get_collection_record("orders", "ord-1", base_dir=data_dir, roots=[])
272 assert json.loads(by_id["items"]) == original
273 assert "\t" not in by_id["items"]
274 assert "\n" not in by_id["items"]
275
276
277@pytest.mark.parametrize("storage", ["classic", "append"])
278def test_realistic_items_update_one_item_then_reread(tmp_path, monkeypatch, storage):
279 """The document-lines-spec's mutation shape: read the whole array,
280 mutate ONE item, re-serialize the WHOLE array, write it back -- then
281 confirm a cold-cache reread agrees deep-equal."""
282 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
283 data_dir = tmp_path / "data"
284 collection = "orders"
285 if storage == "classic":
286 write_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD])
287 else:
288 write_append_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD])
289
290 original = _realistic_items()
291 object_records.create_collection_record(
292 collection, {"id": "ord-1", "items": json.dumps(original)}, base_dir=data_dir, roots=[]
293 )
294
295 _clear_caches()
296 current = object_records.get_collection_record(collection, "ord-1", base_dir=data_dir, roots=[])
297 items = json.loads(current["items"])
298 items[1]["quantity"] += 5
299 items[1]["description"] = items[1]["description"] + " (updated) " + _charclass_string()
300 expected = copy.deepcopy(items)
301
302 object_records.update_collection_record(
303 collection, "ord-1", {"items": json.dumps(items)}, base_dir=data_dir, roots=[]
304 )
305
306 _clear_caches()
307 after = object_records.get_collection_record(collection, "ord-1", base_dir=data_dir, roots=[])
308 assert json.loads(after["items"]) == expected
309
310
311# =============================================================================
312# 2. Adversarial JSON structural edge cases -- both modes
313# =============================================================================
314
315
316@pytest.mark.parametrize("storage", ["classic", "append"])
317def test_adversarial_structural_edge_cases_round_trip(tmp_path, monkeypatch, storage):
318 """unicode keys, deep nesting, null/bool/int/float, a JSON-looking
319 string value, and a string literally containing TSV delimiter chars
320 (tab/comma/semicolon/newline) -- all inside JSON so json.dumps has
321 already escaped the control chars before the TSV layer ever sees
322 them."""
323 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
324 data_dir = tmp_path / "data"
325 collection = "orders"
326 if storage == "classic":
327 write_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD])
328 else:
329 write_append_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD])
330
331 original = _adversarial_items()
332 blob = json.dumps(original)
333
334 object_records.create_collection_record(
335 collection, {"id": "adv-1", "items": blob}, base_dir=data_dir, roots=[]
336 )
337
338 _clear_caches()
339 all_records = object_records.read_collection_records(collection, base_dir=data_dir, roots=[])
340 loaded_fold = json.loads(all_records[0]["items"])
341
342 _clear_caches()
343 by_id = object_records.get_collection_record(collection, "adv-1", base_dir=data_dir, roots=[])
344 loaded_by_id = json.loads(by_id["items"])
345
346 # unicode keys survive
347 assert loaded_fold[0] == original[0]
348 assert loaded_by_id[0] == original[0]
349 assert set(loaded_by_id[0].keys()) == set(original[0].keys())
350
351 # deep nesting survives
352 assert loaded_fold[1] == original[1]
353 assert loaded_by_id[1] == original[1]
354
355 # null/bool/int survive exactly
356 assert loaded_by_id[2]["is_null"] is None
357 assert loaded_by_id[2]["is_true"] is True
358 assert loaded_by_id[2]["is_false"] is False
359 assert loaded_by_id[2]["small_int"] == 42
360 assert loaded_by_id[2]["large_int"] == 9223372036854775807
361 assert loaded_by_id[2]["huge_int"] == 123456789012345678901234567890
362
363 # a JSON-looking string value must come back as a STRING, not be
364 # double-parsed into a nested object
365 assert isinstance(loaded_by_id[2]["json_looking_string"], str)
366 assert loaded_by_id[2]["json_looking_string"] == original[2]["json_looking_string"]
367
368 # a value containing literal delimiter characters (tab/comma/
369 # semicolon/newline) survives -- these are JSON-string-escaped by
370 # json.dumps, so the TSV layer never sees the raw bytes
371 assert loaded_by_id[2]["delimiter_string"] == original[2]["delimiter_string"]
372
373 assert loaded_fold == original
374 assert loaded_by_id == original
375
376
377def test_float_round_trip_report(tmp_path):
378 """Report-only: do JSON floats survive json.dumps -> TSV cell ->
379 json.loads exactly? (Money doctrine elsewhere in this codebase already
380 mandates integer minor units / cents for currency -- this test exists
381 to CONFIRM that recommendation empirically for the items-array use
382 case, not to relitigate it.)"""
383 data_dir = tmp_path / "data"
384 write_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD])
385 original = [{"float_a": 19.99, "float_b": 0.1, "float_c": 1e300, "float_d": 1.1e-10}]
386 blob = json.dumps(original)
387
388 object_records.create_collection_record(
389 "orders", {"id": "f-1", "items": blob}, base_dir=data_dir, roots=[]
390 )
391 _clear_caches()
392 by_id = object_records.get_collection_record("orders", "f-1", base_dir=data_dir, roots=[])
393 loaded = json.loads(by_id["items"])
394
395 # Python's float repr round-trips via repr()/json for IEEE-754
396 # doubles (json encodes via float.__repr__, which is the shortest
397 # string that round-trips) -- so THIS layer (storage) does not lose
398 # precision. The finding is about REPRESENTATION choice, not this
399 # substrate: any float value is inherently binary-imprecise for
400 # currency math (e.g. 19.99 is not exactly representable), so even
401 # though it round-trips bit-for-bit through storage here, arithmetic
402 # on such floats elsewhere will accumulate error. Report both facts.
403 print(f"\n[float report] storage round-trip exact: {loaded == original}")
404 print(f"[float report] original: {original}")
405 print(f"[float report] loaded: {loaded}")
406 assert loaded == original, (
407 "FINDING: float values did NOT survive storage exactly -- "
408 f"original={original!r} loaded={loaded!r}"
409 )
410 print(
411 "[float report] RECOMMENDATION: storage itself preserves float bits "
412 "exactly (json+repr round-trip), but floats remain the wrong "
413 "representation for money -- integer minor units (cents) per the "
414 "codebase's existing money doctrine, not because storage corrupts "
415 "them, but because binary floats can't exactly represent most "
416 "decimal currency values in the first place (e.g. 19.99), so any "
417 "arithmetic over them (subtotals/tax) will drift."
418 )
419
420
421# =============================================================================
422# 3. ensure_ascii True vs False
423# =============================================================================
424
425
426@pytest.mark.parametrize("storage", ["classic", "append"])
427def test_ensure_ascii_true_default_round_trips_and_is_pure_ascii_cell(tmp_path, monkeypatch, storage):
428 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
429 data_dir = tmp_path / "data"
430 collection = "orders"
431 if storage == "classic":
432 write_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD])
433 else:
434 write_append_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD])
435
436 original = _realistic_items()
437 blob = json.dumps(original) # ensure_ascii=True is the json.dumps default
438 assert blob.isascii(), "sanity: default json.dumps must produce a pure-ASCII string"
439
440 object_records.create_collection_record(
441 collection, {"id": "ascii-1", "items": blob}, base_dir=data_dir, roots=[]
442 )
443 _clear_caches()
444 by_id = object_records.get_collection_record(collection, "ascii-1", base_dir=data_dir, roots=[])
445 assert by_id["items"].isascii(), "stored cell should remain pure ASCII (\\uXXXX escapes only)"
446 assert json.loads(by_id["items"]) == original
447
448
449@pytest.mark.parametrize("storage", ["classic", "append"])
450def test_ensure_ascii_false_round_trips_with_raw_multibyte_in_cell(tmp_path, monkeypatch, storage):
451 """ensure_ascii=False puts raw UTF-8 multibyte (and raw emoji, RTL
452 script, etc.) directly in the TSV cell instead of \\uXXXX escapes.
453 This still round-trips through this substrate (the TSV/CSV layer is
454 UTF-8 text throughout), but the cell is no longer pure ASCII on disk."""
455 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
456 data_dir = tmp_path / "data"
457 collection = "orders"
458 if storage == "classic":
459 write_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD])
460 else:
461 write_append_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD])
462
463 original = _realistic_items()
464 blob = json.dumps(original, ensure_ascii=False)
465 assert not blob.isascii(), "sanity: ensure_ascii=False should emit raw multibyte here"
466
467 object_records.create_collection_record(
468 collection, {"id": "nonascii-1", "items": blob}, base_dir=data_dir, roots=[]
469 )
470 _clear_caches()
471 by_id = object_records.get_collection_record(collection, "nonascii-1", base_dir=data_dir, roots=[])
472 assert not by_id["items"].isascii()
473 assert json.loads(by_id["items"]) == original
474 # No raw tab/newline bytes even with ensure_ascii=False -- control-char
475 # escaping in JSON strings is independent of the ensure_ascii setting
476 # (only affects non-ASCII codepoints >= 0x80, not control chars).
477 assert "\t" not in by_id["items"]
478 assert "\n" not in by_id["items"]
479
480
481# =============================================================================
482# 4. Size cap interaction with multibyte content
483# =============================================================================
484
485
486def test_size_cap_multibyte_content_byte_accurate(tmp_path):
487 """A JSON array whose serialized bytes approach MAX_TSV_FIELD_BYTES,
488 built from multibyte (CJK) content so char-count and byte-count
489 diverge sharply -- verify the cap is enforced on BYTES (as
490 MAX_TSV_FIELD_BYTES's docstring/companion test claims), not characters,
491 and that a comfortably-under-cap multibyte blob still round-trips.
492
493 Uses ensure_ascii=False so each CJK codepoint costs its real 3 raw
494 UTF-8 bytes in the cell. NOTE (finding, surfaced in the report): with
495 the codebase's actual convention -- default json.dumps, ensure_ascii=
496 True -- the SAME logical content costs 6 ASCII bytes per CJK
497 codepoint (a "\\uXXXX" escape) instead of 3 raw UTF-8 bytes, i.e.
498 roughly DOUBLE the serialized size for heavily non-ASCII item text.
499 That inflation eats into the effective headroom under MAX_TSV_FIELD_
500 BYTES / max_items caps for non-Latin-script content specifically, and
501 was surprising enough during test construction (a naive byte budget
502 silently produced a blob 2x the intended size) that it's worth
503 reporting explicitly rather than treating as obvious."""
504 data_dir = tmp_path / "data"
505 write_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD])
506
507 # Each item's description is a CJK string (3 bytes/char in UTF-8) --
508 # build enough items to land close to, but safely under, the cap.
509 cjk_char = "日" # 3 bytes UTF-8 (raw); 6 bytes if json.dumps escapes it
510 target_bytes = object_records.MAX_TSV_FIELD_BYTES - 2048 # headroom for JSON syntax overhead
511 # each item is roughly: {"d":"...."} -- budget item body length below
512 item_overhead = len('{"d":""}')
513 n_items = 50
514 chars_per_item = (target_bytes // n_items - item_overhead) // 3
515 items = [{"d": cjk_char * chars_per_item} for _ in range(n_items)]
516 blob = json.dumps(items, ensure_ascii=False)
517 blob_bytes = len(blob.encode("utf-8"))
518 assert blob_bytes < object_records.MAX_TSV_FIELD_BYTES, (
519 f"test construction sanity: blob_bytes={blob_bytes} must stay under cap "
520 f"{object_records.MAX_TSV_FIELD_BYTES} to prove the under-cap case"
521 )
522 # Same logical content, but serialized the codebase's actual way
523 # (ensure_ascii=True, the json.dumps default used everywhere else in
524 # this repo) -- report the inflation factor.
525 blob_ascii_escaped = json.dumps(items)
526 blob_ascii_escaped_bytes = len(blob_ascii_escaped.encode("utf-8"))
527 print(f"\n[size-cap] multibyte blob (ensure_ascii=False, raw UTF-8): "
528 f"{blob_bytes} bytes (cap={object_records.MAX_TSV_FIELD_BYTES}), {len(blob)} chars")
529 print(f"[size-cap] SAME content (ensure_ascii=True, \\uXXXX escapes): "
530 f"{blob_ascii_escaped_bytes} bytes -- "
531 f"{blob_ascii_escaped_bytes / blob_bytes:.2f}x larger")
532
533 object_records.create_collection_record(
534 "orders", {"id": "big-multibyte", "items": blob}, base_dir=data_dir, roots=[]
535 )
536 _clear_caches()
537 by_id = object_records.get_collection_record("orders", "big-multibyte", base_dir=data_dir, roots=[])
538 assert json.loads(by_id["items"]) == items
539
540 # Now push a multibyte blob just OVER the byte cap (but with a
541 # char-count that would be well under it) -- the cap must reject on
542 # BYTES, not code points, or this write would wrongly succeed.
543 over_target_bytes = object_records.MAX_TSV_FIELD_BYTES + 300
544 n_items_over = 60
545 chars_per_item_over = (over_target_bytes // n_items_over - item_overhead) // 3 + 50
546 over_items = [{"d": cjk_char * chars_per_item_over} for _ in range(n_items_over)]
547 over_blob = json.dumps(over_items, ensure_ascii=False)
548 over_bytes = len(over_blob.encode("utf-8"))
549 over_chars = len(over_blob)
550 assert over_bytes > object_records.MAX_TSV_FIELD_BYTES, (
551 f"test construction sanity: need over_bytes > cap, got {over_bytes}"
552 )
553 print(f"[size-cap] over-cap multibyte blob: {over_bytes} bytes, {over_chars} chars "
554 f"(chars < bytes: {over_chars < object_records.MAX_TSV_FIELD_BYTES})")
555
556 with pytest.raises(object_records.InvalidRecordPayloadError):
557 object_records.create_collection_record(
558 "orders", {"id": "too-big-multibyte", "items": over_blob}, base_dir=data_dir, roots=[]
559 )
560
561
562# =============================================================================
563# 5. Idempotent re-serialization (compact-form stability)
564# =============================================================================
565
566
567@pytest.mark.parametrize("storage", ["classic", "append"])
568def test_idempotent_reserialization_stability(tmp_path, monkeypatch, storage):
569 """json.loads(read) then json.dumps(...) again (compact form, same
570 separators/key order as the original write) must reproduce the exact
571 stored blob byte-for-byte -- i.e. round-tripping through this layer
572 introduces no drift a naive re-save-without-edits would compound."""
573 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
574 data_dir = tmp_path / "data"
575 collection = "orders"
576 if storage == "classic":
577 write_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD])
578 else:
579 write_append_schema(data_dir, collection, [ID_FIELD, ITEMS_FIELD])
580
581 original = _realistic_items() + _adversarial_items()
582 blob = json.dumps(original)
583
584 object_records.create_collection_record(
585 collection, {"id": "stable-1", "items": blob}, base_dir=data_dir, roots=[]
586 )
587 _clear_caches()
588 by_id = object_records.get_collection_record(collection, "stable-1", base_dir=data_dir, roots=[])
589 stored = by_id["items"]
590
591 reparsed = json.loads(stored)
592 reserialized = json.dumps(reparsed)
593 assert reserialized == stored, (
594 "idempotency FINDING: re-serializing the parsed value did not "
595 "reproduce the stored blob byte-for-byte"
596 )
597 assert reserialized == blob
598
599
600# =============================================================================
601# 6. HTTP API path -- admin collection-record routes
602# =============================================================================
603
604
605def test_http_create_and_read_realistic_items_round_trip_classic(tmp_path, monkeypatch):
606 data_dir = tmp_path / "data"
607 write_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD])
608 monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir))
609 enable_admin_token(monkeypatch)
610
611 original = _realistic_items()
612 blob = json.dumps(original)
613
614 create_status, _, create_payload = request(
615 "/admin/collections/orders/records",
616 method="POST",
617 body=json.dumps({"id": "http-1", "items": blob}).encode("utf-8"),
618 headers=auth_headers(),
619 )
620 assert create_status == 201, create_payload
621 assert json.loads(create_payload["record"]["items"]) == original
622
623 _clear_caches()
624 list_status, _, list_payload = request(
625 "/admin/collections/orders/records", headers=auth_headers()
626 )
627 assert list_status == 200
628 fold_row = next(r for r in list_payload["records"] if r["id"] == "http-1")
629 assert json.loads(fold_row["items"]) == original
630
631 _clear_caches()
632 get_status, _, get_payload = request(
633 "/admin/collections/orders/records/http-1", headers=auth_headers()
634 )
635 assert get_status == 200
636 assert json.loads(get_payload["record"]["items"]) == original
637
638
639def test_http_create_and_read_realistic_items_round_trip_append(tmp_path, monkeypatch):
640 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
641 data_dir = tmp_path / "data"
642 write_append_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD])
643 monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir))
644 enable_admin_token(monkeypatch)
645
646 original = _adversarial_items()
647 blob = json.dumps(original)
648
649 create_status, _, create_payload = request(
650 "/admin/collections/orders/records",
651 method="POST",
652 body=json.dumps({"id": "http-adv-1", "items": blob}).encode("utf-8"),
653 headers=auth_headers(),
654 )
655 assert create_status == 201, create_payload
656 assert json.loads(create_payload["record"]["items"]) == original
657
658 _clear_caches()
659 get_status, _, get_payload = request(
660 "/admin/collections/orders/records/http-adv-1", headers=auth_headers()
661 )
662 assert get_status == 200
663 assert json.loads(get_payload["record"]["items"]) == original
664
665
666def test_http_update_one_item_then_reread(tmp_path, monkeypatch):
667 """The HTTP path's version of the update-an-item-then-reread probe:
668 POST create, GET, mutate client-side, PUT the whole items array back,
669 GET again (cold cache) and confirm deep equality."""
670 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
671 data_dir = tmp_path / "data"
672 write_append_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD])
673 monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir))
674 enable_admin_token(monkeypatch)
675
676 original = _realistic_items()
677 request(
678 "/admin/collections/orders/records",
679 method="POST",
680 body=json.dumps({"id": "http-upd-1", "items": json.dumps(original)}).encode("utf-8"),
681 headers=auth_headers(),
682 )
683
684 _clear_caches()
685 get_status, _, get_payload = request(
686 "/admin/collections/orders/records/http-upd-1", headers=auth_headers()
687 )
688 assert get_status == 200
689 items = json.loads(get_payload["record"]["items"])
690 items[0]["description"] = _charclass_string() + " EDITED"
691 items[0]["quantity"] = 999
692 expected = copy.deepcopy(items)
693
694 update_status, _, update_payload = request(
695 "/admin/collections/orders/records/http-upd-1",
696 method="PUT",
697 body=json.dumps({"items": json.dumps(items)}).encode("utf-8"),
698 headers=auth_headers(),
699 )
700 assert update_status == 200, update_payload
701 assert json.loads(update_payload["record"]["items"]) == expected
702
703 _clear_caches()
704 reget_status, _, reget_payload = request(
705 "/admin/collections/orders/records/http-upd-1", headers=auth_headers()
706 )
707 assert reget_status == 200
708 assert json.loads(reget_payload["record"]["items"]) == expected
709
710
711def test_http_request_body_cap_is_far_below_the_field_cap_FINDING(tmp_path, monkeypatch):
712 """FINDING: object_server's own HTTP request-body-size gate
713 (DBBASIC_MAX_REQUEST_BYTES, default 1 MiB -- see
714 object_server.DEFAULT_MAX_REQUEST_BYTES) is enforced BEFORE the JSON
715 body is even parsed, and its default (1,048,576 bytes) is 16x SMALLER
716 than object_records.MAX_TSV_FIELD_BYTES (16 MiB). A single `items`
717 field anywhere near the record-layer cap can never actually reach
718 create_collection_record's InvalidRecordPayloadError check via HTTP
719 under default configuration -- it is rejected first, at the transport
720 layer, with 413 "Request body too large" (a different status/shape
721 than the record-layer's 400 InvalidRecordPayloadError). This is not
722 unsafe (both reject the oversize write), but it means the HTTP path's
723 effective per-field ceiling for `items` is ~1 MiB by default, not
724 16 MiB, and any client-facing error handling for "items too large"
725 over HTTP must expect 413, not 400, unless the operator raises
726 DBBASIC_MAX_REQUEST_BYTES."""
727 data_dir = tmp_path / "data"
728 write_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD])
729 monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir))
730 enable_admin_token(monkeypatch)
731
732 # A field comfortably PAST MAX_TSV_FIELD_BYTES, well past the
733 # object_server.DEFAULT_MAX_REQUEST_BYTES (1 MiB) transport gate too.
734 over = json.dumps([{"d": "x" * (object_records.MAX_TSV_FIELD_BYTES + 1000)}])
735 status, _, payload = request(
736 "/admin/collections/orders/records",
737 method="POST",
738 body=json.dumps({"id": "http-too-big", "items": over}).encode("utf-8"),
739 headers=auth_headers(),
740 )
741 print(f"\n[http-cap FINDING] default DBBASIC_MAX_REQUEST_BYTES gate hit first: "
742 f"status={status} payload={payload}")
743 assert status == 413, payload
744 assert payload["status"] == "error"
745
746
747def test_http_oversize_items_payload_rejected_when_request_cap_raised(tmp_path, monkeypatch):
748 """With the transport-level request-body cap explicitly raised above
749 MAX_TSV_FIELD_BYTES (DBBASIC_MAX_REQUEST_BYTES), the request now
750 reaches record-layer validation and the FIELD cap itself is exercised
751 over HTTP: InvalidRecordPayloadError -> structured 400 (see
752 object_server.py's exception-to-status mapping around
753 InvalidRecordPayloadError)."""
754 data_dir = tmp_path / "data"
755 write_schema(data_dir, "orders", [ID_FIELD, ITEMS_FIELD])
756 monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir))
757 monkeypatch.setenv("DBBASIC_MAX_REQUEST_BYTES", str(object_records.MAX_TSV_FIELD_BYTES + 1_000_000))
758 enable_admin_token(monkeypatch)
759
760 over = json.dumps([{"d": "x" * (object_records.MAX_TSV_FIELD_BYTES + 1000)}])
761 status, _, payload = request(
762 "/admin/collections/orders/records",
763 method="POST",
764 body=json.dumps({"id": "http-too-big", "items": over}).encode("utf-8"),
765 headers=auth_headers(),
766 )
767 assert status == 400, payload
768 assert payload["status"] == "error"