Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_object_records.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_object_records.py · 124.1 KB · 3064 lines
1import csv
2import json
3import random
4import unittest.mock
5from pathlib import Path
6from uuid import UUID
7
8import pytest
9
10import object_collections
11import object_permissions
12import object_record_changes
13import object_records
14import object_schemas
15
16
17def write_source(path: Path, content: str = "def GET(request):\n return {}\n") -> Path:
18 path.parent.mkdir(parents=True, exist_ok=True)
19 path.write_text(content)
20 return path
21
22
23def write_records(data_dir: Path, collection: str, content: str) -> Path:
24 path = data_dir / "collections" / collection / "records.tsv"
25 path.parent.mkdir(parents=True, exist_ok=True)
26 path.write_text(content)
27 return path
28
29
30def write_schema(data_dir: Path, collection: str, fields: list[dict] | None = None) -> Path:
31 path = data_dir / "schemas" / f"{collection}.json"
32 path.parent.mkdir(parents=True, exist_ok=True)
33 path.write_text(json.dumps({"fields": fields or [{"name": "id"}]}))
34 return path
35
36
37def test_list_collection_records_reads_tsv_with_pagination(tmp_path):
38 data_dir = tmp_path / "data"
39 write_records(
40 data_dir,
41 "contacts",
42 "id\tfirst_name\tlast_name\n"
43 "c1\tAda\tLovelace\n"
44 "c2\tGrace\tHopper\n"
45 "c3\tKatherine\tJohnson\n",
46 )
47
48 result = object_records.list_collection_records(
49 "contacts",
50 base_dir=data_dir,
51 roots=[],
52 limit=2,
53 offset=1,
54 )
55
56 assert result == {
57 "collection": "contacts",
58 "records": [
59 {"id": "c2", "first_name": "Grace", "last_name": "Hopper"},
60 {"id": "c3", "first_name": "Katherine", "last_name": "Johnson"},
61 ],
62 "count": 2,
63 "total": 3,
64 "limit": 2,
65 "offset": 1,
66 "has_more": False,
67 }
68
69
70def test_get_collection_record_returns_record_by_id(tmp_path):
71 data_dir = tmp_path / "data"
72 write_records(data_dir, "contacts", "id\tname\nc1\tAda\nc2\tGrace\n")
73
74 record = object_records.get_collection_record("contacts", "c2", base_dir=data_dir, roots=[])
75
76 assert record == {"id": "c2", "name": "Grace"}
77
78
79def test_create_collection_record_appends_row_and_new_fields(tmp_path):
80 data_dir = tmp_path / "data"
81 write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
82
83 record = object_records.create_collection_record(
84 "contacts",
85 {"id": "c2", "name": "Grace", "email": "grace@example.com"},
86 base_dir=data_dir,
87 roots=[],
88 )
89
90 assert record == {"id": "c2", "name": "Grace", "email": "grace@example.com"}
91 assert object_records.list_collection_records("contacts", base_dir=data_dir, roots=[])["records"] == [
92 {"id": "c1", "name": "Ada", "email": ""},
93 {"id": "c2", "name": "Grace", "email": "grace@example.com"},
94 ]
95
96
97def test_create_collection_record_generates_uuid_id_when_missing(tmp_path):
98 data_dir = tmp_path / "data"
99 write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
100
101 record = object_records.create_collection_record(
102 "contacts",
103 {"name": "Grace"},
104 base_dir=data_dir,
105 roots=[],
106 )
107
108 parsed = UUID(record["id"])
109 assert parsed.version == 4
110 assert record["name"] == "Grace"
111 assert object_records.get_collection_record(
112 "contacts",
113 record["id"],
114 base_dir=data_dir,
115 roots=[],
116 ) == record
117
118
119def test_create_collection_record_can_start_schema_backed_collection(tmp_path):
120 data_dir = tmp_path / "data"
121 write_schema(data_dir, "invoices")
122
123 record = object_records.create_collection_record(
124 "invoices",
125 {"id": "i1", "total": 120},
126 base_dir=data_dir,
127 roots=[],
128 )
129
130 assert record == {"id": "i1", "total": "120"}
131 assert (data_dir / "collections" / "invoices" / "records.tsv").read_text() == (
132 "id\ttotal\n"
133 "i1\t120\n"
134 )
135
136
137def test_create_collection_record_applies_schema_defaults_and_allows_extra_fields(tmp_path):
138 data_dir = tmp_path / "data"
139 write_schema(
140 data_dir,
141 "invoices",
142 [
143 {"name": "id"},
144 {"name": "invoice_date", "type": "date", "required": True},
145 {"name": "status", "type": "enum", "enum": ["draft", "sent"], "default": "draft"},
146 {"name": "quantity", "type": "integer", "validation": {"min": 1, "max": 20}},
147 {"name": "notes", "validation": {"min_length": 3, "max_length": 20}},
148 ],
149 )
150
151 record = object_records.create_collection_record(
152 "invoices",
153 {
154 "id": "i1",
155 "invoice_date": "2026-04-08",
156 "quantity": 3,
157 "notes": "Thanks",
158 "extra_field": "still allowed",
159 },
160 base_dir=data_dir,
161 roots=[],
162 )
163
164 assert record == {
165 "id": "i1",
166 "invoice_date": "2026-04-08",
167 "quantity": "3",
168 "notes": "Thanks",
169 "extra_field": "still allowed",
170 "status": "draft",
171 }
172
173
174@pytest.mark.parametrize(
175 ("payload", "message"),
176 [
177 ({"id": "i1", "status": "draft"}, "invoice_date' is required"),
178 ({"id": "i1", "invoice_date": "bad-date", "status": "draft"}, "invoice_date' must be a date"),
179 ({"id": "i1", "invoice_date": "2026-04-08", "status": "paid"}, "status' must be one of"),
180 ({"id": "i1", "invoice_date": "2026-04-08", "quantity": "0"}, "quantity' is below min"),
181 ({"id": "i1", "invoice_date": "2026-04-08", "quantity": "3.5"}, "quantity' must be an integer"),
182 ({"id": "i1", "invoice_date": "2026-04-08", "total": "100"}, "total' is computed"),
183 ],
184)
185def test_create_collection_record_validates_schema_fields(tmp_path, payload, message):
186 data_dir = tmp_path / "data"
187 write_schema(
188 data_dir,
189 "invoices",
190 [
191 {"name": "id"},
192 {"name": "invoice_date", "type": "date", "required": True},
193 {"name": "status", "type": "enum", "enum": ["draft", "sent"]},
194 {"name": "quantity", "type": "integer", "validation": {"min": 1, "max": 20}},
195 {"name": "total", "type": "computed", "computed": "sum(line_items)"},
196 ],
197 )
198
199 with pytest.raises(object_records.InvalidRecordPayloadError, match=message):
200 object_records.create_collection_record(
201 "invoices",
202 payload,
203 base_dir=data_dir,
204 roots=[],
205 )
206
207
208def test_create_collection_record_rejects_duplicate_id(tmp_path):
209 data_dir = tmp_path / "data"
210 write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
211
212 with pytest.raises(object_records.DuplicateRecordIdError):
213 object_records.create_collection_record(
214 "contacts",
215 {"id": "c1", "name": "Duplicate"},
216 base_dir=data_dir,
217 roots=[],
218 )
219
220
221def test_update_collection_record_merges_changes_and_preserves_fields(tmp_path):
222 data_dir = tmp_path / "data"
223 write_records(data_dir, "contacts", "id\tname\temail\nc1\tAda\tada@example.com\n")
224
225 record = object_records.update_collection_record(
226 "contacts",
227 "c1",
228 {"name": "Ada Lovelace", "phone": "555-0100"},
229 base_dir=data_dir,
230 roots=[],
231 )
232
233 assert record == {
234 "id": "c1",
235 "name": "Ada Lovelace",
236 "email": "ada@example.com",
237 "phone": "555-0100",
238 }
239 assert object_records.get_collection_record("contacts", "c1", base_dir=data_dir, roots=[]) == record
240
241
242def test_boolean_fields_are_stored_canonically(tmp_path):
243 data_dir = tmp_path / "data"
244 write_schema(
245 data_dir,
246 "notes",
247 [
248 {"name": "id"},
249 {"name": "content"},
250 {"name": "is_public", "type": "boolean", "default": "false"},
251 ],
252 )
253 write_records(data_dir, "notes", "id\tcontent\tis_public\n")
254
255 for record_id, submitted in (("n1", "True"), ("n2", "YES"), ("n3", "1")):
256 record = object_records.create_collection_record(
257 "notes",
258 {"id": record_id, "content": "x", "is_public": submitted},
259 base_dir=data_dir,
260 roots=[],
261 )
262 assert record["is_public"] == "true"
263
264 defaulted = object_records.create_collection_record(
265 "notes", {"id": "n4", "content": "x"}, base_dir=data_dir, roots=[]
266 )
267 assert defaulted["is_public"] == "false"
268
269 updated = object_records.update_collection_record(
270 "notes", "n1", {"is_public": "OFF"}, base_dir=data_dir, roots=[]
271 )
272 assert updated["is_public"] == "false"
273
274
275def test_enum_transitions_are_enforced_on_update(tmp_path):
276 data_dir = tmp_path / "data"
277 write_schema(
278 data_dir,
279 "tasks",
280 [
281 {"name": "id"},
282 {"name": "title"},
283 {
284 "name": "status",
285 "type": "enum",
286 "default": "open",
287 "enum": ["open", "assigned", "done", "cancelled"],
288 "transitions": {
289 "open": ["assigned", "cancelled"],
290 "assigned": ["done", "open", "cancelled"],
291 },
292 },
293 ],
294 )
295 write_records(data_dir, "tasks", "id\ttitle\tstatus\nt1\tShip it\topen\n")
296
297 with pytest.raises(
298 object_records.InvalidRecordPayloadError,
299 match="cannot move from 'open' to 'done'",
300 ):
301 object_records.update_collection_record(
302 "tasks", "t1", {"status": "done"}, base_dir=data_dir, roots=[]
303 )
304
305 record = object_records.update_collection_record(
306 "tasks", "t1", {"status": "assigned"}, base_dir=data_dir, roots=[]
307 )
308 assert record["status"] == "assigned"
309
310 record = object_records.update_collection_record(
311 "tasks", "t1", {"status": "done"}, base_dir=data_dir, roots=[]
312 )
313 assert record["status"] == "done"
314
315 # done is not in the transitions map, so it is terminal.
316 with pytest.raises(object_records.InvalidRecordPayloadError, match="allowed: none"):
317 object_records.update_collection_record(
318 "tasks", "t1", {"status": "open"}, base_dir=data_dir, roots=[]
319 )
320
321 # Unrelated updates to a record sitting in a terminal state stay fine.
322 record = object_records.update_collection_record(
323 "tasks", "t1", {"title": "Shipped"}, base_dir=data_dir, roots=[]
324 )
325 assert record["title"] == "Shipped"
326
327
328def _write_guarded_ticket_schema(data_dir):
329 write_schema(
330 data_dir,
331 "tickets",
332 [
333 {"name": "id"},
334 {"name": "owner_id"},
335 {
336 "name": "status",
337 "type": "enum",
338 "enum": ["open", "closed", "reopened"],
339 "transitions": {
340 "open": ["closed"],
341 "closed": [{"to": "reopened", "when": {"owner_id": "$user_id"}}],
342 },
343 },
344 ],
345 )
346
347
348def test_guarded_transition_without_subject_enforces_validity_but_not_guard(tmp_path):
349 """Direct library callers (daemon, CLI, tests) that don't pass a
350 transition_subject are trusted callers: a guarded move still has to be
351 in the list, but the guard's 'when' clause is not enforced."""
352 data_dir = tmp_path / "data"
353 _write_guarded_ticket_schema(data_dir)
354 write_records(data_dir, "tickets", "id\towner_id\tstatus\nt1\t7\tclosed\n")
355
356 # No subject: the guarded move is valid (present in the list) even
357 # though the record's owner_id doesn't match anyone in particular.
358 record = object_records.update_collection_record(
359 "tickets", "t1", {"status": "reopened"}, base_dir=data_dir, roots=[]
360 )
361 assert record["status"] == "reopened"
362
363 # A move that isn't in the list at all is still rejected outright.
364 with pytest.raises(object_records.InvalidRecordPayloadError):
365 object_records.update_collection_record(
366 "tickets", "t1", {"status": "closed"}, base_dir=data_dir, roots=[]
367 )
368
369
370def test_guarded_transition_with_subject_allows_matching_and_denies_others(tmp_path):
371 data_dir = tmp_path / "data"
372 _write_guarded_ticket_schema(data_dir)
373 write_records(data_dir, "tickets", "id\towner_id\tstatus\nt1\t7\tclosed\nt2\t7\tclosed\n")
374
375 owner = object_permissions.PermissionSubject(user_id="7")
376 other = object_permissions.PermissionSubject(user_id="8")
377
378 with pytest.raises(
379 object_records.TransitionNotAllowedError,
380 match="cannot move from 'closed' to 'reopened'",
381 ):
382 object_records.update_collection_record(
383 "tickets", "t1", {"status": "reopened"}, base_dir=data_dir, roots=[],
384 transition_subject=other,
385 )
386
387 record = object_records.update_collection_record(
388 "tickets", "t2", {"status": "reopened"}, base_dir=data_dir, roots=[],
389 transition_subject=owner,
390 )
391 assert record["status"] == "reopened"
392
393 # Plain string entries (open -> closed here) stay unguarded regardless
394 # of the subject passed in.
395 object_records.create_collection_record(
396 "tickets", {"id": "t3", "owner_id": "7", "status": "open"}, base_dir=data_dir, roots=[]
397 )
398 record = object_records.update_collection_record(
399 "tickets", "t3", {"status": "closed"}, base_dir=data_dir, roots=[],
400 transition_subject=other,
401 )
402 assert record["status"] == "closed"
403
404
405def test_update_collection_record_validates_final_schema_record(tmp_path):
406 data_dir = tmp_path / "data"
407 write_schema(
408 data_dir,
409 "invoices",
410 [
411 {"name": "id"},
412 {"name": "customer_id", "type": "relation", "required": True},
413 {"name": "paid", "type": "boolean"},
414 {"name": "total", "type": "computed", "computed": "sum(line_items)"},
415 ],
416 )
417 write_records(data_dir, "invoices", "id\tcustomer_id\tpaid\n"
418 "i1\tc1\tfalse\n")
419
420 record = object_records.update_collection_record(
421 "invoices",
422 "i1",
423 {"paid": "true"},
424 base_dir=data_dir,
425 roots=[],
426 )
427
428 assert record == {"id": "i1", "customer_id": "c1", "paid": "true"}
429 with pytest.raises(object_records.InvalidRecordPayloadError, match="customer_id' is required"):
430 object_records.update_collection_record(
431 "invoices",
432 "i1",
433 {"customer_id": ""},
434 base_dir=data_dir,
435 roots=[],
436 )
437 with pytest.raises(object_records.InvalidRecordPayloadError, match="total' is computed"):
438 object_records.update_collection_record(
439 "invoices",
440 "i1",
441 {"total": "100"},
442 base_dir=data_dir,
443 roots=[],
444 )
445
446
447def test_update_collection_record_rejects_id_change(tmp_path):
448 data_dir = tmp_path / "data"
449 write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
450
451 with pytest.raises(object_records.InvalidRecordPayloadError, match="id cannot be changed"):
452 object_records.update_collection_record(
453 "contacts",
454 "c1",
455 {"id": "c2", "name": "Grace"},
456 base_dir=data_dir,
457 roots=[],
458 )
459
460
461def test_delete_collection_record_removes_row_and_returns_deleted_record(tmp_path):
462 data_dir = tmp_path / "data"
463 write_records(data_dir, "contacts", "id\tname\nc1\tAda\nc2\tGrace\n")
464
465 record = object_records.delete_collection_record("contacts", "c1", base_dir=data_dir, roots=[])
466
467 assert record == {"id": "c1", "name": "Ada"}
468 assert object_records.list_collection_records("contacts", base_dir=data_dir, roots=[])["records"] == [
469 {"id": "c2", "name": "Grace"},
470 ]
471
472
473def test_record_payload_rejects_unsafe_fields_and_non_scalar_values():
474 with pytest.raises(object_records.InvalidRecordPayloadError, match="field name"):
475 object_records.normalize_record_payload({"bad\tfield": "value"})
476
477 with pytest.raises(object_records.InvalidRecordPayloadError, match="scalar"):
478 object_records.normalize_record_payload({"id": "c1", "tags": ["vip"]})
479
480
481def test_schema_backed_collection_without_records_returns_empty_list(tmp_path):
482 data_dir = tmp_path / "data"
483 write_schema(data_dir, "invoices")
484
485 result = object_records.list_collection_records("invoices", base_dir=data_dir, roots=[])
486
487 assert result == {
488 "collection": "invoices",
489 "records": [],
490 "count": 0,
491 "total": 0,
492 "limit": 100,
493 "offset": 0,
494 "has_more": False,
495 }
496
497
498def test_object_backed_collection_without_records_returns_empty_list(tmp_path):
499 data_dir = tmp_path / "data"
500 root = tmp_path / "objects"
501 write_source(root / "contacts" / "directory.py")
502
503 result = object_records.list_collection_records("contacts", base_dir=data_dir, roots=[root])
504
505 assert result["records"] == []
506 assert result["total"] == 0
507
508
509def test_get_collection_record_rejects_missing_record(tmp_path):
510 data_dir = tmp_path / "data"
511 write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
512
513 with pytest.raises(object_records.RecordNotFoundError):
514 object_records.get_collection_record("contacts", "missing", base_dir=data_dir, roots=[])
515
516
517def test_list_collection_records_rejects_missing_collection(tmp_path):
518 with pytest.raises(object_collections.CollectionNotFoundError):
519 object_records.list_collection_records("missing", base_dir=tmp_path / "data", roots=[])
520
521
522def test_collection_records_reject_unsafe_names(tmp_path):
523 with pytest.raises(object_collections.InvalidCollectionNameError):
524 object_records.list_collection_records("../bad", base_dir=tmp_path / "data", roots=[])
525
526 data_dir = tmp_path / "data"
527 write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
528 with pytest.raises(object_records.InvalidRecordIdError):
529 object_records.get_collection_record("contacts", "../bad", base_dir=data_dir, roots=[])
530
531
532def test_records_file_requires_id_column(tmp_path):
533 data_dir = tmp_path / "data"
534 write_records(data_dir, "contacts", "name\nAda\n")
535
536 with pytest.raises(ValueError, match="id column"):
537 object_records.list_collection_records("contacts", base_dir=data_dir, roots=[])
538
539
540def test_records_file_rejects_duplicate_fields(tmp_path):
541 data_dir = tmp_path / "data"
542 write_records(data_dir, "contacts", "id\tname\tname\nc1\tAda\tLovelace\n")
543
544 with pytest.raises(ValueError, match="duplicate"):
545 object_records.list_collection_records("contacts", base_dir=data_dir, roots=[])
546
547
548def test_records_file_rejects_extra_row_fields(tmp_path):
549 data_dir = tmp_path / "data"
550 write_records(data_dir, "contacts", "id\tname\nc1\tAda\textra\n")
551
552 with pytest.raises(ValueError, match="extra fields"):
553 object_records.list_collection_records("contacts", base_dir=data_dir, roots=[])
554
555
556def test_iter_record_collections_lists_safe_record_directories(tmp_path):
557 data_dir = tmp_path / "data"
558 write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
559 write_records(data_dir, "bad-name", "id\tname\nbad\tBad\n")
560
561 assert object_records.iter_record_collections(data_dir) == ["contacts"]
562
563
564def test_relation_fields_require_existing_target_records(tmp_path):
565 write_records(tmp_path, "projects", "id\tname\np1\tWebsite\n")
566 schema_file = tmp_path / "schemas" / "tasks.json"
567 schema_file.parent.mkdir(parents=True, exist_ok=True)
568 schema_file.write_text(
569 json.dumps(
570 {
571 "fields": [
572 {"name": "id"},
573 {"name": "title", "required": True},
574 {"name": "project_id", "relation": {"collection": "projects", "display_field": "name"}},
575 ]
576 }
577 )
578 )
579
580 created = object_records.create_collection_record(
581 "tasks",
582 {"id": "t1", "title": "Launch", "project_id": "p1"},
583 base_dir=tmp_path,
584 )
585 assert created["project_id"] == "p1"
586
587 with pytest.raises(object_records.InvalidRecordPayloadError) as missing:
588 object_records.create_collection_record(
589 "tasks",
590 {"id": "t2", "title": "Broken", "project_id": "p999"},
591 base_dir=tmp_path,
592 )
593 assert "missing record: projects/p999" in str(missing.value)
594
595 # Empty relation values stay allowed unless the field is required.
596 optional = object_records.create_collection_record(
597 "tasks",
598 {"id": "t3", "title": "No project"},
599 base_dir=tmp_path,
600 )
601 assert optional["project_id"] == ""
602
603
604def test_relation_accepts_string_shorthand_and_rejects_bad_shapes(tmp_path):
605 write_records(tmp_path, "projects", "id\tname\np1\tWebsite\n")
606 schema_file = tmp_path / "schemas" / "tasks.json"
607 schema_file.parent.mkdir(parents=True, exist_ok=True)
608 schema_file.write_text(
609 json.dumps({"fields": [{"name": "id"}, {"name": "project_id", "relation": "projects"}]})
610 )
611
612 created = object_records.create_collection_record(
613 "tasks", {"id": "t1", "project_id": "p1"}, base_dir=tmp_path
614 )
615 assert created["project_id"] == "p1"
616
617 schema_file.write_text(
618 json.dumps({"fields": [{"name": "id"}, {"name": "project_id", "relation": 42}]})
619 )
620 with pytest.raises(object_records.InvalidRecordPayloadError):
621 object_records.create_collection_record(
622 "tasks", {"id": "t2", "project_id": "p1"}, base_dir=tmp_path
623 )
624
625
626def test_created_at_is_server_set_on_create(tmp_path):
627 data_dir = tmp_path / "data"
628 write_schema(
629 data_dir,
630 "links",
631 [
632 {"name": "id"},
633 {"name": "title", "required": True},
634 {"name": "created_at", "type": "datetime", "read_only": True},
635 ],
636 )
637 write_records(data_dir, "links", "id\ttitle\tcreated_at\n")
638
639 # client omits created_at -> server fills it, read_only is not violated
640 rec = object_records.create_collection_record(
641 "links", {"id": "l1", "title": "x"}, base_dir=data_dir, roots=[]
642 )
643 assert rec["created_at"].endswith("Z") and "T" in rec["created_at"]
644
645 # client cannot spoof a read_only created_at
646 import pytest
647 with pytest.raises(object_records.InvalidRecordPayloadError, match="read-only"):
648 object_records.create_collection_record(
649 "links", {"id": "l2", "title": "y", "created_at": "2000-01-01T00:00:00Z"},
650 base_dir=data_dir, roots=[],
651 )
652
653
654# --- Phase 4b: the `extra` extension field (docs/upgrade-and-customization.md Rule 3) ---
655
656
657def test_backward_compat_no_store_extra_fields_means_no_extra_column(tmp_path):
658 """A collection with no store:extra fields and no submitted `extra` key
659 must behave byte-identically to before the `extra` feature existed: no
660 `extra` column ever appears on disk."""
661 data_dir = tmp_path / "data"
662 write_schema(
663 data_dir,
664 "contacts",
665 [{"name": "id"}, {"name": "name", "required": True}, {"name": "email"}],
666 )
667
668 created = object_records.create_collection_record(
669 "contacts",
670 {"id": "c1", "name": "Ada", "email": "ada@example.com"},
671 base_dir=data_dir,
672 roots=[],
673 )
674 assert created == {"id": "c1", "name": "Ada", "email": "ada@example.com"}
675
676 updated = object_records.update_collection_record(
677 "contacts", "c1", {"email": "ada@example.org"}, base_dir=data_dir, roots=[]
678 )
679 assert updated == {"id": "c1", "name": "Ada", "email": "ada@example.org"}
680
681 fetched = object_records.get_collection_record("contacts", "c1", base_dir=data_dir, roots=[])
682 assert fetched == updated
683
684 listed = object_records.list_collection_records("contacts", base_dir=data_dir, roots=[])["records"]
685 assert listed == [updated]
686
687 tsv_text = (data_dir / "collections" / "contacts" / "records.tsv").read_text()
688 assert tsv_text == "id\tname\temail\nc1\tAda\tada@example.org\n"
689 assert "extra" not in tsv_text.splitlines()[0].split("\t")
690
691
692def test_store_extra_view_field_is_stored_in_extra_column_not_own_column(tmp_path):
693 data_dir = tmp_path / "data"
694 write_schema(
695 data_dir,
696 "notes",
697 [
698 {"name": "id"},
699 {"name": "title", "required": True},
700 {"name": "x_mood", "type": "text", "store": "extra"},
701 ],
702 )
703
704 record = object_records.create_collection_record(
705 "notes",
706 {"id": "n1", "title": "Hi", "x_mood": "curious"},
707 base_dir=data_dir,
708 roots=[],
709 )
710
711 assert record["title"] == "Hi"
712 assert record["x_mood"] == "curious"
713
714 path = data_dir / "collections" / "notes" / "records.tsv"
715 with path.open(newline="") as handle:
716 rows = list(csv.reader(handle, delimiter="\t"))
717 header = rows[0]
718 assert "extra" in header
719 assert "x_mood" not in header
720
721 extra_cell = rows[1][header.index("extra")]
722 assert json.loads(extra_cell) == {"x_mood": "curious"}
723
724 fetched = object_records.get_collection_record("notes", "n1", base_dir=data_dir, roots=[])
725 assert fetched["x_mood"] == "curious"
726 assert json.loads(fetched["extra"]) == {"x_mood": "curious"}
727
728 listed = object_records.list_collection_records("notes", base_dir=data_dir, roots=[])["records"]
729 assert listed[0]["x_mood"] == "curious"
730
731
732def test_store_extra_view_field_still_validates(tmp_path):
733 data_dir = tmp_path / "data"
734 write_schema(
735 data_dir,
736 "notes",
737 [
738 {"name": "id"},
739 {"name": "title", "required": True},
740 {
741 "name": "x_mood",
742 "type": "enum",
743 "enum": ["happy", "curious", "grumpy"],
744 "required": True,
745 "store": "extra",
746 },
747 ],
748 )
749
750 with pytest.raises(object_records.InvalidRecordPayloadError, match="x_mood' is required"):
751 object_records.create_collection_record(
752 "notes", {"id": "n1", "title": "Hi"}, base_dir=data_dir, roots=[]
753 )
754
755 with pytest.raises(object_records.InvalidRecordPayloadError, match="x_mood' must be one of"):
756 object_records.create_collection_record(
757 "notes",
758 {"id": "n1", "title": "Hi", "x_mood": "furious"},
759 base_dir=data_dir,
760 roots=[],
761 )
762
763 record = object_records.create_collection_record(
764 "notes",
765 {"id": "n1", "title": "Hi", "x_mood": "curious"},
766 base_dir=data_dir,
767 roots=[],
768 )
769 assert record["x_mood"] == "curious"
770
771
772def test_partial_update_preserves_untouched_extra_keys(tmp_path):
773 data_dir = tmp_path / "data"
774 write_schema(
775 data_dir,
776 "notes",
777 [
778 {"name": "id"},
779 {"name": "title"},
780 {"name": "x_mood", "type": "text", "store": "extra"},
781 ],
782 )
783
784 created = object_records.create_collection_record(
785 "notes",
786 {"id": "n1", "title": "Hi", "x_mood": "curious", "extra": {"note": "hi"}},
787 base_dir=data_dir,
788 roots=[],
789 )
790 assert created["x_mood"] == "curious"
791 assert json.loads(created["extra"]) == {"x_mood": "curious", "note": "hi"}
792
793 # Updating only `title` must not wipe x_mood or the undeclared `note` key.
794 after_title = object_records.update_collection_record(
795 "notes", "n1", {"title": "Hello"}, base_dir=data_dir, roots=[]
796 )
797 assert after_title["title"] == "Hello"
798 assert after_title["x_mood"] == "curious"
799 assert json.loads(after_title["extra"]) == {"x_mood": "curious", "note": "hi"}
800
801 # Updating only x_mood must not wipe `note`, and must not touch title.
802 after_mood = object_records.update_collection_record(
803 "notes", "n1", {"x_mood": "grumpy"}, base_dir=data_dir, roots=[]
804 )
805 assert after_mood["title"] == "Hello"
806 assert after_mood["x_mood"] == "grumpy"
807 assert json.loads(after_mood["extra"]) == {"x_mood": "grumpy", "note": "hi"}
808
809 fetched = object_records.get_collection_record("notes", "n1", base_dir=data_dir, roots=[])
810 assert fetched["x_mood"] == "grumpy"
811 assert json.loads(fetched["extra"]) == {"x_mood": "grumpy", "note": "hi"}
812
813
814def test_raw_extra_passthrough_accepts_dict_and_json_string(tmp_path):
815 data_dir = tmp_path / "data"
816 write_records(data_dir, "widgets", "id\tname\n")
817
818 from_dict = object_records.create_collection_record(
819 "widgets",
820 {"id": "w1", "name": "Gadget", "extra": {"a": "1"}},
821 base_dir=data_dir,
822 roots=[],
823 )
824 assert json.loads(from_dict["extra"]) == {"a": "1"}
825
826 from_string = object_records.create_collection_record(
827 "widgets",
828 {"id": "w2", "name": "Gizmo", "extra": json.dumps({"a": "1"})},
829 base_dir=data_dir,
830 roots=[],
831 )
832 assert json.loads(from_string["extra"]) == {"a": "1"}
833
834 fetched = object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
835 assert json.loads(fetched["extra"]) == {"a": "1"}
836
837
838@pytest.mark.parametrize("bad_extra", [["a", "1"], "[1, 2]", "not json", 5, True])
839def test_invalid_extra_payload_is_rejected(tmp_path, bad_extra):
840 data_dir = tmp_path / "data"
841 write_records(data_dir, "widgets", "id\tname\n")
842
843 with pytest.raises(object_records.InvalidRecordPayloadError, match="extra"):
844 object_records.create_collection_record(
845 "widgets",
846 {"id": "w1", "name": "Gadget", "extra": bad_extra},
847 base_dir=data_dir,
848 roots=[],
849 )
850
851
852# --- Perf pass: records cache, id index, csv.reader-based parsing ---
853#
854# These tests describe behavior that must hold both BEFORE and AFTER the
855# caching/parsing change (there is no cache pre-change, so "a mutation
856# never leaks" and "external changes are always seen" hold trivially; the
857# point is that they must ALSO hold once a cache is introduced).
858
859
860def test_short_row_missing_fields_become_empty_string(tmp_path):
861 """A data row with fewer tab-separated values than the header (e.g. a
862 hand-edited or partially-written TSV) must fill the missing trailing
863 fields with "" -- this mirrors csv.DictReader's restval=None behavior,
864 which the surrounding code already converts to "" (see the
865 `value if value is not None else ""` normalization)."""
866 data_dir = tmp_path / "data"
867 write_records(data_dir, "contacts", "id\tname\temail\tphone\nc1\tAda\n")
868
869 result = object_records.list_collection_records("contacts", base_dir=data_dir, roots=[])
870 assert result["records"] == [{"id": "c1", "name": "Ada", "email": "", "phone": ""}]
871
872 record = object_records.get_collection_record("contacts", "c1", base_dir=data_dir, roots=[])
873 assert record == {"id": "c1", "name": "Ada", "email": "", "phone": ""}
874
875
876def test_long_row_extra_fields_raise_matching_original_error(tmp_path):
877 data_dir = tmp_path / "data"
878 write_records(data_dir, "contacts", "id\tname\nc1\tAda\tLovelace\textra\n")
879
880 with pytest.raises(ValueError, match="extra fields on row 2"):
881 object_records.list_collection_records("contacts", base_dir=data_dir, roots=[])
882
883
884def test_quoted_value_containing_tab_round_trips(tmp_path):
885 """A field value containing a literal tab (and newline) is legitimate --
886 only field NAMES/ids are restricted, not values (see
887 _normalize_record_payload). csv.DictWriter's default QUOTE_MINIMAL
888 quotes such a value on write; the read path must undo that quoting
889 exactly, never naively split on tab."""
890 data_dir = tmp_path / "data"
891 write_records(data_dir, "notes", "id\tbody\n")
892
893 tricky_value = "line one\tline two\nline three"
894 created = object_records.create_collection_record(
895 "notes",
896 {"id": "n1", "body": tricky_value},
897 base_dir=data_dir,
898 roots=[],
899 )
900 assert created["body"] == tricky_value
901
902 fetched = object_records.get_collection_record("notes", "n1", base_dir=data_dir, roots=[])
903 assert fetched["body"] == tricky_value
904
905 listed = object_records.list_collection_records("notes", base_dir=data_dir, roots=[])["records"]
906 assert listed[0]["body"] == tricky_value
907
908
909def test_get_collection_record_with_duplicate_ids_returns_first_match(tmp_path):
910 """Malformed/hand-edited data can contain a duplicate id. Any id index
911 used for O(1) lookup must replicate the original linear scan's
912 first-match-wins semantics exactly."""
913 data_dir = tmp_path / "data"
914 write_records(data_dir, "contacts", "id\tname\nc1\tAda\nc1\tGrace\n")
915
916 record = object_records.get_collection_record("contacts", "c1", base_dir=data_dir, roots=[])
917 assert record == {"id": "c1", "name": "Ada"}
918
919
920def test_mutating_returned_record_does_not_corrupt_next_read(tmp_path):
921 """Every record dict/list handed back across the public API must be a
922 fresh copy: a caller mutating what it got back must never affect a
923 later, independent call. This is the aliasing guard for the records
924 cache -- cached objects must never be returned by reference."""
925 data_dir = tmp_path / "data"
926 write_records(data_dir, "contacts", "id\tname\nc1\tAda\nc2\tGrace\n")
927
928 first = object_records.get_collection_record("contacts", "c1", base_dir=data_dir, roots=[])
929 first["name"] = "CORRUPTED"
930 first["new_field"] = "also corrupted"
931
932 second = object_records.get_collection_record("contacts", "c1", base_dir=data_dir, roots=[])
933 assert second == {"id": "c1", "name": "Ada"}
934
935 records = object_records.list_collection_records("contacts", base_dir=data_dir, roots=[])["records"]
936 records[0]["name"] = "ALSO CORRUPTED"
937
938 still_clean = object_records.get_collection_record("contacts", "c1", base_dir=data_dir, roots=[])
939 assert still_clean["name"] == "Ada"
940 still_listed = object_records.list_collection_records("contacts", base_dir=data_dir, roots=[])["records"]
941 assert still_listed[0]["name"] == "Ada"
942
943
944def test_cache_reflects_external_file_change(tmp_path):
945 """Another process (or the same process writing directly, bypassing the
946 API) can rewrite records.tsv on disk. The next API read must observe
947 the new content -- the stat-signature check the records cache relies
948 on for cross-process consistency."""
949 data_dir = tmp_path / "data"
950 path = write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
951
952 warm = object_records.get_collection_record("contacts", "c1", base_dir=data_dir, roots=[])
953 assert warm["name"] == "Ada"
954
955 # Rewrite the file directly, outside the object_records API.
956 path.write_text("id\tname\nc1\tAda Lovelace\nc2\tGrace Hopper\n")
957
958 refreshed = object_records.get_collection_record("contacts", "c1", base_dir=data_dir, roots=[])
959 assert refreshed["name"] == "Ada Lovelace"
960
961 listed = object_records.list_collection_records("contacts", base_dir=data_dir, roots=[])["records"]
962 assert listed == [
963 {"id": "c1", "name": "Ada Lovelace"},
964 {"id": "c2", "name": "Grace Hopper"},
965 ]
966
967
968def test_update_after_cached_read_returns_fresh_data(tmp_path):
969 """A get/list right after a create/update must never return a value
970 that predates the write, whether or not an earlier read happened to
971 warm the cache first."""
972 data_dir = tmp_path / "data"
973 write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
974
975 warm = object_records.get_collection_record("contacts", "c1", base_dir=data_dir, roots=[])
976 assert warm["name"] == "Ada"
977
978 object_records.update_collection_record(
979 "contacts", "c1", {"name": "Ada Lovelace"}, base_dir=data_dir, roots=[]
980 )
981
982 refreshed = object_records.get_collection_record("contacts", "c1", base_dir=data_dir, roots=[])
983 assert refreshed["name"] == "Ada Lovelace"
984
985 listed = object_records.list_collection_records("contacts", base_dir=data_dir, roots=[])["records"]
986 assert listed == [{"id": "c1", "name": "Ada Lovelace"}]
987
988
989def test_classic_write_keeps_an_open_readers_fd_on_a_complete_version(tmp_path):
990 """The "Q2 property" that live cross-process editing depends on, pinned
991 by a test -- a code comment (object_records.py, the CONCURRENCY block)
992 can't fail CI. Classic create/update/delete each rewrite via a temp file
993 + atomic replace (_write_collection_records), so a reader in another
994 process that already has the file open keeps seeing the COMPLETE
995 pre-write version -- the atomic rename swaps the directory entry to a new
996 inode while the reader's fd holds the old one -- and a reader that opens
997 AFTER the write sees the COMPLETE post-write version. Never a torn mix.
998
999 This is load-bearing for any consumer that edits a collection live from
1000 more than one process at once -- e.g. a request handler and a background
1001 daemon mutating the same objects while clients are actively reading them.
1002 The regression this guards against is a future "optimization" that
1003 rewrites in place via open(path, "w"): that truncates the very inode a
1004 concurrent reader's fd points at, turning live editing into live
1005 corruption. Such a writer fails the mid-write assertions below.
1006
1007 (Append mode deliberately mutates the same inode and upholds the same
1008 guarantee via the torn-tail rule instead -- see
1009 test_append_mode_torn_tail_is_ignored_and_self_heals.)
1010 """
1011 data_dir = tmp_path / "data"
1012 path = write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
1013
1014 # UPDATE: a reader fd opened before the write still reads the whole
1015 # pre-write file; a fresh read afterward sees the whole post-write file.
1016 with path.open("rb") as held:
1017 object_records.update_collection_record(
1018 "contacts", "c1", {"name": "Ada Lovelace"}, base_dir=data_dir, roots=[]
1019 )
1020 assert held.read() == b"id\tname\nc1\tAda\n"
1021 assert path.read_text() == "id\tname\nc1\tAda Lovelace\n"
1022
1023 # CREATE
1024 with path.open("rb") as held:
1025 object_records.create_collection_record(
1026 "contacts", {"id": "c2", "name": "Grace"}, base_dir=data_dir, roots=[]
1027 )
1028 assert held.read() == b"id\tname\nc1\tAda Lovelace\n"
1029 assert path.read_text() == "id\tname\nc1\tAda Lovelace\nc2\tGrace\n"
1030
1031 # DELETE
1032 with path.open("rb") as held:
1033 object_records.delete_collection_record("contacts", "c2", base_dir=data_dir, roots=[])
1034 assert held.read() == b"id\tname\nc1\tAda Lovelace\nc2\tGrace\n"
1035 assert path.read_text() == "id\tname\nc1\tAda Lovelace\n"
1036
1037
1038def test_delete_collection_record_unaffected_by_extra(tmp_path):
1039 data_dir = tmp_path / "data"
1040 write_schema(
1041 data_dir,
1042 "notes",
1043 [
1044 {"name": "id"},
1045 {"name": "title"},
1046 {"name": "x_mood", "type": "text", "store": "extra"},
1047 ],
1048 )
1049 object_records.create_collection_record(
1050 "notes",
1051 {"id": "n1", "title": "Hi", "x_mood": "curious"},
1052 base_dir=data_dir,
1053 roots=[],
1054 )
1055
1056 removed = object_records.delete_collection_record("notes", "n1", base_dir=data_dir, roots=[])
1057 assert removed["id"] == "n1"
1058
1059 assert object_records.list_collection_records("notes", base_dir=data_dir, roots=[])["records"] == []
1060
1061
1062def _cache_key(collection: str, data_dir: Path) -> str:
1063 path = object_records.collection_records_file(collection, base_dir=data_dir)
1064 return str(path.resolve(strict=False))
1065
1066
1067def test_records_cache_lru_evicts_least_recently_used(tmp_path, monkeypatch):
1068 """With capacity 2, reading a third collection must evict the least
1069 recently touched one (not necessarily the first-read one), and
1070 re-reading the evicted collection must still return correct data via
1071 the reparse path."""
1072 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ENTRIES", "2")
1073 data_dir = tmp_path / "data"
1074 write_records(data_dir, "coll_a", "id\tname\na1\tAda\n")
1075 write_records(data_dir, "coll_b", "id\tname\nb1\tBob\n")
1076 write_records(data_dir, "coll_c", "id\tname\nc1\tCleo\n")
1077
1078 object_records._RECORDS_CACHE.clear()
1079 object_records.read_collection_records("coll_a", base_dir=data_dir, roots=[])
1080 object_records.read_collection_records("coll_b", base_dir=data_dir, roots=[])
1081 # coll_a is now the least-recently-used of {coll_a, coll_b}; reading
1082 # coll_c should push the cache over capacity and evict coll_a.
1083 object_records.read_collection_records("coll_c", base_dir=data_dir, roots=[])
1084
1085 keys = set(object_records._RECORDS_CACHE.keys())
1086 assert keys == {_cache_key("coll_b", data_dir), _cache_key("coll_c", data_dir)}
1087 assert _cache_key("coll_a", data_dir) not in object_records._RECORDS_CACHE
1088
1089 # Re-reading the evicted collection must still work correctly.
1090 result = object_records.read_collection_records("coll_a", base_dir=data_dir, roots=[])
1091 assert result == [{"id": "a1", "name": "Ada"}]
1092 assert _cache_key("coll_a", data_dir) in object_records._RECORDS_CACHE
1093
1094
1095def test_records_cache_lru_hit_refreshes_recency(tmp_path, monkeypatch):
1096 """A cache HIT (not just a store) must count as use: touching coll_a
1097 again before reading coll_c should save it from eviction."""
1098 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ENTRIES", "2")
1099 data_dir = tmp_path / "data"
1100 write_records(data_dir, "coll_a", "id\tname\na1\tAda\n")
1101 write_records(data_dir, "coll_b", "id\tname\nb1\tBob\n")
1102 write_records(data_dir, "coll_c", "id\tname\nc1\tCleo\n")
1103
1104 object_records._RECORDS_CACHE.clear()
1105 object_records.read_collection_records("coll_a", base_dir=data_dir, roots=[])
1106 object_records.read_collection_records("coll_b", base_dir=data_dir, roots=[])
1107 # Touch coll_a again -- it is now the most-recently-used, so coll_b
1108 # (untouched since its initial read) should be evicted instead.
1109 object_records.read_collection_records("coll_a", base_dir=data_dir, roots=[])
1110 object_records.read_collection_records("coll_c", base_dir=data_dir, roots=[])
1111
1112 keys = set(object_records._RECORDS_CACHE.keys())
1113 assert keys == {_cache_key("coll_a", data_dir), _cache_key("coll_c", data_dir)}
1114
1115
1116def test_records_cache_row_threshold_excludes_large_collections(tmp_path, monkeypatch):
1117 """A collection parsing to more rows than DBBASIC_RECORDS_CACHE_MAX_ROWS
1118 is returned correctly but never stored -- and must not evict a
1119 small, hot collection just because it was read once."""
1120 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "10")
1121 data_dir = tmp_path / "data"
1122 small_rows = "".join(f"s{i}\tName{i}\n" for i in range(5))
1123 write_records(data_dir, "small", "id\tname\n" + small_rows)
1124 large_rows = "".join(f"l{i}\tName{i}\n" for i in range(20))
1125 write_records(data_dir, "large", "id\tname\n" + large_rows)
1126
1127 object_records._RECORDS_CACHE.clear()
1128
1129 small_result = object_records.read_collection_records("small", base_dir=data_dir, roots=[])
1130 assert len(small_result) == 5
1131 assert _cache_key("small", data_dir) in object_records._RECORDS_CACHE
1132
1133 large_result = object_records.read_collection_records("large", base_dir=data_dir, roots=[])
1134 assert len(large_result) == 20
1135 assert large_result[0]["id"] == "l0"
1136 assert large_result[-1]["id"] == "l19"
1137
1138 assert _cache_key("large", data_dir) not in object_records._RECORDS_CACHE
1139 # The small collection's entry must have survived reading the large one.
1140 assert _cache_key("small", data_dir) in object_records._RECORDS_CACHE
1141
1142
1143def test_records_cache_row_threshold_reparses_correctly_every_time(tmp_path, monkeypatch):
1144 """A collection over the row threshold is never cached, so repeated
1145 reads (and a read after an update) must each reparse from disk and
1146 return correct, current data -- not a stale first result."""
1147 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "10")
1148 data_dir = tmp_path / "data"
1149 large_rows = "".join(f"l{i}\tName{i}\n" for i in range(20))
1150 write_records(data_dir, "large", "id\tname\n" + large_rows)
1151
1152 object_records._RECORDS_CACHE.clear()
1153 first = object_records.get_collection_record("large", "l5", base_dir=data_dir, roots=[])
1154 assert first["name"] == "Name5"
1155 assert _cache_key("large", data_dir) not in object_records._RECORDS_CACHE
1156
1157 object_records.update_collection_record(
1158 "large", "l5", {"name": "Updated"}, base_dir=data_dir, roots=[]
1159 )
1160 second = object_records.get_collection_record("large", "l5", base_dir=data_dir, roots=[])
1161 assert second["name"] == "Updated"
1162
1163
1164def test_list_collection_records_window_copy_and_boundaries(tmp_path):
1165 """The pagination window copy path: correct slice, correct total/
1166 has_more at both interior and edge offsets, and mutating a returned
1167 record must not leak into the cache or a later independent call."""
1168 data_dir = tmp_path / "data"
1169 rows = "".join(f"r{i}\tName{i}\n" for i in range(10))
1170 write_records(data_dir, "items", "id\tname\n" + rows)
1171
1172 # Warm the cache first.
1173 object_records.read_collection_records("items", base_dir=data_dir, roots=[])
1174
1175 page = object_records.list_collection_records(
1176 "items", base_dir=data_dir, roots=[], limit=3, offset=2
1177 )
1178 assert page["records"] == [
1179 {"id": "r2", "name": "Name2"},
1180 {"id": "r3", "name": "Name3"},
1181 {"id": "r4", "name": "Name4"},
1182 ]
1183 assert page["total"] == 10
1184 assert page["count"] == 3
1185 assert page["has_more"] is True
1186
1187 # Aliasing guard: mutating a returned record must not affect a later,
1188 # independent get/list call.
1189 page["records"][0]["name"] = "CORRUPTED"
1190 page["records"][0]["new_field"] = "also corrupted"
1191 again = object_records.get_collection_record("items", "r2", base_dir=data_dir, roots=[])
1192 assert again == {"id": "r2", "name": "Name2"}
1193 relisted = object_records.list_collection_records(
1194 "items", base_dir=data_dir, roots=[], limit=3, offset=2
1195 )
1196 assert relisted["records"][0] == {"id": "r2", "name": "Name2"}
1197
1198 # offset beyond the end of the collection.
1199 empty_page = object_records.list_collection_records(
1200 "items", base_dir=data_dir, roots=[], limit=5, offset=100
1201 )
1202 assert empty_page == {
1203 "collection": "items",
1204 "records": [],
1205 "count": 0,
1206 "total": 10,
1207 "limit": 5,
1208 "offset": 100,
1209 "has_more": False,
1210 }
1211
1212 # limit reaching past the end of the collection.
1213 tail_page = object_records.list_collection_records(
1214 "items", base_dir=data_dir, roots=[], limit=100, offset=8
1215 )
1216 assert [record["id"] for record in tail_page["records"]] == ["r8", "r9"]
1217 assert tail_page["count"] == 2
1218 assert tail_page["total"] == 10
1219 assert tail_page["has_more"] is False
1220
1221
1222def test_list_collection_records_warm_copies_bounded_by_window(tmp_path, monkeypatch):
1223 """Perf guard for the pagination fix: a warm list with a small limit on
1224 a large collection must copy only the window's records, not every row
1225 in the collection. Verified precisely (not by timing, which is
1226 unreliable on a network share) by shadowing the module's `dict` name
1227 and counting calls made while servicing the list request."""
1228 data_dir = tmp_path / "data"
1229 path = data_dir / "collections" / "big" / "records.tsv"
1230 path.parent.mkdir(parents=True, exist_ok=True)
1231 with path.open("w", newline="") as handle:
1232 handle.write("id\tname\n")
1233 for i in range(50_000):
1234 handle.write(f"r{i}\tName{i}\n")
1235
1236 # Warm the cache with an initial read, outside the counted region.
1237 warm = object_records.read_collection_records("big", base_dir=data_dir, roots=[])
1238 assert len(warm) == 50_000
1239 cache_records = object_records._cached_records_ref("big", base_dir=data_dir)
1240 assert len(cache_records) == 50_000
1241
1242 copy_calls: list[None] = []
1243 real_dict = dict
1244
1245 def counting_dict(*args, **kwargs):
1246 copy_calls.append(None)
1247 return real_dict(*args, **kwargs)
1248
1249 monkeypatch.setattr(object_records, "dict", counting_dict, raising=False)
1250
1251 result = object_records.list_collection_records(
1252 "big", base_dir=data_dir, roots=[], limit=10, offset=0
1253 )
1254
1255 assert result["total"] == 50_000
1256 assert result["count"] == 10
1257 assert len(result["records"]) == 10
1258 for i, record in enumerate(result["records"]):
1259 assert record == {"id": f"r{i}", "name": f"Name{i}"}
1260 # Each returned record is its own copy, never the cache's dict.
1261 assert record is not cache_records[i]
1262
1263 # Exactly one dict() copy per windowed record -- not one per row in
1264 # the 50k-row collection.
1265 assert len(copy_calls) == 10
1266
1267
1268# --- Append-only storage (docs/append-only-storage-design.md) ---
1269
1270
1271def write_append_schema(data_dir: Path, collection: str, fields: list[dict] | None = None) -> Path:
1272 """Like write_schema, but opts the collection into append storage."""
1273 path = data_dir / "schemas" / f"{collection}.json"
1274 path.parent.mkdir(parents=True, exist_ok=True)
1275 path.write_text(
1276 json.dumps({"fields": fields or [{"name": "id"}], "storage": "append"})
1277 )
1278 return path
1279
1280
1281def append_physical_rows(
1282 path: Path, physical_fields: list[str], rows: list[dict[str, str]]
1283) -> None:
1284 """Simulate an external writer (another process, the CLI, a sibling
1285 pool worker) appending physical rows directly to an already
1286 append-physical records.tsv -- a plain file write that goes through
1287 none of object_records' own API or cache-refresh machinery, matching
1288 exactly the physical format object_records._append_records_rows
1289 itself writes: tab-delimited, "\\n" line terminator, QUOTE_MINIMAL,
1290 one row per dict in `physical_fields` order (including the `_op`
1291 column -- pass object_records.OP_FIELD explicitly in each row)."""
1292 with path.open("a", newline="") as handle:
1293 writer = csv.writer(
1294 handle, delimiter="\t", lineterminator="\n", quoting=csv.QUOTE_MINIMAL
1295 )
1296 for row in rows:
1297 writer.writerow([row.get(field, "") for field in physical_fields])
1298
1299
1300def test_op_field_is_rejected_in_user_payloads(tmp_path):
1301 data_dir = tmp_path / "data"
1302 write_records(data_dir, "widgets", "id\tname\n")
1303
1304 with pytest.raises(object_records.InvalidRecordPayloadError, match="reserved"):
1305 object_records.create_collection_record(
1306 "widgets", {"id": "w1", "name": "Gadget", "_op": "del"}, base_dir=data_dir, roots=[]
1307 )
1308
1309
1310def test_schema_field_named_op_is_rejected(tmp_path):
1311 with pytest.raises(ValueError, match="reserved"):
1312 object_schemas.normalize_schema("widgets", {"fields": [{"name": "_op"}]})
1313
1314
1315def test_schema_storage_key_rejects_invalid_values(tmp_path):
1316 with pytest.raises(ValueError, match="storage"):
1317 object_schemas.normalize_schema("widgets", {"fields": [], "storage": "bogus"})
1318
1319
1320def test_schema_storage_key_survives_normalization(tmp_path):
1321 normalized = object_schemas.normalize_schema(
1322 "widgets", {"fields": [{"name": "id"}], "storage": "append"}
1323 )
1324 assert normalized["storage"] == "append"
1325
1326 # Absent by default -- byte-identical to a schema that never mentions
1327 # storage (see the classic-mode-untouched test below).
1328 classic = object_schemas.normalize_schema("widgets", {"fields": [{"name": "id"}]})
1329 assert "storage" not in classic
1330
1331
1332def test_classic_mode_is_byte_identical_to_before_append_storage_existed(tmp_path):
1333 """A collection with no `storage` key must write records.tsv exactly
1334 as it always has -- no `_op` column, no folding, no torn-tail
1335 tolerance. This is the opt-in guarantee: default/absent means
1336 unchanged behavior."""
1337 data_dir = tmp_path / "data"
1338 write_schema(data_dir, "contacts", [{"name": "id"}, {"name": "name"}])
1339
1340 object_records.create_collection_record(
1341 "contacts", {"id": "c1", "name": "Ada"}, base_dir=data_dir, roots=[]
1342 )
1343 object_records.create_collection_record(
1344 "contacts", {"id": "c2", "name": "Grace"}, base_dir=data_dir, roots=[]
1345 )
1346 object_records.update_collection_record(
1347 "contacts", "c1", {"name": "Ada Lovelace"}, base_dir=data_dir, roots=[]
1348 )
1349 object_records.delete_collection_record("contacts", "c2", base_dir=data_dir, roots=[])
1350
1351 path = data_dir / "collections" / "contacts" / "records.tsv"
1352 assert path.read_text() == "id\tname\nc1\tAda Lovelace\n"
1353 assert "_op" not in path.read_text().splitlines()[0].split("\t")
1354
1355
1356def test_append_mode_create_update_delete_use_op_column(tmp_path):
1357 data_dir = tmp_path / "data"
1358 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
1359 path = data_dir / "collections" / "widgets" / "records.tsv"
1360
1361 object_records.create_collection_record(
1362 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[]
1363 )
1364 object_records.create_collection_record(
1365 "widgets", {"id": "w2", "name": "Beta"}, base_dir=data_dir, roots=[]
1366 )
1367 with path.open(newline="") as handle:
1368 rows = list(csv.reader(handle, delimiter="\t"))
1369 assert rows[0] == ["_op", "id", "name"]
1370 assert rows[1] == ["", "w1", "Alpha"]
1371 assert rows[2] == ["", "w2", "Beta"]
1372
1373 object_records.update_collection_record(
1374 "widgets", "w1", {"name": "Alpha2"}, base_dir=data_dir, roots=[]
1375 )
1376 with path.open(newline="") as handle:
1377 rows = list(csv.reader(handle, delimiter="\t"))
1378 # update appends a superseding row rather than rewriting in place.
1379 assert len(rows) == 4
1380 assert rows[3] == ["", "w1", "Alpha2"]
1381
1382 removed = object_records.delete_collection_record("widgets", "w2", base_dir=data_dir, roots=[])
1383 assert removed == {"id": "w2", "name": "Beta"}
1384 with path.open(newline="") as handle:
1385 rows = list(csv.reader(handle, delimiter="\t"))
1386 # Tombstone visible in the raw file, pre-compaction: obvious in a cat.
1387 assert rows[4] == ["del", "w2", ""]
1388
1389 with pytest.raises(object_records.RecordNotFoundError):
1390 object_records.delete_collection_record("widgets", "w2", base_dir=data_dir, roots=[])
1391
1392 listing = object_records.list_collection_records("widgets", base_dir=data_dir, roots=[])["records"]
1393 assert listing == [{"id": "w1", "name": "Alpha2"}]
1394
1395
1396def test_append_mode_resurrection_after_delete(tmp_path):
1397 data_dir = tmp_path / "data"
1398 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
1399
1400 object_records.create_collection_record(
1401 "widgets", {"id": "w1", "name": "one"}, base_dir=data_dir, roots=[]
1402 )
1403 object_records.delete_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
1404
1405 with pytest.raises(object_records.RecordNotFoundError):
1406 object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
1407
1408 resurrected = object_records.create_collection_record(
1409 "widgets", {"id": "w1", "name": "reborn"}, base_dir=data_dir, roots=[]
1410 )
1411 assert resurrected == {"id": "w1", "name": "reborn"}
1412 assert object_records.get_collection_record(
1413 "widgets", "w1", base_dir=data_dir, roots=[]
1414 ) == {"id": "w1", "name": "reborn"}
1415
1416
1417def test_append_mode_transition_on_next_write_adds_op_column(tmp_path):
1418 """Switching a classic collection to storage:append doesn't touch the
1419 file until the collection's next write, which performs one final
1420 classic-shaped rewrite that adds the `_op` header column (existing
1421 rows get "")."""
1422 data_dir = tmp_path / "data"
1423 schema_path = write_schema(data_dir, "notes", [{"name": "id"}, {"name": "title"}])
1424 object_records.create_collection_record(
1425 "notes", {"id": "n1", "title": "one"}, base_dir=data_dir, roots=[]
1426 )
1427 object_records.create_collection_record(
1428 "notes", {"id": "n2", "title": "two"}, base_dir=data_dir, roots=[]
1429 )
1430 path = data_dir / "collections" / "notes" / "records.tsv"
1431 assert path.read_text() == "id\ttitle\nn1\tone\nn2\ttwo\n"
1432
1433 # Opt in -- the file on disk does not change yet, only the schema does.
1434 schema_path.write_text(
1435 json.dumps({"fields": [{"name": "id"}, {"name": "title"}], "storage": "append"})
1436 )
1437 assert path.read_text() == "id\ttitle\nn1\tone\nn2\ttwo\n"
1438
1439 updated = object_records.update_collection_record(
1440 "notes", "n1", {"title": "one-updated"}, base_dir=data_dir, roots=[]
1441 )
1442 assert updated == {"id": "n1", "title": "one-updated"}
1443
1444 with path.open(newline="") as handle:
1445 rows = list(csv.reader(handle, delimiter="\t"))
1446 assert rows[0] == ["_op", "id", "title"]
1447 assert rows[1] == ["", "n1", "one-updated"]
1448 assert rows[2] == ["", "n2", "two"]
1449 assert len(rows) == 3
1450
1451
1452def test_append_mode_switch_back_compacts_to_classic_form(tmp_path):
1453 """Removing storage:append causes the collection's next write to fold
1454 current content and drop the `_op` column -- byte-compatible with a
1455 classic file (docs/append-only-storage-design.md Migration and
1456 Compatibility: "A compacted append-only file is byte-compatible with
1457 a classic one")."""
1458 data_dir = tmp_path / "data"
1459 schema_path = write_append_schema(data_dir, "notes", [{"name": "id"}, {"name": "title"}])
1460 object_records.create_collection_record(
1461 "notes", {"id": "n1", "title": "one"}, base_dir=data_dir, roots=[]
1462 )
1463 object_records.update_collection_record(
1464 "notes", "n1", {"title": "one-updated"}, base_dir=data_dir, roots=[]
1465 )
1466 object_records.create_collection_record(
1467 "notes", {"id": "n2", "title": "two"}, base_dir=data_dir, roots=[]
1468 )
1469 path = data_dir / "collections" / "notes" / "records.tsv"
1470 with path.open(newline="") as handle:
1471 rows_before = list(csv.reader(handle, delimiter="\t"))
1472 assert rows_before[0][0] == "_op"
1473 assert len(rows_before) == 4 # header + 3 physical rows (1 superseded)
1474
1475 # Opt back out.
1476 schema_path.write_text(
1477 json.dumps({"fields": [{"name": "id"}, {"name": "title"}]})
1478 )
1479 object_records.update_collection_record(
1480 "notes", "n2", {"title": "two-updated"}, base_dir=data_dir, roots=[]
1481 )
1482
1483 assert path.read_text() == "id\ttitle\nn1\tone-updated\nn2\ttwo-updated\n"
1484
1485
1486def test_append_mode_new_field_falls_back_to_full_rewrite_and_extends_header(tmp_path):
1487 """Append cannot extend an existing physical header: a write that
1488 introduces a field the current header doesn't have must fall back to
1489 a full rewrite that folds current content and adds the column -- the
1490 new value must not be lost."""
1491 data_dir = tmp_path / "data"
1492 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
1493 object_records.create_collection_record(
1494 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[]
1495 )
1496 object_records.create_collection_record(
1497 "widgets", {"id": "w2", "name": "Beta"}, base_dir=data_dir, roots=[]
1498 )
1499 path = data_dir / "collections" / "widgets" / "records.tsv"
1500 with path.open(newline="") as handle:
1501 header_before = next(csv.reader(handle, delimiter="\t"))
1502 assert "color" not in header_before
1503
1504 updated = object_records.update_collection_record(
1505 "widgets", "w1", {"color": "red"}, base_dir=data_dir, roots=[]
1506 )
1507 assert updated == {"id": "w1", "name": "Alpha", "color": "red"}
1508
1509 with path.open(newline="") as handle:
1510 rows = list(csv.reader(handle, delimiter="\t"))
1511 assert rows[0] == ["_op", "id", "name", "color"]
1512 assert "color" in rows[0]
1513
1514 fetched = object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
1515 assert fetched["color"] == "red"
1516 listing = object_records.list_collection_records("widgets", base_dir=data_dir, roots=[])["records"]
1517 assert {r["id"]: r["color"] for r in listing} == {"w1": "red", "w2": ""}
1518
1519
1520def test_compact_collection_removes_superseded_and_tombstoned_rows(tmp_path):
1521 data_dir = tmp_path / "data"
1522 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
1523 object_records.create_collection_record(
1524 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[]
1525 )
1526 object_records.create_collection_record(
1527 "widgets", {"id": "w2", "name": "Beta"}, base_dir=data_dir, roots=[]
1528 )
1529 object_records.update_collection_record(
1530 "widgets", "w1", {"name": "Alpha2"}, base_dir=data_dir, roots=[]
1531 )
1532 object_records.update_collection_record(
1533 "widgets", "w1", {"name": "Alpha3"}, base_dir=data_dir, roots=[]
1534 )
1535 object_records.delete_collection_record("widgets", "w2", base_dir=data_dir, roots=[])
1536
1537 path = data_dir / "collections" / "widgets" / "records.tsv"
1538 raw_before = path.read_text()
1539 assert "del" in raw_before # tombstone present pre-compaction
1540
1541 result = object_records.compact_collection("widgets", base_dir=data_dir, roots=[])
1542 assert result["rows_before"] == 5
1543 assert result["rows_after"] == 1
1544 assert result["bytes_after"] < result["bytes_before"]
1545
1546 raw_after = path.read_text()
1547 assert "del" not in raw_after
1548 assert raw_after == "_op\tid\tname\n\tw1\tAlpha3\n"
1549
1550 # Data is unaffected by compaction.
1551 listing = object_records.list_collection_records("widgets", base_dir=data_dir, roots=[])["records"]
1552 assert listing == [{"id": "w1", "name": "Alpha3"}]
1553
1554
1555def test_compact_collection_is_a_reported_no_op_for_classic_collections(tmp_path):
1556 data_dir = tmp_path / "data"
1557 write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
1558
1559 result = object_records.compact_collection("contacts", base_dir=data_dir, roots=[])
1560 assert result == {"rows_before": 1, "rows_after": 1, "bytes_before": result["bytes_before"], "bytes_after": result["bytes_before"]}
1561 assert (data_dir / "collections" / "contacts" / "records.tsv").read_text() == "id\tname\nc1\tAda\n"
1562
1563
1564def test_append_mode_auto_compact_triggers_on_next_write_past_threshold(tmp_path, monkeypatch):
1565 """When a cold parse observes physical rows past the (monkeypatched
1566 small) threshold, with superseded+deleted rows outnumbering live
1567 rows, it flags the collection -- but performs no write itself (reads
1568 stay read-only). The collection's NEXT write then compacts instead of
1569 appending."""
1570 monkeypatch.setenv("DBBASIC_APPEND_COMPACT_MIN_ROWS", "5")
1571 data_dir = tmp_path / "data"
1572 write_append_schema(data_dir, "churn", [{"name": "id"}, {"name": "value"}])
1573
1574 object_records.create_collection_record(
1575 "churn", {"id": "c1", "value": "v0"}, base_dir=data_dir, roots=[]
1576 )
1577 for i in range(10):
1578 object_records.update_collection_record(
1579 "churn", "c1", {"value": f"v{i}"}, base_dir=data_dir, roots=[]
1580 )
1581
1582 path = data_dir / "collections" / "churn" / "records.tsv"
1583 physical_rows_before = len(path.read_text().splitlines()) - 1
1584 assert physical_rows_before > 5
1585
1586 cache_key = str(object_records.collection_records_file("churn", base_dir=data_dir).resolve())
1587
1588 # Force a cold parse (simulating cache eviction / a fresh process) --
1589 # a pure read.
1590 object_records._RECORDS_CACHE.clear()
1591 before_stat = path.stat()
1592 object_records.get_collection_record("churn", "c1", base_dir=data_dir, roots=[])
1593 after_stat = path.stat()
1594 assert before_stat.st_mtime_ns == after_stat.st_mtime_ns
1595 assert before_stat.st_size == after_stat.st_size
1596 assert cache_key in object_records._PENDING_COMPACTION
1597
1598 object_records.update_collection_record(
1599 "churn", "c1", {"value": "final"}, base_dir=data_dir, roots=[]
1600 )
1601 physical_rows_after = len(path.read_text().splitlines()) - 1
1602 assert physical_rows_after == 1
1603 assert cache_key not in object_records._PENDING_COMPACTION
1604
1605 listing = object_records.list_collection_records("churn", base_dir=data_dir, roots=[])["records"]
1606 assert listing == [{"id": "c1", "value": "final"}]
1607
1608
1609def test_append_mode_torn_tail_is_ignored_and_self_heals(tmp_path):
1610 data_dir = tmp_path / "data"
1611 write_append_schema(data_dir, "logs", [{"name": "id"}, {"name": "value"}])
1612 object_records.create_collection_record(
1613 "logs", {"id": "l1", "value": "one"}, base_dir=data_dir, roots=[]
1614 )
1615 object_records.create_collection_record(
1616 "logs", {"id": "l2", "value": "two"}, base_dir=data_dir, roots=[]
1617 )
1618 path = data_dir / "collections" / "logs" / "records.tsv"
1619 full_text = path.read_text()
1620
1621 # Simulate a crash mid-write: chop off the last few bytes of the file,
1622 # tearing the final row (no trailing newline).
1623 path.write_text(full_text[:-4])
1624 assert not path.read_text().endswith("\n")
1625
1626 object_records._RECORDS_CACHE.clear()
1627 before_stat = path.stat()
1628 listing = object_records.list_collection_records("logs", base_dir=data_dir, roots=[])["records"]
1629 after_stat = path.stat()
1630
1631 # Read succeeds, ignoring the torn fragment -- and never writes.
1632 assert listing == [{"id": "l1", "value": "one"}]
1633 assert before_stat.st_mtime_ns == after_stat.st_mtime_ns
1634 assert before_stat.st_size == after_stat.st_size
1635
1636 # The next write self-heals: the fragment is truncated away (not
1637 # resurrected), and a fresh, well-formed row is appended after it.
1638 object_records.create_collection_record(
1639 "logs", {"id": "l3", "value": "three"}, base_dir=data_dir, roots=[]
1640 )
1641 assert path.read_text().endswith("\n")
1642
1643 # Subsequent reads are clean -- the fragment never reappears.
1644 object_records._RECORDS_CACHE.clear()
1645 listing_after_heal = object_records.list_collection_records(
1646 "logs", base_dir=data_dir, roots=[]
1647 )["records"]
1648 assert listing_after_heal == [
1649 {"id": "l1", "value": "one"},
1650 {"id": "l3", "value": "three"},
1651 ]
1652
1653
1654def test_append_mode_cache_refreshes_on_same_inode_append(tmp_path):
1655 """Unlike a classic full rewrite (always a fresh inode via atomic
1656 replace), a fast append mutates the SAME inode in place. The cache
1657 must still detect and correctly refresh on this change (only size/
1658 mtime move) -- verified here by asserting the inode is unchanged
1659 across the write while a subsequent cache-hit read returns fresh
1660 data, not stale data from the pre-append signature."""
1661 data_dir = tmp_path / "data"
1662 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
1663 object_records.create_collection_record(
1664 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[]
1665 )
1666 path = data_dir / "collections" / "widgets" / "records.tsv"
1667 inode_before = path.stat().st_ino
1668
1669 object_records.update_collection_record(
1670 "widgets", "w1", {"name": "Alpha2"}, base_dir=data_dir, roots=[]
1671 )
1672 inode_after = path.stat().st_ino
1673 assert inode_after == inode_before # same inode: a fast append, not a rewrite
1674
1675 # A cache HIT (no reparse) must still return the fresh value.
1676 cache_key = str(path.resolve())
1677 signature, _, cached_records, _, _ = object_records._RECORDS_CACHE[cache_key]
1678 assert signature == object_records._stat_signature(path)
1679 assert cached_records == [{"id": "w1", "name": "Alpha2"}]
1680
1681 fetched = object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
1682 assert fetched == {"id": "w1", "name": "Alpha2"}
1683
1684
1685def test_append_mode_equivalent_to_classic_mode_across_random_operations(tmp_path):
1686 """THE EQUIVALENCE PROPERTY. An identical, seeded, randomized sequence
1687 of create/update/delete/get/list operations run against a classic
1688 collection and an append-mode collection (same schema otherwise) must
1689 produce API-visible-identical results after EVERY single operation:
1690 same returned records, same exception types, same list contents (see
1691 _fold_append_rows for why order matches classic exactly), same
1692 totals. Deterministic seed, no wall-clock dependence -- every value
1693 written is derived from the loop counter, never wall-clock time.
1694 """
1695 classic_dir = tmp_path / "classic"
1696 append_dir = tmp_path / "append"
1697 write_schema(classic_dir, "items", [{"name": "id"}, {"name": "value"}])
1698 write_append_schema(append_dir, "items", [{"name": "id"}, {"name": "value"}])
1699
1700 def apply(data_dir, kind, record_id, payload):
1701 try:
1702 if kind == "create":
1703 return ("ok", object_records.create_collection_record(
1704 "items", {"id": record_id, "value": payload}, base_dir=data_dir, roots=[]
1705 ))
1706 if kind == "update":
1707 return ("ok", object_records.update_collection_record(
1708 "items", record_id, {"value": payload}, base_dir=data_dir, roots=[]
1709 ))
1710 if kind == "delete":
1711 return ("ok", object_records.delete_collection_record(
1712 "items", record_id, base_dir=data_dir, roots=[]
1713 ))
1714 if kind == "get":
1715 return ("ok", object_records.get_collection_record(
1716 "items", record_id, base_dir=data_dir, roots=[]
1717 ))
1718 return ("ok", object_records.list_collection_records(
1719 "items", base_dir=data_dir, roots=[]
1720 ))
1721 except Exception as exc: # noqa: BLE001 -- comparing exception SHAPES across both runs
1722 return ("error", type(exc), str(exc))
1723
1724 rng = random.Random(20260718)
1725 id_pool = [f"e{i}" for i in range(12)]
1726
1727 for step in range(200):
1728 kind = rng.choice(["create", "update", "delete", "get", "list"])
1729 record_id = rng.choice(id_pool)
1730 payload = f"v{step}"
1731
1732 classic_outcome = apply(classic_dir, kind, record_id, payload)
1733 append_outcome = apply(append_dir, kind, record_id, payload)
1734
1735 assert classic_outcome == append_outcome, (
1736 f"step {step} op={kind!r} id={record_id!r}: "
1737 f"classic={classic_outcome!r} append={append_outcome!r}"
1738 )
1739
1740 # Independent final-state cross-check, beyond the per-step comparisons.
1741 classic_listing = object_records.list_collection_records("items", base_dir=classic_dir, roots=[])
1742 append_listing = object_records.list_collection_records("items", base_dir=append_dir, roots=[])
1743 assert classic_listing == append_listing
1744
1745 # And a full compaction on the append side must not change what it
1746 # reports afterward, either.
1747 object_records.compact_collection("items", base_dir=append_dir, roots=[])
1748 append_listing_after_compact = object_records.list_collection_records(
1749 "items", base_dir=append_dir, roots=[]
1750 )
1751 assert append_listing_after_compact == classic_listing
1752
1753
1754# ---------------------------------------------------------------------------
1755# id -> byte-offset sidecar (docs/append-only-storage-design.md item 4).
1756#
1757# Every test below forces DBBASIC_RECORDS_CACHE_MAX_ROWS to 0 so that
1758# _RECORDS_CACHE never stores an entry for these (tiny, real-world-normal-
1759# sized) test collections -- i.e. every point op is a cache MISS, which is
1760# exactly the condition _fast_record_lookup uses to try the sidecar. This
1761# is what lets a handful of records exercise the same code path a real
1762# over-threshold, million-row collection would use, without the tests
1763# needing to actually write a million rows.
1764# ---------------------------------------------------------------------------
1765
1766
1767def test_oidx_sidecar_builds_lazily_and_serves_point_ops(tmp_path, monkeypatch):
1768 """No sidecar file exists until the first point op against an already
1769 append-physical file with a cold cache; from then on, get/create's
1770 duplicate check/update/delete all resolve correctly through it."""
1771 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
1772 data_dir = tmp_path / "data"
1773 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
1774 path = data_dir / "collections" / "widgets" / "records.tsv"
1775 oidx_path = object_records._oidx_path(path)
1776
1777 assert not oidx_path.exists()
1778 object_records.create_collection_record(
1779 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[]
1780 )
1781 # This first write is a TRANSITION-IN full rewrite (the file only
1782 # just became append-physical) -- nothing has read through the
1783 # sidecar yet, so it still doesn't exist.
1784 assert not oidx_path.exists()
1785
1786 object_records.create_collection_record(
1787 "widgets", {"id": "w2", "name": "Beta"}, base_dir=data_dir, roots=[]
1788 )
1789 # This create's duplicate check is the first point op against an
1790 # already append-physical file with a cold cache -- it must have
1791 # built the sidecar.
1792 assert oidx_path.exists()
1793
1794 with pytest.raises(object_records.DuplicateRecordIdError):
1795 object_records.create_collection_record(
1796 "widgets", {"id": "w1", "name": "Gamma"}, base_dir=data_dir, roots=[]
1797 )
1798
1799 assert object_records.get_collection_record(
1800 "widgets", "w2", base_dir=data_dir, roots=[]
1801 ) == {"id": "w2", "name": "Beta"}
1802
1803 updated = object_records.update_collection_record(
1804 "widgets", "w1", {"name": "Alpha2"}, base_dir=data_dir, roots=[]
1805 )
1806 assert updated == {"id": "w1", "name": "Alpha2"}
1807 assert object_records.get_collection_record(
1808 "widgets", "w1", base_dir=data_dir, roots=[]
1809 ) == {"id": "w1", "name": "Alpha2"}
1810
1811 removed = object_records.delete_collection_record(
1812 "widgets", "w2", base_dir=data_dir, roots=[]
1813 )
1814 assert removed == {"id": "w2", "name": "Beta"}
1815 with pytest.raises(object_records.RecordNotFoundError):
1816 object_records.get_collection_record("widgets", "w2", base_dir=data_dir, roots=[])
1817
1818 listing = object_records.list_collection_records(
1819 "widgets", base_dir=data_dir, roots=[]
1820 )["records"]
1821 assert listing == [{"id": "w1", "name": "Alpha2"}]
1822
1823
1824def test_oidx_sidecar_catches_up_when_behind(tmp_path, monkeypatch):
1825 """A sidecar whose body is missing its last few lines (simulating a
1826 crash between a data append and its matching idx append, or another
1827 writer that didn't maintain the sidecar at all) is caught up by
1828 scanning only the uncovered tail of records.tsv, not rebuilt from
1829 scratch."""
1830 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
1831 data_dir = tmp_path / "data"
1832 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
1833 path = data_dir / "collections" / "widgets" / "records.tsv"
1834 oidx_path = object_records._oidx_path(path)
1835
1836 for i in range(5):
1837 object_records.create_collection_record(
1838 "widgets", {"id": f"w{i}", "name": f"n{i}"}, base_dir=data_dir, roots=[]
1839 )
1840 object_records._OIDX_CACHE.clear()
1841 object_records.get_collection_record("widgets", "w0", base_dir=data_dir, roots=[])
1842 assert oidx_path.exists()
1843
1844 lines = oidx_path.read_text().splitlines(keepends=True)
1845 assert len(lines) == 6 # header + 5 data lines
1846 oidx_path.write_text("".join(lines[:-2]))
1847 object_records._OIDX_CACHE.clear()
1848
1849 fetched4 = object_records.get_collection_record("widgets", "w4", base_dir=data_dir, roots=[])
1850 assert fetched4 == {"id": "w4", "name": "n4"}
1851 fetched0 = object_records.get_collection_record("widgets", "w0", base_dir=data_dir, roots=[])
1852 assert fetched0 == {"id": "w0", "name": "n0"}
1853
1854 id_offsets, coherent = object_records._load_oidx(path)
1855 assert coherent
1856 assert set(id_offsets) == {"w0", "w1", "w2", "w3", "w4"}
1857 assert len(oidx_path.read_text().splitlines()) == 6
1858
1859
1860def test_oidx_sidecar_rebuilds_on_stale_ino_header(tmp_path, monkeypatch):
1861 """A sidecar whose header ino doesn't match the data file's current
1862 inode (e.g. left over from before a compaction) must never be
1863 trusted -- _load_oidx rebuilds from scratch instead."""
1864 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
1865 data_dir = tmp_path / "data"
1866 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
1867 path = data_dir / "collections" / "widgets" / "records.tsv"
1868 oidx_path = object_records._oidx_path(path)
1869
1870 object_records.create_collection_record(
1871 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[]
1872 )
1873
1874 # Hand-write a sidecar with a bogus header ino and a body entry for
1875 # an id that was never actually written -- if the mismatch weren't
1876 # detected, this would leak straight into the answer.
1877 oidx_path.write_text("oidx1\t999999999\n0\t20\t\tbogus\n")
1878 object_records._OIDX_CACHE.clear()
1879
1880 id_offsets, coherent = object_records._load_oidx(path)
1881 assert coherent
1882 assert "bogus" not in id_offsets
1883 assert "w1" in id_offsets
1884 assert object_records.get_collection_record(
1885 "widgets", "w1", base_dir=data_dir, roots=[]
1886 ) == {"id": "w1", "name": "Alpha"}
1887
1888
1889def test_oidx_sidecar_rebuilds_on_corrupt_sidecar_file(tmp_path, monkeypatch):
1890 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
1891 data_dir = tmp_path / "data"
1892 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
1893 path = data_dir / "collections" / "widgets" / "records.tsv"
1894 oidx_path = object_records._oidx_path(path)
1895
1896 object_records.create_collection_record(
1897 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[]
1898 )
1899 object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
1900 assert oidx_path.exists()
1901
1902 oidx_path.write_text("not even close to a valid sidecar\n\x00garbage\tmore\tfields\there\n")
1903 object_records._OIDX_CACHE.clear()
1904
1905 # Never raises -- self-heals via a rebuild, and the API call still
1906 # returns the correct record.
1907 fetched = object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
1908 assert fetched == {"id": "w1", "name": "Alpha"}
1909 id_offsets, coherent = object_records._load_oidx(path)
1910 assert coherent
1911 assert set(id_offsets) == {"w1"}
1912
1913
1914def test_oidx_sidecar_rebuilds_when_file_missing(tmp_path, monkeypatch):
1915 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
1916 data_dir = tmp_path / "data"
1917 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
1918 path = data_dir / "collections" / "widgets" / "records.tsv"
1919 oidx_path = object_records._oidx_path(path)
1920
1921 object_records.create_collection_record(
1922 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[]
1923 )
1924 object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
1925 assert oidx_path.exists()
1926
1927 oidx_path.unlink()
1928 object_records._OIDX_CACHE.clear()
1929
1930 fetched = object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
1931 assert fetched == {"id": "w1", "name": "Alpha"}
1932 assert oidx_path.exists()
1933
1934
1935def test_oidx_sidecar_rebuilds_after_compaction_changes_inode(tmp_path, monkeypatch):
1936 """Compaction rewrites records.tsv atomically (new inode) and
1937 explicitly discards the sidecar (docs/append-only-storage-design.md:
1938 "delete or rebuild sidecar after the rewrite") -- a subsequent point
1939 op rebuilds correctly against the compacted content."""
1940 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
1941 data_dir = tmp_path / "data"
1942 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
1943 path = data_dir / "collections" / "widgets" / "records.tsv"
1944 oidx_path = object_records._oidx_path(path)
1945
1946 object_records.create_collection_record(
1947 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[]
1948 )
1949 object_records.update_collection_record(
1950 "widgets", "w1", {"name": "Alpha2"}, base_dir=data_dir, roots=[]
1951 )
1952 object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
1953 assert oidx_path.exists()
1954 ino_before = path.stat().st_ino
1955
1956 object_records.compact_collection("widgets", base_dir=data_dir, roots=[])
1957 assert not oidx_path.exists()
1958 assert path.stat().st_ino != ino_before
1959
1960 fetched = object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
1961 assert fetched == {"id": "w1", "name": "Alpha2"}
1962 assert oidx_path.exists()
1963
1964
1965def test_oidx_sidecar_torn_idx_tail_is_dropped(tmp_path, monkeypatch):
1966 """A torn sidecar line (no trailing newline on its own final physical
1967 line, from a crash mid idx-write) is dropped, not resurrected as a
1968 bogus entry -- the catch-up scan of records.tsv resupplies the
1969 correct data for whatever the torn line was trying to record."""
1970 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
1971 data_dir = tmp_path / "data"
1972 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
1973 path = data_dir / "collections" / "widgets" / "records.tsv"
1974 oidx_path = object_records._oidx_path(path)
1975
1976 object_records.create_collection_record(
1977 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[]
1978 )
1979 object_records.create_collection_record(
1980 "widgets", {"id": "w2", "name": "Beta"}, base_dir=data_dir, roots=[]
1981 )
1982 object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
1983 assert oidx_path.exists()
1984
1985 text = oidx_path.read_text()
1986 assert text.endswith("\n")
1987 oidx_path.write_text(text[:-4])
1988 assert not oidx_path.read_text().endswith("\n")
1989
1990 object_records._OIDX_CACHE.clear()
1991 id_offsets, coherent = object_records._load_oidx(path)
1992 assert coherent
1993 assert set(id_offsets) == {"w1", "w2"}
1994 assert object_records.get_collection_record(
1995 "widgets", "w2", base_dir=data_dir, roots=[]
1996 ) == {"id": "w2", "name": "Beta"}
1997
1998
1999def test_oidx_sidecar_survives_torn_data_tail_repair(tmp_path, monkeypatch):
2000 """A crash mid-write can leave records.tsv itself with a torn final
2001 row. The next append self-heals it (_repair_torn_tail truncates the
2002 fragment away before appending), which can leave a previously-built
2003 sidecar's covered_bytes pointing past the (now shorter, then
2004 re-grown) file -- _load_oidx must detect and rebuild rather than
2005 trust stale offsets, at every step of that sequence."""
2006 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
2007 data_dir = tmp_path / "data"
2008 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2009 object_records.create_collection_record(
2010 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[]
2011 )
2012 object_records.create_collection_record(
2013 "widgets", {"id": "w2", "name": "Beta"}, base_dir=data_dir, roots=[]
2014 )
2015 path = data_dir / "collections" / "widgets" / "records.tsv"
2016 oidx_path = object_records._oidx_path(path)
2017 object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
2018 assert oidx_path.exists()
2019
2020 full_text = path.read_text()
2021 path.write_text(full_text[:-3])
2022 assert not path.read_text().endswith("\n")
2023
2024 object_records._OIDX_CACHE.clear()
2025 listing = object_records.list_collection_records("widgets", base_dir=data_dir, roots=[])["records"]
2026 assert listing == [{"id": "w1", "name": "Alpha"}]
2027
2028 object_records._OIDX_CACHE.clear()
2029 created = object_records.create_collection_record(
2030 "widgets", {"id": "w3", "name": "Gamma"}, base_dir=data_dir, roots=[]
2031 )
2032 assert created == {"id": "w3", "name": "Gamma"}
2033 assert path.read_text().endswith("\n")
2034
2035 object_records._OIDX_CACHE.clear()
2036 assert object_records.get_collection_record(
2037 "widgets", "w3", base_dir=data_dir, roots=[]
2038 ) == {"id": "w3", "name": "Gamma"}
2039 listing_after = object_records.list_collection_records(
2040 "widgets", base_dir=data_dir, roots=[]
2041 )["records"]
2042 assert listing_after == [{"id": "w1", "name": "Alpha"}, {"id": "w3", "name": "Gamma"}]
2043
2044
2045def test_oidx_sidecar_resurrection_after_delete(tmp_path, monkeypatch):
2046 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
2047 data_dir = tmp_path / "data"
2048 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2049 path = data_dir / "collections" / "widgets" / "records.tsv"
2050 oidx_path = object_records._oidx_path(path)
2051
2052 object_records.create_collection_record(
2053 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[]
2054 )
2055 object_records.create_collection_record(
2056 "widgets", {"id": "w2", "name": "Beta"}, base_dir=data_dir, roots=[]
2057 )
2058 assert oidx_path.exists() # built by w2's duplicate check
2059
2060 object_records.delete_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
2061 with pytest.raises(object_records.RecordNotFoundError):
2062 object_records.get_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
2063
2064 # Resurrect: creating "w1" again must succeed (not raise Duplicate),
2065 # resolved via the sidecar's absence-is-authoritative answer.
2066 resurrected = object_records.create_collection_record(
2067 "widgets", {"id": "w1", "name": "Gamma"}, base_dir=data_dir, roots=[]
2068 )
2069 assert resurrected == {"id": "w1", "name": "Gamma"}
2070 assert object_records.get_collection_record(
2071 "widgets", "w1", base_dir=data_dir, roots=[]
2072 ) == {"id": "w1", "name": "Gamma"}
2073
2074
2075def test_classic_mode_never_creates_oidx_sidecar(tmp_path, monkeypatch):
2076 """A classic-mode collection's records.tsv is never `_op`-columned,
2077 so _fast_record_lookup's sidecar branch is never reached -- confirm
2078 no `.records.oidx` ever appears, across create/update/delete/get/list,
2079 even with the cache forced cold on every single op."""
2080 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
2081 data_dir = tmp_path / "data"
2082 write_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2083 path = data_dir / "collections" / "widgets" / "records.tsv"
2084 oidx_path = object_records._oidx_path(path)
2085
2086 for i in range(5):
2087 object_records.create_collection_record(
2088 "widgets", {"id": f"w{i}", "name": f"n{i}"}, base_dir=data_dir, roots=[]
2089 )
2090 for i in range(5):
2091 object_records.update_collection_record(
2092 "widgets", f"w{i}", {"name": f"n{i}-updated"}, base_dir=data_dir, roots=[]
2093 )
2094 object_records.delete_collection_record("widgets", "w0", base_dir=data_dir, roots=[])
2095 for i in range(1, 5):
2096 object_records.get_collection_record("widgets", f"w{i}", base_dir=data_dir, roots=[])
2097 object_records.list_collection_records("widgets", base_dir=data_dir, roots=[])
2098
2099 assert not oidx_path.exists()
2100 assert str(path.resolve()) not in object_records._OIDX_CACHE
2101
2102
2103def test_append_mode_equivalent_to_classic_mode_with_sidecar_forced(tmp_path, monkeypatch):
2104 """Same equivalence property as
2105 test_append_mode_equivalent_to_classic_mode_across_random_operations,
2106 but with DBBASIC_RECORDS_CACHE_MAX_ROWS forced to 0 so every single
2107 op on both sides is a _RECORDS_CACHE miss -- meaning every append-side
2108 create/update/delete/get resolves through the id->offset sidecar
2109 (build, catch-up, or its full-rewrite fallback) for the entire 200-op
2110 sequence, never the warm-cache fast path."""
2111 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
2112 classic_dir = tmp_path / "classic"
2113 append_dir = tmp_path / "append"
2114 write_schema(classic_dir, "items", [{"name": "id"}, {"name": "value"}])
2115 write_append_schema(append_dir, "items", [{"name": "id"}, {"name": "value"}])
2116
2117 def apply(data_dir, kind, record_id, payload):
2118 try:
2119 if kind == "create":
2120 return ("ok", object_records.create_collection_record(
2121 "items", {"id": record_id, "value": payload}, base_dir=data_dir, roots=[]
2122 ))
2123 if kind == "update":
2124 return ("ok", object_records.update_collection_record(
2125 "items", record_id, {"value": payload}, base_dir=data_dir, roots=[]
2126 ))
2127 if kind == "delete":
2128 return ("ok", object_records.delete_collection_record(
2129 "items", record_id, base_dir=data_dir, roots=[]
2130 ))
2131 if kind == "get":
2132 return ("ok", object_records.get_collection_record(
2133 "items", record_id, base_dir=data_dir, roots=[]
2134 ))
2135 return ("ok", object_records.list_collection_records(
2136 "items", base_dir=data_dir, roots=[]
2137 ))
2138 except Exception as exc: # noqa: BLE001 -- comparing exception SHAPES across both runs
2139 return ("error", type(exc), str(exc))
2140
2141 rng = random.Random(20260718)
2142 id_pool = [f"e{i}" for i in range(12)]
2143
2144 for step in range(200):
2145 kind = rng.choice(["create", "update", "delete", "get", "list"])
2146 record_id = rng.choice(id_pool)
2147 payload = f"v{step}"
2148
2149 classic_outcome = apply(classic_dir, kind, record_id, payload)
2150 append_outcome = apply(append_dir, kind, record_id, payload)
2151
2152 assert classic_outcome == append_outcome, (
2153 f"step {step} op={kind!r} id={record_id!r}: "
2154 f"classic={classic_outcome!r} append={append_outcome!r}"
2155 )
2156
2157 classic_listing = object_records.list_collection_records("items", base_dir=classic_dir, roots=[])
2158 append_listing = object_records.list_collection_records("items", base_dir=append_dir, roots=[])
2159 assert classic_listing == append_listing
2160
2161
2162# ---------------------------------------------------------------------------
2163# Records-cache tail-delta incremental read (docs/append-only-storage-
2164# design.md Sidecars, bullet 2). Every test below writes external append
2165# rows directly with append_physical_rows -- bypassing object_records'
2166# API and its own cache-refresh calls entirely -- to simulate a SEPARATE
2167# writer (another process, the CLI, a sibling pool worker) growing an
2168# append-mode file underneath a cache entry this process already has warm.
2169# ---------------------------------------------------------------------------
2170
2171
2172def test_incremental_cache_folds_external_append_without_full_reparse(tmp_path, monkeypatch):
2173 """A warm cache entry for an append-mode file, grown in place by an
2174 external writer (same inode, pure growth), must be caught up via the
2175 tail-delta incremental fold -- not a full re-parse of the whole file.
2176 Proven two ways: the module's incremental-fold counter advances by
2177 exactly one, and _parse_records_file is monkeypatched to blow up if
2178 it's ever called from this point on."""
2179 data_dir = tmp_path / "data"
2180 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2181 object_records.create_collection_record(
2182 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[]
2183 )
2184 warm = object_records.read_collection_records("widgets", base_dir=data_dir, roots=[])
2185 assert warm == [{"id": "w1", "name": "Alpha"}]
2186
2187 path = data_dir / "collections" / "widgets" / "records.tsv"
2188 inode_before = path.stat().st_ino
2189
2190 def _boom(*_args, **_kwargs):
2191 raise AssertionError(
2192 "_parse_records_file ran: the incremental tail-delta path did not take effect"
2193 )
2194
2195 monkeypatch.setattr(object_records, "_parse_records_file", _boom)
2196
2197 append_physical_rows(
2198 path,
2199 [object_records.OP_FIELD, "id", "name"],
2200 [{"id": "w2", "name": "Beta", object_records.OP_FIELD: ""}],
2201 )
2202 assert path.stat().st_ino == inode_before # confirms this was growth-in-place, not a rewrite
2203
2204 before_folds = object_records._INCREMENTAL_CACHE_FOLDS
2205 result = object_records.read_collection_records("widgets", base_dir=data_dir, roots=[])
2206 assert result == [{"id": "w1", "name": "Alpha"}, {"id": "w2", "name": "Beta"}]
2207 assert object_records._INCREMENTAL_CACHE_FOLDS == before_folds + 1
2208
2209
2210def test_incremental_cache_folds_external_update_at_same_position(tmp_path):
2211 """An externally appended upsert for an id already present must
2212 replace it AT ITS EXISTING POSITION (matching a full fold's
2213 last-wins-in-place semantics -- see _fold_append_rows), not move it
2214 to the end. Cross-checked against a forced cold full parse of the
2215 identical on-disk content."""
2216 data_dir = tmp_path / "data"
2217 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2218 for i in range(3):
2219 object_records.create_collection_record(
2220 "widgets", {"id": f"w{i}", "name": f"n{i}"}, base_dir=data_dir, roots=[]
2221 )
2222 warm = object_records.read_collection_records("widgets", base_dir=data_dir, roots=[])
2223 assert [record["id"] for record in warm] == ["w0", "w1", "w2"]
2224
2225 path = data_dir / "collections" / "widgets" / "records.tsv"
2226 before_folds = object_records._INCREMENTAL_CACHE_FOLDS
2227 append_physical_rows(
2228 path,
2229 [object_records.OP_FIELD, "id", "name"],
2230 [{"id": "w1", "name": "n1-updated", object_records.OP_FIELD: ""}],
2231 )
2232
2233 result = object_records.read_collection_records("widgets", base_dir=data_dir, roots=[])
2234 assert object_records._INCREMENTAL_CACHE_FOLDS == before_folds + 1
2235 assert result == [
2236 {"id": "w0", "name": "n0"},
2237 {"id": "w1", "name": "n1-updated"},
2238 {"id": "w2", "name": "n2"},
2239 ]
2240
2241 # Cross-check against an independent cold full parse of the exact same
2242 # on-disk bytes -- the cache is untouched by this call.
2243 _, cold_records = object_records._parse_records_file("widgets", path)
2244 assert cold_records == result
2245
2246
2247def test_incremental_cache_falls_back_to_full_fold_on_tombstone(tmp_path):
2248 """A delta containing ANY tombstone must bail out of the incremental
2249 merge entirely and fall back to a full fold (deletes shift every
2250 later position, which the cheap replace-or-append merge does not
2251 model) -- but the RESULT must still be correct, and the fold counter
2252 must show the incremental path did NOT run."""
2253 data_dir = tmp_path / "data"
2254 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2255 for i in range(3):
2256 object_records.create_collection_record(
2257 "widgets", {"id": f"w{i}", "name": f"n{i}"}, base_dir=data_dir, roots=[]
2258 )
2259 object_records.read_collection_records("widgets", base_dir=data_dir, roots=[]) # warm it
2260
2261 path = data_dir / "collections" / "widgets" / "records.tsv"
2262 before_folds = object_records._INCREMENTAL_CACHE_FOLDS
2263 append_physical_rows(
2264 path,
2265 [object_records.OP_FIELD, "id", "name"],
2266 [{"id": "w1", "name": "", object_records.OP_FIELD: object_records.OP_DELETE}],
2267 )
2268
2269 result = object_records.read_collection_records("widgets", base_dir=data_dir, roots=[])
2270 assert object_records._INCREMENTAL_CACHE_FOLDS == before_folds # did NOT take the fast path
2271 assert result == [{"id": "w0", "name": "n0"}, {"id": "w2", "name": "n2"}]
2272
2273
2274def test_incremental_cache_ignores_torn_tail_fragment_in_delta(tmp_path):
2275 """A torn (unterminated) fragment at EOF -- the classic log torn-tail
2276 case, from a writer that crashed mid-row -- must be excluded from the
2277 incremental fold's delta, exactly as a full fold would exclude it,
2278 and must never resurface once a later, legitimate write repairs and
2279 extends the file."""
2280 data_dir = tmp_path / "data"
2281 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2282 object_records.create_collection_record(
2283 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[]
2284 )
2285 object_records.read_collection_records("widgets", base_dir=data_dir, roots=[]) # warm it
2286
2287 path = data_dir / "collections" / "widgets" / "records.tsv"
2288 append_physical_rows(
2289 path,
2290 [object_records.OP_FIELD, "id", "name"],
2291 [{"id": "w2", "name": "Beta", object_records.OP_FIELD: ""}],
2292 )
2293 # A crash mid-write: a fragment with no trailing newline.
2294 with path.open("a", newline="") as handle:
2295 handle.write("\tw3\tGam")
2296 assert not path.read_text().endswith("\n")
2297
2298 result = object_records.read_collection_records("widgets", base_dir=data_dir, roots=[])
2299 assert result == [{"id": "w1", "name": "Alpha"}, {"id": "w2", "name": "Beta"}]
2300
2301 # A subsequent legitimate write must self-heal (truncate the fragment
2302 # away) and append cleanly -- the fragment must never come back.
2303 created = object_records.create_collection_record(
2304 "widgets", {"id": "w3", "name": "Gamma"}, base_dir=data_dir, roots=[]
2305 )
2306 assert created == {"id": "w3", "name": "Gamma"}
2307 assert path.read_text().endswith("\n")
2308 listing = object_records.list_collection_records(
2309 "widgets", base_dir=data_dir, roots=[]
2310 )["records"]
2311 assert listing == [
2312 {"id": "w1", "name": "Alpha"},
2313 {"id": "w2", "name": "Beta"},
2314 {"id": "w3", "name": "Gamma"},
2315 ]
2316
2317
2318def test_incremental_cache_evicts_when_growth_crosses_row_threshold(tmp_path, monkeypatch):
2319 """A successful incremental fold that pushes a collection's row count
2320 past DBBASIC_RECORDS_CACHE_MAX_ROWS must still return the correct
2321 (larger) result, but must NOT leave an entry cached -- same rule the
2322 full-parse path already applies (test_records_cache_row_threshold_
2323 excludes_large_collections)."""
2324 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "3")
2325 data_dir = tmp_path / "data"
2326 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2327 for i in range(3):
2328 object_records.create_collection_record(
2329 "widgets", {"id": f"w{i}", "name": f"n{i}"}, base_dir=data_dir, roots=[]
2330 )
2331 object_records.read_collection_records("widgets", base_dir=data_dir, roots=[])
2332 cache_key = _cache_key("widgets", data_dir)
2333 assert cache_key in object_records._RECORDS_CACHE
2334
2335 path = data_dir / "collections" / "widgets" / "records.tsv"
2336 append_physical_rows(
2337 path,
2338 [object_records.OP_FIELD, "id", "name"],
2339 [
2340 {"id": "w3", "name": "n3", object_records.OP_FIELD: ""},
2341 {"id": "w4", "name": "n4", object_records.OP_FIELD: ""},
2342 ],
2343 )
2344
2345 result = object_records.read_collection_records("widgets", base_dir=data_dir, roots=[])
2346 assert len(result) == 5
2347 assert cache_key not in object_records._RECORDS_CACHE
2348
2349 # Must still reparse correctly on the very next read (uncached, but
2350 # correct every time).
2351 result_again = object_records.read_collection_records("widgets", base_dir=data_dir, roots=[])
2352 assert result_again == result
2353
2354
2355def test_incremental_cache_equivalence_across_random_external_append_batches(tmp_path):
2356 """PROPERTY: for a long, seeded, randomized sequence of external
2357 append batches (each batch a handful of upserts and occasional
2358 deletes, written directly to disk exactly as a separate process
2359 would), the WARM read path -- whichever of incremental-fold or
2360 fallback-full-fold it takes -- must always agree with an independent
2361 COLD full parse of the identical on-disk bytes. ~50 iterations."""
2362 data_dir = tmp_path / "data"
2363 write_append_schema(data_dir, "items", [{"name": "id"}, {"name": "value"}])
2364 path = data_dir / "collections" / "items" / "records.tsv"
2365 physical_fields = [object_records.OP_FIELD, "id", "value"]
2366
2367 object_records.create_collection_record(
2368 "items", {"id": "seed", "value": "0"}, base_dir=data_dir, roots=[]
2369 )
2370 object_records.read_collection_records("items", base_dir=data_dir, roots=[]) # warm once
2371
2372 rng = random.Random(20260718)
2373 id_pool = [f"e{i}" for i in range(8)]
2374
2375 for iteration in range(50):
2376 batch = []
2377 for _ in range(rng.randint(1, 4)):
2378 record_id = rng.choice(id_pool)
2379 if rng.random() < 0.15:
2380 batch.append({
2381 "id": record_id,
2382 "value": "",
2383 object_records.OP_FIELD: object_records.OP_DELETE,
2384 })
2385 else:
2386 batch.append({
2387 "id": record_id,
2388 "value": f"v{iteration}",
2389 object_records.OP_FIELD: "",
2390 })
2391 append_physical_rows(path, physical_fields, batch)
2392
2393 warm_result = object_records.read_collection_records("items", base_dir=data_dir, roots=[])
2394 _, cold_records = object_records._parse_records_file("items", path)
2395 assert warm_result == cold_records, (
2396 f"iteration {iteration}: warm={warm_result!r} cold={cold_records!r}"
2397 )
2398
2399
2400# --- Compaction observability (docs/append-only-storage-design.md
2401# "Compaction", docs/storage-modes.md) ---------------------------------------
2402
2403
2404def _seed_widgets(data_dir: Path) -> None:
2405 """5 creates, 1 update (supersedes), 1 delete (tombstone): 7 physical
2406 rows, 4 live (w0 updated, w1 deleted -> 3 remain plus updated w0), 3
2407 dead. Exact numbers every stats test below relies on."""
2408 for i in range(5):
2409 object_records.create_collection_record(
2410 "widgets", {"id": f"w{i}", "name": f"n{i}"}, base_dir=data_dir, roots=[]
2411 )
2412 object_records.update_collection_record(
2413 "widgets", "w0", {"name": "n0-updated"}, base_dir=data_dir, roots=[]
2414 )
2415 object_records.delete_collection_record("widgets", "w1", base_dir=data_dir, roots=[])
2416
2417
2418def test_append_collection_stats_is_none_for_classic_collections(tmp_path):
2419 data_dir = tmp_path / "data"
2420 write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
2421 assert object_records.append_collection_stats("contacts", base_dir=data_dir, roots=[]) is None
2422
2423
2424def test_append_collection_stats_is_none_for_schema_with_no_storage_key(tmp_path):
2425 data_dir = tmp_path / "data"
2426 write_schema(data_dir, "contacts", [{"name": "id"}, {"name": "name"}])
2427 object_records.create_collection_record(
2428 "contacts", {"id": "c1", "name": "Ada"}, base_dir=data_dir, roots=[]
2429 )
2430 assert object_records.append_collection_stats("contacts", base_dir=data_dir, roots=[]) is None
2431
2432
2433def test_append_collection_stats_for_never_written_collection_is_honest_zeros(tmp_path):
2434 data_dir = tmp_path / "data"
2435 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2436 assert object_records.append_collection_stats("widgets", base_dir=data_dir, roots=[]) == {
2437 "collection": "widgets",
2438 "storage": "append",
2439 "physical_rows": 0,
2440 "live_rows": 0,
2441 "dead_rows": 0,
2442 "bloat_ratio": 0.0,
2443 "file_bytes": 0,
2444 "sidecar_present": False,
2445 "compaction_flagged": False,
2446 }
2447
2448
2449def test_append_collection_stats_via_warm_records_cache(tmp_path):
2450 """Default cache threshold: the seeded collection stays warm in
2451 _RECORDS_CACHE, so stats must be served from it (no sidecar
2452 involved)."""
2453 data_dir = tmp_path / "data"
2454 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2455 _seed_widgets(data_dir)
2456
2457 path = object_records.collection_records_file("widgets", base_dir=data_dir)
2458 cache_key = str(path.resolve())
2459 assert cache_key in object_records._RECORDS_CACHE # warm precondition
2460
2461 stats = object_records.append_collection_stats("widgets", base_dir=data_dir, roots=[])
2462 assert stats == {
2463 "collection": "widgets",
2464 "storage": "append",
2465 "physical_rows": 7,
2466 "live_rows": 4,
2467 "dead_rows": 3,
2468 "bloat_ratio": 0.75,
2469 "file_bytes": path.stat().st_size,
2470 "sidecar_present": False,
2471 "compaction_flagged": False,
2472 }
2473
2474
2475def test_append_collection_stats_via_sidecar_with_cold_cache(tmp_path, monkeypatch):
2476 """DBBASIC_RECORDS_CACHE_MAX_ROWS=0 forces every point op through the
2477 id->offset sidecar instead of _RECORDS_CACHE (same technique the oidx
2478 test block above this one uses) -- stats must then be answered by a
2479 cheap sidecar line-count read, not a full fold."""
2480 monkeypatch.setenv("DBBASIC_RECORDS_CACHE_MAX_ROWS", "0")
2481 data_dir = tmp_path / "data"
2482 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2483 _seed_widgets(data_dir)
2484
2485 path = object_records.collection_records_file("widgets", base_dir=data_dir)
2486 cache_key = str(path.resolve())
2487 assert cache_key not in object_records._RECORDS_CACHE # cold precondition
2488 oidx_path = object_records._oidx_path(path)
2489 assert oidx_path.exists() # built lazily by the point ops above
2490
2491 # Belt and suspenders: an actual full fold must not run for this case.
2492 def _boom(*args, **kwargs):
2493 raise AssertionError("full fold ran when the sidecar should have answered")
2494
2495 monkeypatch.setattr(object_records, "_parse_append_body", _boom)
2496
2497 stats = object_records.append_collection_stats("widgets", base_dir=data_dir, roots=[])
2498 assert stats == {
2499 "collection": "widgets",
2500 "storage": "append",
2501 "physical_rows": 7,
2502 "live_rows": 4,
2503 "dead_rows": 3,
2504 "bloat_ratio": 0.75,
2505 "file_bytes": path.stat().st_size,
2506 "sidecar_present": True,
2507 "compaction_flagged": False,
2508 }
2509
2510
2511def test_append_collection_stats_falls_back_to_full_fold_when_allowed(tmp_path, monkeypatch):
2512 """Neither cache nor sidecar available (sidecar never built: default
2513 cache threshold means no point op ever missed _RECORDS_CACHE) --
2514 allow_fold=True (the default) must still produce the correct numbers
2515 via a full parse."""
2516 data_dir = tmp_path / "data"
2517 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2518 _seed_widgets(data_dir)
2519
2520 path = object_records.collection_records_file("widgets", base_dir=data_dir)
2521 oidx_path = object_records._oidx_path(path)
2522 assert not oidx_path.exists() # never built: cache always answered point ops
2523 object_records._RECORDS_CACHE.clear()
2524
2525 stats = object_records.append_collection_stats("widgets", base_dir=data_dir, roots=[])
2526 assert stats == {
2527 "collection": "widgets",
2528 "storage": "append",
2529 "physical_rows": 7,
2530 "live_rows": 4,
2531 "dead_rows": 3,
2532 "bloat_ratio": 0.75,
2533 "file_bytes": path.stat().st_size,
2534 "sidecar_present": False,
2535 "compaction_flagged": False,
2536 }
2537
2538
2539def test_append_collection_stats_allow_fold_false_returns_estimated_without_folding(tmp_path):
2540 """allow_fold=False must never pay for a full parse: with the warm
2541 cache dropped and no sidecar built, it returns an "estimated" entry
2542 (file_bytes only, physical/live/dead/bloat null) instead of folding."""
2543 data_dir = tmp_path / "data"
2544 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2545 _seed_widgets(data_dir)
2546 object_records._RECORDS_CACHE.clear()
2547
2548 path = object_records.collection_records_file("widgets", base_dir=data_dir)
2549 assert not object_records._oidx_path(path).exists()
2550
2551 calls = []
2552 original = object_records._parse_append_body
2553
2554 def _tracking(*args, **kwargs):
2555 calls.append(True)
2556 return original(*args, **kwargs)
2557
2558 with unittest.mock.patch.object(object_records, "_parse_append_body", _tracking):
2559 stats = object_records.append_collection_stats(
2560 "widgets", base_dir=data_dir, roots=[], allow_fold=False
2561 )
2562
2563 assert calls == [] # sentinel: no full parse ran
2564 assert stats == {
2565 "collection": "widgets",
2566 "storage": "append",
2567 "physical_rows": None,
2568 "live_rows": None,
2569 "dead_rows": None,
2570 "bloat_ratio": None,
2571 "file_bytes": path.stat().st_size,
2572 "sidecar_present": False,
2573 "compaction_flagged": False,
2574 "estimated": True,
2575 }
2576
2577
2578def test_append_collection_stats_reflects_compaction_flag(tmp_path, monkeypatch):
2579 monkeypatch.setenv("DBBASIC_APPEND_COMPACT_MIN_ROWS", "5")
2580 data_dir = tmp_path / "data"
2581 write_append_schema(data_dir, "churn", [{"name": "id"}, {"name": "value"}])
2582
2583 object_records.create_collection_record(
2584 "churn", {"id": "c1", "value": "v0"}, base_dir=data_dir, roots=[]
2585 )
2586 for i in range(10):
2587 object_records.update_collection_record(
2588 "churn", "c1", {"value": f"v{i}"}, base_dir=data_dir, roots=[]
2589 )
2590
2591 path = object_records.collection_records_file("churn", base_dir=data_dir)
2592 cache_key = str(path.resolve())
2593 object_records._RECORDS_CACHE.pop(cache_key, None) # force a cold parse to flag
2594 object_records._read_collection_records("churn", base_dir=data_dir)
2595 assert cache_key in object_records._PENDING_COMPACTION
2596
2597 stats = object_records.append_collection_stats("churn", base_dir=data_dir, roots=[])
2598 assert stats["compaction_flagged"] is True
2599
2600
2601def test_append_collection_stats_after_compaction_is_bloat_free(tmp_path):
2602 data_dir = tmp_path / "data"
2603 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2604 _seed_widgets(data_dir)
2605
2606 object_records.compact_collection("widgets", base_dir=data_dir, roots=[])
2607
2608 stats = object_records.append_collection_stats("widgets", base_dir=data_dir, roots=[])
2609 assert stats["physical_rows"] == 4
2610 assert stats["live_rows"] == 4
2611 assert stats["dead_rows"] == 0
2612 assert stats["bloat_ratio"] == 0.0
2613
2614
2615def test_list_append_collection_stats_only_includes_append_mode_collections(tmp_path):
2616 data_dir = tmp_path / "data"
2617 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2618 _seed_widgets(data_dir)
2619 write_schema(data_dir, "contacts", [{"name": "id"}, {"name": "name"}])
2620 object_records.create_collection_record(
2621 "contacts", {"id": "c1", "name": "Ada"}, base_dir=data_dir, roots=[]
2622 )
2623
2624 stats = object_records.list_append_collection_stats(base_dir=data_dir, roots=[])
2625 assert [entry["collection"] for entry in stats] == ["widgets"]
2626 assert stats[0]["physical_rows"] == 7
2627
2628
2629def test_list_append_collection_stats_skips_missing_collection_records_file(tmp_path):
2630 """A schema declaring storage:append with no records.tsv yet must show
2631 up with zeroed stats, not be skipped or raise."""
2632 data_dir = tmp_path / "data"
2633 write_append_schema(data_dir, "empties", [{"name": "id"}])
2634
2635 stats = object_records.list_append_collection_stats(base_dir=data_dir, roots=[])
2636 assert len(stats) == 1
2637 assert stats[0]["collection"] == "empties"
2638 assert stats[0]["physical_rows"] == 0
2639
2640
2641def test_list_append_collection_stats_empty_when_no_append_collections(tmp_path):
2642 data_dir = tmp_path / "data"
2643 write_schema(data_dir, "contacts", [{"name": "id"}])
2644 object_records.create_collection_record(
2645 "contacts", {"id": "c1"}, base_dir=data_dir, roots=[]
2646 )
2647 assert object_records.list_append_collection_stats(base_dir=data_dir, roots=[]) == []
2648
2649
2650# --- Universal attribution: every write emits exactly one attributed change ---
2651# (create/update/delete_collection_record now emit via object_record_changes
2652# themselves, regardless of caller -- HTTP API, CLI, package installs, etc.)
2653
2654
2655def test_create_collection_record_with_no_actor_emits_unattributed_change(tmp_path):
2656 data_dir = tmp_path / "data"
2657 write_records(data_dir, "contacts", "id\tname\n")
2658
2659 object_records.create_collection_record(
2660 "contacts", {"id": "c1", "name": "Ada"}, base_dir=data_dir, roots=[]
2661 )
2662
2663 changes = object_record_changes.list_record_changes("contacts", base_dir=data_dir)["changes"]
2664 assert len(changes) == 1
2665 assert changes[0]["actor"] == "unattributed"
2666 assert changes[0]["action"] == "create"
2667 assert changes[0]["record_id"] == "c1"
2668 assert changes[0]["before"] is None
2669 assert changes[0]["after"] == {"id": "c1", "name": "Ada"}
2670
2671
2672def test_create_collection_record_with_actor_records_it(tmp_path):
2673 data_dir = tmp_path / "data"
2674 write_records(data_dir, "contacts", "id\tname\n")
2675
2676 object_records.create_collection_record(
2677 "contacts", {"id": "c1", "name": "Ada"}, base_dir=data_dir, roots=[], actor="x"
2678 )
2679
2680 changes = object_record_changes.list_record_changes("contacts", base_dir=data_dir)["changes"]
2681 assert len(changes) == 1
2682 assert changes[0]["actor"] == "x"
2683
2684
2685def test_update_collection_record_with_no_actor_emits_unattributed_change(tmp_path):
2686 data_dir = tmp_path / "data"
2687 write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
2688
2689 object_records.update_collection_record(
2690 "contacts", "c1", {"name": "Ada Lovelace"}, base_dir=data_dir, roots=[]
2691 )
2692
2693 changes = object_record_changes.list_record_changes("contacts", base_dir=data_dir)["changes"]
2694 assert len(changes) == 1
2695 assert changes[0]["actor"] == "unattributed"
2696 assert changes[0]["action"] == "update"
2697 assert changes[0]["before"] == {"id": "c1", "name": "Ada"}
2698 assert changes[0]["after"] == {"id": "c1", "name": "Ada Lovelace"}
2699
2700
2701def test_update_collection_record_with_actor_records_it(tmp_path):
2702 data_dir = tmp_path / "data"
2703 write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
2704
2705 object_records.update_collection_record(
2706 "contacts", "c1", {"name": "Ada Lovelace"}, base_dir=data_dir, roots=[], actor="x"
2707 )
2708
2709 changes = object_record_changes.list_record_changes("contacts", base_dir=data_dir)["changes"]
2710 assert len(changes) == 1
2711 assert changes[0]["actor"] == "x"
2712
2713
2714def test_delete_collection_record_with_no_actor_emits_unattributed_change(tmp_path):
2715 data_dir = tmp_path / "data"
2716 write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
2717
2718 object_records.delete_collection_record("contacts", "c1", base_dir=data_dir, roots=[])
2719
2720 changes = object_record_changes.list_record_changes("contacts", base_dir=data_dir)["changes"]
2721 assert len(changes) == 1
2722 assert changes[0]["actor"] == "unattributed"
2723 assert changes[0]["action"] == "delete"
2724 assert changes[0]["before"] == {"id": "c1", "name": "Ada"}
2725 assert changes[0]["after"] is None
2726
2727
2728def test_delete_collection_record_with_actor_records_it(tmp_path):
2729 data_dir = tmp_path / "data"
2730 write_records(data_dir, "contacts", "id\tname\nc1\tAda\n")
2731
2732 object_records.delete_collection_record("contacts", "c1", base_dir=data_dir, roots=[], actor="x")
2733
2734 changes = object_record_changes.list_record_changes("contacts", base_dir=data_dir)["changes"]
2735 assert len(changes) == 1
2736 assert changes[0]["actor"] == "x"
2737
2738
2739def test_compaction_does_not_emit_a_change(tmp_path):
2740 """Compaction is a physical-only rewrite -- it must never emit a change."""
2741 data_dir = tmp_path / "data"
2742 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2743 object_records.create_collection_record(
2744 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[], actor="x"
2745 )
2746 object_records.update_collection_record(
2747 "widgets", "w1", {"name": "Alpha2"}, base_dir=data_dir, roots=[], actor="x"
2748 )
2749 before = object_record_changes.list_record_changes("widgets", base_dir=data_dir)["total"]
2750
2751 object_records.compact_collection("widgets", base_dir=data_dir, roots=[])
2752
2753 after = object_record_changes.list_record_changes("widgets", base_dir=data_dir)["total"]
2754 assert after == before
2755
2756
2757def test_append_mode_create_update_delete_attribution(tmp_path):
2758 """Universal attribution also applies to append-storage collections."""
2759 data_dir = tmp_path / "data"
2760 write_append_schema(data_dir, "widgets", [{"name": "id"}, {"name": "name"}])
2761
2762 object_records.create_collection_record(
2763 "widgets", {"id": "w1", "name": "Alpha"}, base_dir=data_dir, roots=[], actor="seed"
2764 )
2765 object_records.update_collection_record(
2766 "widgets", "w1", {"name": "Alpha2"}, base_dir=data_dir, roots=[]
2767 )
2768 object_records.delete_collection_record("widgets", "w1", base_dir=data_dir, roots=[], actor="cleanup")
2769
2770 changes = object_record_changes.list_record_changes("widgets", base_dir=data_dir)["changes"]
2771 assert [change["action"] for change in changes] == ["delete", "update", "create"]
2772 assert changes[2]["actor"] == "seed"
2773 assert changes[1]["actor"] == "unattributed"
2774 assert changes[0]["actor"] == "cleanup"
2775
2776
2777def test_record_write_emits_exactly_one_change_no_recursion(tmp_path, monkeypatch):
2778 """A record write must emit its own change exactly once -- and must never
2779
2780 trigger a change-about-a-change: object_record_changes.append_record_change
2781 writes its JSONL entry directly (it never calls back into object_records'
2782 create/update/delete_collection_record), so there is no path for one write
2783 to recursively emit a second, unbounded chain of changes.
2784 """
2785 data_dir = tmp_path / "data"
2786 write_records(data_dir, "contacts", "id\tname\n")
2787
2788 calls: list[str] = []
2789 original = object_record_changes.append_record_change
2790
2791 def counting(*args, **kwargs):
2792 calls.append(kwargs.get("collection"))
2793 return original(*args, **kwargs)
2794
2795 monkeypatch.setattr(object_record_changes, "append_record_change", counting)
2796
2797 object_records.create_collection_record(
2798 "contacts", {"id": "c1", "name": "Ada"}, base_dir=data_dir, roots=[], actor="x"
2799 )
2800
2801 # Exactly one emission for one write -- not zero, not two, not a chain.
2802 assert calls == ["contacts"]
2803 # The change log itself is never stored as a queryable collection record,
2804 # so there is nothing for a write to recurse into in the first place.
2805 assert not (data_dir / "collections" / "record_changes").exists()
2806
2807
2808def test_record_change_entry_shape_is_unchanged(tmp_path):
2809 """Backward compat: the stored change entry keeps its original shape
2810 whether emission is triggered from the write path or from a direct
2811 object_record_changes.append_record_change call.
2812 """
2813 data_dir = tmp_path / "data"
2814 write_records(data_dir, "contacts", "id\tname\n")
2815
2816 object_records.create_collection_record(
2817 "contacts", {"id": "c1", "name": "Ada"}, base_dir=data_dir, roots=[], actor="x"
2818 )
2819
2820 entry = object_record_changes.list_record_changes("contacts", base_dir=data_dir)["changes"][0]
2821 assert set(entry) == {
2822 "change_id",
2823 "timestamp",
2824 "collection",
2825 "record_id",
2826 "action",
2827 "actor",
2828 "message",
2829 "correlation_id",
2830 "changed_fields",
2831 "before",
2832 "after",
2833 }
2834 UUID(entry["change_id"])
2835
2836
2837# --- 58 query/filter: filter_records, list_collection_records(where=...), ---
2838# --- normalize_filter_value --------------------------------------------
2839
2840
2841def cond(op, value):
2842 return object_permissions.filter_condition(op, value)
2843
2844
2845CONTACTS_TSV = (
2846 "id\tname\tstatus\towner_id\ttotal\tcreated_at\n"
2847 "c1\tAda\thot\t7\t150\t2026-07-01\n"
2848 "c2\tGrace\tcold\t7\t50\t2026-07-15\n"
2849 "c3\tKatherine\thot\t8\t300\t2026-07-20\n"
2850 "c4\tHedy\topen\t8\t20\t2026-06-01\n"
2851)
2852
2853
2854def test_filter_records_eq_default_operator(tmp_path):
2855 data_dir = tmp_path / "data"
2856 write_records(data_dir, "contacts", CONTACTS_TSV)
2857 records = object_records.read_collection_records("contacts", base_dir=data_dir, roots=[])
2858
2859 matched = object_records.filter_records(records, {"status": "hot"})
2860
2861 assert [r["id"] for r in matched] == ["c1", "c3"]
2862
2863
2864def test_filter_records_ne(tmp_path):
2865 data_dir = tmp_path / "data"
2866 write_records(data_dir, "contacts", CONTACTS_TSV)
2867 records = object_records.read_collection_records("contacts", base_dir=data_dir, roots=[])
2868
2869 matched = object_records.filter_records(records, {"status": cond("ne", "hot")})
2870
2871 assert [r["id"] for r in matched] == ["c2", "c4"]
2872
2873
2874def test_filter_records_in(tmp_path):
2875 data_dir = tmp_path / "data"
2876 write_records(data_dir, "contacts", CONTACTS_TSV)
2877 records = object_records.read_collection_records("contacts", base_dir=data_dir, roots=[])
2878
2879 matched = object_records.filter_records(records, {"status": cond("in", ("hot", "open"))})
2880
2881 assert [r["id"] for r in matched] == ["c1", "c3", "c4"]
2882
2883
2884def test_filter_records_ordered_operators(tmp_path):
2885 data_dir = tmp_path / "data"
2886 write_records(data_dir, "contacts", CONTACTS_TSV)
2887 records = object_records.read_collection_records("contacts", base_dir=data_dir, roots=[])
2888
2889 assert [r["id"] for r in object_records.filter_records(records, {"total": cond("gte", "150")})] == [
2890 "c1",
2891 "c3",
2892 ]
2893 assert [r["id"] for r in object_records.filter_records(records, {"total": cond("lte", "50")})] == [
2894 "c2",
2895 "c4",
2896 ]
2897 assert [r["id"] for r in object_records.filter_records(records, {"total": cond("gt", "150")})] == ["c3"]
2898 assert [r["id"] for r in object_records.filter_records(records, {"total": cond("lt", "50")})] == ["c4"]
2899
2900
2901def test_filter_records_multiple_conditions_are_anded_range_query(tmp_path):
2902 data_dir = tmp_path / "data"
2903 write_records(data_dir, "contacts", CONTACTS_TSV)
2904 records = object_records.read_collection_records("contacts", base_dir=data_dir, roots=[])
2905
2906 where = {"created_at": [cond("gte", "2026-07-01"), cond("lte", "2026-07-15")]}
2907 matched = object_records.filter_records(records, where)
2908
2909 assert [r["id"] for r in matched] == ["c1", "c2"]
2910
2911
2912def test_filter_records_no_where_returns_full_copy_not_alias(tmp_path):
2913 data_dir = tmp_path / "data"
2914 write_records(data_dir, "contacts", CONTACTS_TSV)
2915 records = object_records.read_collection_records("contacts", base_dir=data_dir, roots=[])
2916
2917 matched_none = object_records.filter_records(records, None)
2918 matched_empty = object_records.filter_records(records, {})
2919
2920 assert matched_none == records
2921 assert matched_empty == records
2922 assert matched_none[0] is not records[0]
2923 matched_none[0]["name"] = "mutated"
2924 assert records[0]["name"] == "Ada"
2925
2926
2927def test_filter_records_matching_nothing_is_an_empty_list_not_an_error(tmp_path):
2928 data_dir = tmp_path / "data"
2929 write_records(data_dir, "contacts", CONTACTS_TSV)
2930 records = object_records.read_collection_records("contacts", base_dir=data_dir, roots=[])
2931
2932 matched = object_records.filter_records(records, {"status": "nonexistent"})
2933
2934 assert matched == []
2935
2936
2937def test_filter_records_can_only_narrow_its_own_input_never_widen(tmp_path):
2938 """The structural narrow-only guarantee (58's Permissions Posture):
2939 filter_records only ever removes rows from what it is handed. Feed it
2940 an already-narrowed (e.g. row-permission-filtered) subset and a filter
2941 that -- if it read the ORIGINAL collection -- would match a row outside
2942 that subset; the excluded row can never reappear."""
2943 data_dir = tmp_path / "data"
2944 write_records(data_dir, "contacts", CONTACTS_TSV)
2945 all_records = object_records.read_collection_records("contacts", base_dir=data_dir, roots=[])
2946 # Simulate a permission row filter that already excluded c3/c4 (owner_id 8).
2947 row_filtered = [r for r in all_records if r["owner_id"] == "7"]
2948
2949 # A field filter matching owner_id=8 rows (c3, c4) against the FULL
2950 # collection would return them -- but applied to the already-narrowed
2951 # input, it can only ever be a subset of that input.
2952 matched = object_records.filter_records(row_filtered, {"owner_id": "8"})
2953
2954 assert matched == []
2955 assert all(record["id"] in {"c1", "c2"} for record in row_filtered)
2956
2957
2958def test_list_collection_records_where_filters_before_pagination(tmp_path):
2959 data_dir = tmp_path / "data"
2960 write_records(data_dir, "contacts", CONTACTS_TSV)
2961
2962 result = object_records.list_collection_records(
2963 "contacts",
2964 base_dir=data_dir,
2965 roots=[],
2966 where={"status": "hot"},
2967 limit=1,
2968 offset=1,
2969 )
2970
2971 assert result["total"] == 2
2972 assert result["count"] == 1
2973 assert result["records"] == [{"id": "c3", "name": "Katherine", "status": "hot", "owner_id": "8", "total": "300", "created_at": "2026-07-20"}]
2974 assert result["has_more"] is False
2975
2976
2977def test_list_collection_records_where_none_is_unaffected(tmp_path):
2978 data_dir = tmp_path / "data"
2979 write_records(data_dir, "contacts", CONTACTS_TSV)
2980
2981 result = object_records.list_collection_records("contacts", base_dir=data_dir, roots=[], limit=2)
2982
2983 assert result["total"] == 4
2984 assert result["count"] == 2
2985
2986
2987def test_list_collection_records_where_on_append_mode_sees_folded_state(tmp_path):
2988 """58's Storage/Append-mode note: a filter applies to the folded
2989 (current) view, never a superseded physical version or a tombstone."""
2990 data_dir = tmp_path / "data"
2991 write_append_schema(
2992 data_dir,
2993 "widgets",
2994 [{"name": "id"}, {"name": "name"}, {"name": "status"}],
2995 )
2996
2997 object_records.create_collection_record(
2998 "widgets", {"id": "w1", "name": "Alpha", "status": "draft"}, base_dir=data_dir, roots=[]
2999 )
3000 object_records.create_collection_record(
3001 "widgets", {"id": "w2", "name": "Beta", "status": "live"}, base_dir=data_dir, roots=[]
3002 )
3003 object_records.create_collection_record(
3004 "widgets", {"id": "w3", "name": "Gamma", "status": "live"}, base_dir=data_dir, roots=[]
3005 )
3006 # w1 supersedes to "live" -- an unfiltered pre-update read would have
3007 # missed it under status=live; the fold must happen before filtering.
3008 object_records.update_collection_record(
3009 "widgets", "w1", {"status": "live"}, base_dir=data_dir, roots=[]
3010 )
3011 # w3 is tombstoned -- must never show up in a filtered result either.
3012 object_records.delete_collection_record("widgets", "w3", base_dir=data_dir, roots=[])
3013
3014 result = object_records.list_collection_records(
3015 "widgets", base_dir=data_dir, roots=[], where={"status": "live"}
3016 )
3017
3018 assert sorted(r["id"] for r in result["records"]) == ["w1", "w2"]
3019 assert result["total"] == 2
3020
3021
3022def test_normalize_filter_value_validates_and_normalizes_boolean(tmp_path):
3023 field = {"name": "is_public", "type": "boolean"}
3024
3025 assert object_records.normalize_filter_value(field, "eq", "1") == "true"
3026 assert object_records.normalize_filter_value(field, "eq", "yes") == "true"
3027 assert object_records.normalize_filter_value(field, "eq", "0") == "false"
3028 with pytest.raises(object_records.InvalidRecordPayloadError):
3029 object_records.normalize_filter_value(field, "eq", "maybe")
3030
3031
3032def test_normalize_filter_value_validates_types(tmp_path):
3033 int_field = {"name": "total", "type": "integer"}
3034 date_field = {"name": "created_at", "type": "date"}
3035 enum_field = {"name": "status", "type": "enum", "enum": ["draft", "live"]}
3036
3037 assert object_records.normalize_filter_value(int_field, "gte", "150") == "150"
3038 with pytest.raises(object_records.InvalidRecordPayloadError):
3039 object_records.normalize_filter_value(int_field, "eq", "not-a-number")
3040
3041 assert object_records.normalize_filter_value(date_field, "gte", "2026-07-01") == "2026-07-01"
3042 with pytest.raises(object_records.InvalidRecordPayloadError):
3043 object_records.normalize_filter_value(date_field, "eq", "not-a-date")
3044
3045 assert object_records.normalize_filter_value(enum_field, "eq", "live") == "live"
3046 with pytest.raises(object_records.InvalidRecordPayloadError):
3047 object_records.normalize_filter_value(enum_field, "eq", "bogus")
3048
3049
3050def test_normalize_filter_value_rejects_ordered_op_on_unsupported_type(tmp_path):
3051 text_field = {"name": "name", "type": "text"}
3052
3053 with pytest.raises(object_records.InvalidRecordPayloadError, match="does not support operator"):
3054 object_records.normalize_filter_value(text_field, "gte", "A")
3055
3056 # eq/ne/in stay fine for text.
3057 assert object_records.normalize_filter_value(text_field, "eq", "Ada") == "Ada"
3058
3059
3060def test_normalize_filter_value_permissive_when_field_is_none(tmp_path):
3061 """A schemaless/derived collection has no field def to validate
3062 against; 58 treats that as permissive, matching every other
3063 schema-optional check in this module."""
3064 assert object_records.normalize_filter_value(None, "gte", "anything") == "anything"