Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_object_import.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_object_import.py · 21.1 KB · 548 lines
1"""Tests for object_import.py (plan/vocabulary/13-import-export-spec.md, CLI slice).
2
3Fixtures use synthetic collection/schema names ("legacy_accounts",
4"legacy_widgets") standing in for an arbitrary spreadsheet import -- never
5real data.
6"""
7
8from __future__ import annotations
9
10import json
11from pathlib import Path
12
13import pytest
14
15import object_collections
16import object_import
17import object_record_changes
18import object_records
19
20
21# --------------------------------------------------------------------------
22# Fixture helpers
23# --------------------------------------------------------------------------
24
25def write_schema(data_dir: Path, collection: str, fields: list[dict]) -> Path:
26 path = data_dir / "schemas" / f"{collection}.json"
27 path.parent.mkdir(parents=True, exist_ok=True)
28 path.write_text(json.dumps({"name": collection, "fields": fields}))
29 return path
30
31
32def write_json(path: Path, payload) -> Path:
33 path.parent.mkdir(parents=True, exist_ok=True)
34 path.write_text(json.dumps(payload))
35 return path
36
37
38ACCOUNTS_FIELDS = [
39 {"name": "id"},
40 {
41 "name": "name",
42 "type": "text",
43 "required": True,
44 "validation": {"min_length": 1, "max_length": 120},
45 },
46 {"name": "tier", "type": "enum", "enum": ["bronze", "silver", "gold"], "default": "bronze"},
47 {"name": "created_at", "type": "datetime", "read_only": True},
48]
49
50WIDGETS_FIELDS = [
51 {"name": "id"},
52 {"name": "title", "type": "text", "required": True},
53 {"name": "account_id", "relation": {"collection": "legacy_accounts", "display_field": "name"}},
54 {"name": "priority", "type": "enum", "enum": ["low", "normal", "high"], "default": "normal"},
55 {"name": "source_system", "type": "text"},
56 {"name": "created_at", "type": "datetime", "read_only": True},
57]
58
59
60def setup_schemas(data_dir: Path) -> None:
61 write_schema(data_dir, "legacy_accounts", ACCOUNTS_FIELDS)
62 write_schema(data_dir, "legacy_widgets", WIDGETS_FIELDS)
63
64
65def base_widgets_config(input_name: str = "widgets.json") -> dict:
66 return {
67 "input": input_name,
68 "mapping": {
69 "WidgetId": "id",
70 "Title": "title",
71 "AccountId": "account_id",
72 "Priority": "priority",
73 "InternalCode": "extra",
74 },
75 "extra_fields": ["LegacyNote"],
76 "value_maps": [{"field": "priority", "map": {"med": "normal"}}],
77 "constants": {"source_system": "legacy_crm"},
78 }
79
80
81def base_accounts_config(input_name: str = "accounts.json") -> dict:
82 return {
83 "input": input_name,
84 "mapping": {"AccountId": "id", "AccountName": "name", "Tier": "tier"},
85 }
86
87
88def write_plan(tmp_path: Path, *, accounts_cfg: dict, widgets_cfg: dict | None = None) -> Path:
89 collections = {"legacy_accounts": accounts_cfg}
90 order = ["legacy_accounts"]
91 if widgets_cfg is not None:
92 collections["legacy_widgets"] = widgets_cfg
93 order.append("legacy_widgets")
94 return write_json(tmp_path / "plan.json", {"order": order, "collections": collections})
95
96
97# --------------------------------------------------------------------------
98# map_row
99# --------------------------------------------------------------------------
100
101def test_map_row_applies_rename_value_map_extra_and_constants():
102 cfg = object_import._normalize_collection_config(
103 "legacy_widgets", base_widgets_config(), base=Path(".")
104 )
105 source_row = {
106 "WidgetId": "w_1",
107 "Title": "Gadget",
108 "AccountId": "acc_1",
109 "Priority": "med",
110 "InternalCode": "X1",
111 "LegacyNote": "first",
112 }
113
114 dest, errors = object_import.map_row(source_row, cfg)
115
116 assert errors == []
117 assert dest["id"] == "w_1"
118 assert dest["title"] == "Gadget"
119 assert dest["account_id"] == "acc_1"
120 assert dest["priority"] == "normal" # value_map: med -> normal
121 assert dest["source_system"] == "legacy_crm" # constant stamp
122 assert dest["extra"] == {"InternalCode": "X1", "LegacyNote": "first"}
123
124
125def test_map_row_reports_unmapped_column_as_error():
126 cfg = object_import._normalize_collection_config(
127 "legacy_widgets",
128 {"input": "widgets.json", "mapping": {"WidgetId": "id"}},
129 base=Path("."),
130 )
131
132 dest, errors = object_import.map_row({"WidgetId": "w_1", "Mystery": "?"}, cfg)
133
134 assert any("Mystery" in err for err in errors)
135
136
137def test_map_row_reports_missing_id():
138 cfg = object_import._normalize_collection_config(
139 "legacy_widgets", {"input": "widgets.json", "mapping": {"Title": "title"}}, base=Path(".")
140 )
141
142 dest, errors = object_import.map_row({"Title": "Gadget"}, cfg)
143
144 assert any("id" in err for err in errors)
145
146
147# --------------------------------------------------------------------------
148# load_plan
149# --------------------------------------------------------------------------
150
151def test_load_plan_resolves_relative_input_paths_and_preserves_order(tmp_path):
152 plan_path = write_plan(
153 tmp_path, accounts_cfg=base_accounts_config(), widgets_cfg=base_widgets_config()
154 )
155
156 plan = object_import.load_plan(plan_path)
157
158 assert plan["order"] == ["legacy_accounts", "legacy_widgets"]
159 assert plan["collections"]["legacy_accounts"]["input"] == tmp_path / "accounts.json"
160 assert plan["collections"]["legacy_widgets"]["input"] == tmp_path / "widgets.json"
161
162
163def test_load_plan_rejects_order_naming_unconfigured_collection(tmp_path):
164 write_json(
165 tmp_path / "plan.json",
166 {"order": ["legacy_accounts", "ghost"], "collections": {"legacy_accounts": base_accounts_config()}},
167 )
168
169 with pytest.raises(object_import.ImportConfigError):
170 object_import.load_plan(tmp_path / "plan.json")
171
172
173# --------------------------------------------------------------------------
174# Dry run: report correctness, dependency order, relation-within-run
175# --------------------------------------------------------------------------
176
177def test_dry_run_reports_ok_enum_violation_and_missing_relation(tmp_path):
178 data_dir = tmp_path / "data"
179 setup_schemas(data_dir)
180
181 write_json(
182 tmp_path / "accounts.json",
183 [
184 {"AccountId": "acc_1", "AccountName": "Acme Corp", "Tier": "gold"},
185 {"AccountId": "acc_2", "AccountName": "Globex", "Tier": "silver"},
186 ],
187 )
188 write_json(
189 tmp_path / "widgets.json",
190 [
191 {"WidgetId": "w_1", "Title": "Gadget One", "AccountId": "acc_1",
192 "Priority": "med", "InternalCode": "X1", "LegacyNote": "n1"},
193 {"WidgetId": "w_2", "Title": "Gadget Two", "AccountId": "acc_2",
194 "Priority": "high", "InternalCode": "X2", "LegacyNote": "n2"},
195 {"WidgetId": "w_bad_enum", "Title": "Bad Priority", "AccountId": "acc_1",
196 "Priority": "urgent", "InternalCode": "X3", "LegacyNote": "n3"},
197 {"WidgetId": "w_missing_rel", "Title": "Orphan", "AccountId": "acc_missing",
198 "Priority": "low", "InternalCode": "X4", "LegacyNote": "n4"},
199 ],
200 )
201 plan_path = write_plan(
202 tmp_path, accounts_cfg=base_accounts_config(), widgets_cfg=base_widgets_config()
203 )
204 plan = object_import.load_plan(plan_path)
205
206 reports = object_import.run_import(plan, base_dir=data_dir, roots=[], dry_run=True)
207
208 accounts_report, widgets_report = reports
209 assert accounts_report.ok == 2
210 assert accounts_report.errors == 0
211
212 assert widgets_report.ok == 2
213 assert widgets_report.errors == 2
214 error_ids = {row.id for row in widgets_report.rows if row.status == "error"}
215 assert error_ids == {"w_bad_enum", "w_missing_rel"}
216 bad_enum_row = next(row for row in widgets_report.rows if row.id == "w_bad_enum")
217 assert any("priority" in reason for reason in bad_enum_row.reasons)
218 missing_rel_row = next(row for row in widgets_report.rows if row.id == "w_missing_rel")
219 assert any("missing record" in reason for reason in missing_rel_row.reasons)
220
221 # Dry run writes nothing.
222 assert object_records.read_collection_records("legacy_accounts", base_dir=data_dir, roots=[]) == []
223
224
225def test_dry_run_detects_duplicate_id_within_file(tmp_path):
226 data_dir = tmp_path / "data"
227 setup_schemas(data_dir)
228 write_json(
229 tmp_path / "accounts.json",
230 [
231 {"AccountId": "acc_1", "AccountName": "Acme Corp", "Tier": "gold"},
232 {"AccountId": "acc_1", "AccountName": "Duplicate", "Tier": "silver"},
233 ],
234 )
235 plan = object_import.load_plan(write_plan(tmp_path, accounts_cfg=base_accounts_config()))
236
237 reports = object_import.run_import(plan, base_dir=data_dir, roots=[], dry_run=True)
238
239 (report,) = reports
240 assert report.ok == 1
241 assert report.errors == 1
242 dup_row = report.rows[1]
243 assert dup_row.status == "error"
244 assert any("duplicate id" in reason for reason in dup_row.reasons)
245
246
247# --------------------------------------------------------------------------
248# Real run: id preservation, created_at preservation, actor attribution
249# --------------------------------------------------------------------------
250
251def test_real_run_creates_records_preserving_ids_and_attributing_actor(tmp_path):
252 data_dir = tmp_path / "data"
253 setup_schemas(data_dir)
254 write_json(
255 tmp_path / "accounts.json",
256 [{"AccountId": "acc_1", "AccountName": "Acme Corp", "Tier": "gold"}],
257 )
258 write_json(
259 tmp_path / "widgets.json",
260 [{"WidgetId": "w_1", "Title": "Gadget One", "AccountId": "acc_1",
261 "Priority": "high", "InternalCode": "X1", "LegacyNote": "n1"}],
262 )
263 plan = object_import.load_plan(
264 write_plan(tmp_path, accounts_cfg=base_accounts_config(), widgets_cfg=base_widgets_config())
265 )
266
267 reports = object_import.run_import(
268 plan, base_dir=data_dir, roots=[], dry_run=False, actor="import-cli-test"
269 )
270
271 accounts_report, widgets_report = reports
272 assert accounts_report.created == 1
273 assert widgets_report.created == 1
274
275 account = object_records.get_collection_record("legacy_accounts", "acc_1", base_dir=data_dir, roots=[])
276 assert account["id"] == "acc_1" # source id preserved exactly
277 assert account["name"] == "Acme Corp"
278
279 widget = object_records.get_collection_record("legacy_widgets", "w_1", base_dir=data_dir, roots=[])
280 assert widget["account_id"] == "acc_1"
281 assert widget["priority"] == "high"
282 assert widget["source_system"] == "legacy_crm"
283 extra = json.loads(widget["extra"])
284 assert extra == {"InternalCode": "X1", "LegacyNote": "n1"}
285
286 account_changes = object_record_changes.list_record_changes(
287 "legacy_accounts", record_id="acc_1", base_dir=data_dir
288 )["changes"]
289 assert account_changes[0]["actor"] == "import-cli-test"
290 assert account_changes[0]["action"] == "create"
291
292 widget_changes = object_record_changes.list_record_changes(
293 "legacy_widgets", record_id="w_1", base_dir=data_dir
294 )["changes"]
295 assert widget_changes[0]["actor"] == "import-cli-test"
296
297
298def test_real_run_requires_actor():
299 plan = {"order": ["legacy_accounts"], "collections": {"legacy_accounts": {}}}
300 with pytest.raises(object_import.ImportRunError):
301 object_import.run_import(plan, base_dir=Path("data"), dry_run=False, actor=None)
302
303
304def test_real_run_preserves_source_created_at(tmp_path):
305 data_dir = tmp_path / "data"
306 setup_schemas(data_dir)
307 write_json(
308 tmp_path / "accounts.json",
309 [{"AccountId": "acc_1", "AccountName": "Acme Corp", "Tier": "gold",
310 "CreatedAt": "2020-01-01T00:00:00Z"}],
311 )
312 cfg = base_accounts_config()
313 cfg["mapping"]["CreatedAt"] = "created_at"
314 plan = object_import.load_plan(write_plan(tmp_path, accounts_cfg=cfg))
315
316 object_import.run_import(plan, base_dir=data_dir, roots=[], dry_run=False, actor="import-cli-test")
317
318 account = object_records.get_collection_record("legacy_accounts", "acc_1", base_dir=data_dir, roots=[])
319 assert account["created_at"] == "2020-01-01T00:00:00Z"
320
321
322def test_create_collection_record_still_rejects_read_only_by_default(tmp_path):
323 """preserve_read_only defaults False: an ordinary write path is unaffected."""
324 data_dir = tmp_path / "data"
325 setup_schemas(data_dir)
326
327 with pytest.raises(object_records.InvalidRecordPayloadError):
328 object_records.create_collection_record(
329 "legacy_accounts",
330 {"id": "acc_1", "name": "Acme", "created_at": "2020-01-01T00:00:00Z"},
331 base_dir=data_dir,
332 roots=[],
333 )
334
335
336# --------------------------------------------------------------------------
337# Idempotency: exists-skip and --update-existing
338# --------------------------------------------------------------------------
339
340def test_second_run_skips_existing_ids_by_default(tmp_path):
341 data_dir = tmp_path / "data"
342 setup_schemas(data_dir)
343 write_json(
344 tmp_path / "accounts.json",
345 [{"AccountId": "acc_1", "AccountName": "Acme Corp", "Tier": "gold"}],
346 )
347 plan = object_import.load_plan(write_plan(tmp_path, accounts_cfg=base_accounts_config()))
348
349 object_import.run_import(plan, base_dir=data_dir, roots=[], dry_run=False, actor="first-run")
350 reports = object_import.run_import(plan, base_dir=data_dir, roots=[], dry_run=False, actor="second-run")
351
352 (report,) = reports
353 assert report.exists == 1
354 assert report.created == 0
355 assert report.skipped == 1
356
357 records = object_records.read_collection_records("legacy_accounts", base_dir=data_dir, roots=[])
358 assert len(records) == 1 # not duplicated
359
360 changes = object_record_changes.list_record_changes(
361 "legacy_accounts", record_id="acc_1", base_dir=data_dir
362 )["changes"]
363 assert len(changes) == 1 # the skip never touched the record
364 assert changes[0]["actor"] == "first-run"
365
366
367def test_update_existing_upserts_and_never_moves_created_at(tmp_path):
368 data_dir = tmp_path / "data"
369 setup_schemas(data_dir)
370 write_json(
371 tmp_path / "accounts.json",
372 [{"AccountId": "acc_1", "AccountName": "Acme Corp", "Tier": "gold",
373 "CreatedAt": "2020-01-01T00:00:00Z"}],
374 )
375 cfg = base_accounts_config()
376 cfg["mapping"]["CreatedAt"] = "created_at"
377 plan = object_import.load_plan(write_plan(tmp_path, accounts_cfg=cfg))
378 object_import.run_import(plan, base_dir=data_dir, roots=[], dry_run=False, actor="first-run")
379
380 # A repeat import with a renamed org and a (deliberately ignored on update) new created_at.
381 write_json(
382 tmp_path / "accounts.json",
383 [{"AccountId": "acc_1", "AccountName": "Acme Corporation", "Tier": "gold",
384 "CreatedAt": "2099-01-01T00:00:00Z"}],
385 )
386 reports = object_import.run_import(
387 plan, base_dir=data_dir, roots=[], dry_run=False, actor="second-run", update_existing=True
388 )
389
390 (report,) = reports
391 assert report.updated == 1
392 account = object_records.get_collection_record("legacy_accounts", "acc_1", base_dir=data_dir, roots=[])
393 assert account["name"] == "Acme Corporation"
394 assert account["created_at"] == "2020-01-01T00:00:00Z" # untouched by update
395
396
397# --------------------------------------------------------------------------
398# Export
399# --------------------------------------------------------------------------
400
401def test_export_collection_returns_records_as_the_read_api_does(tmp_path):
402 data_dir = tmp_path / "data"
403 setup_schemas(data_dir)
404 write_json(
405 tmp_path / "accounts.json",
406 [{"AccountId": "acc_1", "AccountName": "Acme Corp", "Tier": "gold"}],
407 )
408 plan = object_import.load_plan(write_plan(tmp_path, accounts_cfg=base_accounts_config()))
409 object_import.run_import(plan, base_dir=data_dir, roots=[], dry_run=False, actor="seed")
410
411 exported = object_import.export_collection("legacy_accounts", base_dir=data_dir, roots=[])
412 api_read = object_records.read_collection_records("legacy_accounts", base_dir=data_dir, roots=[])
413
414 assert exported == api_read
415 assert exported[0]["id"] == "acc_1"
416
417
418# --------------------------------------------------------------------------
419# CLI
420# --------------------------------------------------------------------------
421
422def test_cli_dry_run_reports_errors_and_exits_nonzero(tmp_path, capsys):
423 data_dir = tmp_path / "data"
424 setup_schemas(data_dir)
425 write_json(tmp_path / "accounts.json", [{"AccountId": "acc_1", "AccountName": "Acme", "Tier": "platinum"}])
426 plan_path = write_plan(tmp_path, accounts_cfg=base_accounts_config())
427
428 exit_code = object_import.main(
429 ["--data-dir", str(data_dir), "run", str(plan_path), "--dry-run", "--json"]
430 )
431
432 assert exit_code == 1
433 payload = json.loads(capsys.readouterr().out)
434 assert payload[0]["errors"] == 1
435 assert object_records.read_collection_records("legacy_accounts", base_dir=data_dir, roots=[]) == []
436
437
438def test_cli_real_run_requires_actor(tmp_path, capsys):
439 data_dir = tmp_path / "data"
440 setup_schemas(data_dir)
441 write_json(tmp_path / "accounts.json", [{"AccountId": "acc_1", "AccountName": "Acme", "Tier": "gold"}])
442 plan_path = write_plan(tmp_path, accounts_cfg=base_accounts_config())
443
444 exit_code = object_import.main(["--data-dir", str(data_dir), "run", str(plan_path)])
445
446 assert exit_code == 1
447 assert "actor" in capsys.readouterr().err
448 assert object_records.read_collection_records("legacy_accounts", base_dir=data_dir, roots=[]) == []
449
450
451def test_cli_run_and_export_round_trip(tmp_path, capsys):
452 data_dir = tmp_path / "data"
453 setup_schemas(data_dir)
454 write_json(tmp_path / "accounts.json", [{"AccountId": "acc_1", "AccountName": "Acme", "Tier": "gold"}])
455 plan_path = write_plan(tmp_path, accounts_cfg=base_accounts_config())
456
457 exit_code = object_import.main(
458 ["--data-dir", str(data_dir), "run", str(plan_path), "--actor", "cli-actor"]
459 )
460 assert exit_code == 0
461 capsys.readouterr()
462
463 output_path = tmp_path / "export.json"
464 exit_code = object_import.main(
465 ["--data-dir", str(data_dir), "export", "legacy_accounts", "--output", str(output_path)]
466 )
467 assert exit_code == 0
468 exported = json.loads(output_path.read_text())
469 assert exported[0]["id"] == "acc_1"
470 assert exported[0]["name"] == "Acme"
471
472
473# --------------------------------------------------------------------------
474# Schema delta (part B) and comments package (part C) sanity checks
475# --------------------------------------------------------------------------
476
477REPO_ROOT = Path(__file__).resolve().parents[1]
478
479
480def test_tasks_schema_gains_draft_status():
481 schema = json.loads((REPO_ROOT / "packages/app-tasks/schemas/tasks.json").read_text())
482 status_field = next(f for f in schema["fields"] if f["name"] == "status")
483 assert "draft" in status_field["enum"]
484 # draft moves are guarded to the task owner (see FEATURE 1: transition guards).
485 draft_targets = {entry["to"] for entry in status_field["transitions"]["draft"]}
486 assert draft_targets == {"open", "assigned", "cancelled"}
487 assert all(
488 entry.get("when") == {"owner_id": "$user_id"}
489 for entry in status_field["transitions"]["draft"]
490 )
491
492
493def test_tasks_schema_draft_transition_enforced(tmp_path):
494 data_dir = tmp_path / "data"
495 schema_path = data_dir / "schemas" / "tasks.json"
496 schema_path.parent.mkdir(parents=True, exist_ok=True)
497 schema_path.write_text((REPO_ROOT / "packages/app-tasks/schemas/tasks.json").read_text())
498
499 task = object_records.create_collection_record(
500 "tasks", {"id": "t1", "title": "Draft task", "status": "draft"}, base_dir=data_dir, roots=[]
501 )
502 assert task["status"] == "draft"
503
504 # draft -> assigned is allowed
505 updated = object_records.update_collection_record(
506 "tasks", "t1", {"status": "assigned"}, base_dir=data_dir, roots=[]
507 )
508 assert updated["status"] == "assigned"
509
510 # assigned -> draft is not in the transitions map
511 task2 = object_records.create_collection_record(
512 "tasks", {"id": "t2", "title": "Another", "status": "assigned"}, base_dir=data_dir, roots=[]
513 )
514 with pytest.raises(object_records.InvalidRecordPayloadError):
515 object_records.update_collection_record(
516 "tasks", "t2", {"status": "draft"}, base_dir=data_dir, roots=[]
517 )
518
519
520def test_comments_schema_and_permissions_shape():
521 """The early-landed comments package must match the full thread block's
522 schema exactly (collection thread_comments, parent_* pointer fields,
523 owner_id author, moderation status with transitions) so adopting the
524 block later never requires a data migration."""
525 schema = json.loads(
526 (REPO_ROOT / "packages/app-thread/schemas/thread_comments.json").read_text()
527 )
528 assert schema["name"] == "thread_comments"
529 assert schema["storage"] == "append"
530 names = {f["name"] for f in schema["fields"]}
531 assert {
532 "id", "parent_collection", "parent_id", "reply_to_id", "body",
533 "owner_id", "author_name", "status", "created_at", "edited_at",
534 } <= names
535 status = next(f for f in schema["fields"] if f["name"] == "status")
536 assert status["default"] == "published"
537 assert status["transitions"]["removed"] == []
538
539 manifest = json.loads((REPO_ROOT / "packages/app-thread/dbbasic-package.json").read_text())
540 assert manifest["id"] == "app-thread"
541 assert manifest["schemas"] == [
542 {"collection": "thread_comments", "path": "schemas/thread_comments.json"}
543 ]
544
545 rules = json.loads((REPO_ROOT / "packages/app-thread/permissions/rules.json").read_text())
546 owner_rule = rules["rules"][0]
547 assert owner_rule["collection"] == "thread_comments"
548 assert owner_rule["row_filter"] == {"owner_id": "$user_id"}