askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_import.py · 27.8 KB · 719 lines
| 1 | """Generic import/export for TSV-backed collections. |
| 2 | |
| 3 | Implements the CLI-scale slice of plan/vocabulary/13-import-export-spec.md |
| 4 | ("the import/export block"): point a JSON/CSV/TSV source at a collection, |
| 5 | map its columns, dry-run every row against the target schema before |
| 6 | anything is written, then write via the ordinary record API so schema |
| 7 | rules and universal attribution apply exactly as a hand-typed form would. |
| 8 | The HTTP wizard, saved `import_recipes`/`import_runs` collections, chunked |
| 9 | execute, and the classic-mode bulk-rewrite entry point that spec also |
| 10 | describes are out of scope here -- this is the stdlib CLI half. |
| 11 | |
| 12 | RUN CONFIG (a JSON file passed to ``run``): one file describes a whole |
| 13 | import run -- the collections it touches, in dependency order, and how |
| 14 | each collection's source rows map onto its schema. |
| 15 | |
| 16 | { |
| 17 | "order": ["legacy_crm_orgs", "legacy_crm_contacts"], |
| 18 | "collections": { |
| 19 | "legacy_crm_orgs": { |
| 20 | "input": "orgs.json", |
| 21 | "mapping": {"OrgId": "id", "OrgName": "name"}, |
| 22 | "constants": {"owner_id": "u_import"} |
| 23 | }, |
| 24 | "legacy_crm_contacts": { |
| 25 | "input": "contacts.csv", |
| 26 | "mapping": { |
| 27 | "ContactId": "id", |
| 28 | "FullName": "name", |
| 29 | "OrgId": "org_id", |
| 30 | "Urgency": "urgency", |
| 31 | "LegacyNotes": "extra" |
| 32 | }, |
| 33 | "extra_fields": ["InternalRefCode"], |
| 34 | "ignore_fields": ["RowNumber"], |
| 35 | "value_maps": [{"field": "urgency", "map": {"medium": "normal"}}], |
| 36 | "constants": {"owner_id": "u_import"} |
| 37 | } |
| 38 | } |
| 39 | } |
| 40 | |
| 41 | Per collection: |
| 42 | |
| 43 | - ``input`` -- path to the source file (relative paths resolve against the |
| 44 | run config's own directory). Format is inferred from the extension |
| 45 | (``.json``/``.csv``/``.tsv``); a JSON source is a list of flat objects. |
| 46 | - ``mapping`` -- ``{source_column: dest_field}``. Every source column must |
| 47 | be accounted for by ``mapping``, ``extra_fields``, or ``ignore_fields``; |
| 48 | an unmapped column is a dry-run error ("unknown fields not covered by |
| 49 | mapping" -- see the module-level note under Spec Conflicts below), not a |
| 50 | silent drop. Map a column to ``"id"`` to supply the record id (source |
| 51 | ids are kept exactly -- see ID Preservation below); map one to |
| 52 | ``"extra"`` to route it into the record's JSON overflow field under its |
| 53 | own source-column name. |
| 54 | - ``extra_fields`` -- source columns routed into ``extra`` the same way, |
| 55 | spelled as a plain list instead of repeating ``"extra"`` in ``mapping`` |
| 56 | for every one. |
| 57 | - ``ignore_fields`` -- source columns intentionally dropped (e.g. a |
| 58 | spreadsheet's own row-number column). |
| 59 | - ``value_maps`` -- ``[{"field": dest_field, "map": {source_value: |
| 60 | dest_value}}]``, applied to the mapped/renamed value. |
| 61 | - ``constants`` -- ``{dest_field: value}`` stamped onto every row, |
| 62 | overriding whatever the row itself supplied (e.g. attributing every |
| 63 | imported record to one operator-chosen owner). |
| 64 | |
| 65 | ID PRESERVATION: whatever a row resolves as ``id`` (via ``mapping`` or a |
| 66 | constant) is written exactly as given -- object_import never generates a |
| 67 | fresh id, because other collections in the same run (and existing data) |
| 68 | may already reference it. |
| 69 | |
| 70 | CREATED_AT PRESERVATION: a schema's ``created_at`` is ordinarily |
| 71 | ``read_only`` -- object_records.create_collection_record rejects a |
| 72 | client-submitted value for it, same as any other read-only field, so a |
| 73 | hand-typed form can never spoof "created two years ago". An import |
| 74 | replaying another system's history needs the opposite: the source row's |
| 75 | own timestamp, not "now". Real-run creates in this module always pass |
| 76 | ``create_collection_record(..., preserve_read_only=True)`` -- a new, |
| 77 | narrowly-scoped kwarg added to object_records.py alongside this module |
| 78 | (see its docstring) that allows a *plain* read-only field's submitted |
| 79 | value through while still rejecting a genuinely ``computed`` field. Map a |
| 80 | source column to ``created_at`` (or any other read-only-but-not-computed |
| 81 | field) in the mapping and its value is preserved verbatim; leave it |
| 82 | unmapped and the server stamps "now", exactly as an ordinary create would. |
| 83 | Updates (the ``--update-existing`` duplicate path) never touch read-only |
| 84 | fields, preserved or not -- an existing record's own history shouldn't |
| 85 | shift just because a repeat import touched other fields on it. |
| 86 | |
| 87 | SPEC CONFLICTS (13-import-export-spec.md vs. this module, per the task |
| 88 | that produced it -- the task wins, noted here for the record): |
| 89 | |
| 90 | - The spec's ``import_recipes.mapping`` auto-routes any source column |
| 91 | *absent* from the map into ``extra`` by default. This module instead |
| 92 | requires every source column to be explicitly accounted for (mapping / |
| 93 | extra_fields / ignore_fields) and reports an unmapped column as a |
| 94 | dry-run error, per this task's explicit "unknown fields not covered by |
| 95 | mapping" validation requirement. |
| 96 | - No ``import_recipes``/``import_runs`` collections, no chunked/resumable |
| 97 | execute, no classic-mode bulk-rewrite entry point, no delimiter/encoding |
| 98 | sniffing, no ``partial_import_policy`` toggle (a row that fails |
| 99 | validation is always skipped-and-reported; valid rows in the same file |
| 100 | are always still written -- there is no "block the whole run" mode |
| 101 | here). All namable "implementation-session decisions" the spec's Open |
| 102 | Questions section explicitly deferred; this module is the narrower CLI |
| 103 | slice the task asked for, not the full block. |
| 104 | """ |
| 105 | |
| 106 | from __future__ import annotations |
| 107 | |
| 108 | import argparse |
| 109 | import csv |
| 110 | import json |
| 111 | import os |
| 112 | import sys |
| 113 | from dataclasses import dataclass, field as dataclass_field |
| 114 | from pathlib import Path |
| 115 | from typing import Any, Iterable |
| 116 | |
| 117 | import object_collections |
| 118 | import object_record_changes |
| 119 | import object_records |
| 120 | import object_schemas |
| 121 | from object_versions import DEFAULT_DATA_DIR |
| 122 | |
| 123 | DATA_DIR_ENV = "DBBASIC_DATA_DIR" |
| 124 | EXTRA_ROUTE = "extra" |
| 125 | SOURCE_FORMATS = {"json", "csv", "tsv"} |
| 126 | |
| 127 | |
| 128 | class ImportConfigError(ValueError): |
| 129 | """Raised when a run config or a collection's mapping is malformed.""" |
| 130 | |
| 131 | |
| 132 | class ImportRunError(ValueError): |
| 133 | """Raised when a run itself can't proceed (e.g. no --actor for a real run).""" |
| 134 | |
| 135 | |
| 136 | # -------------------------------------------------------------------------- |
| 137 | # Run config |
| 138 | # -------------------------------------------------------------------------- |
| 139 | |
| 140 | def load_plan(config_path: Path) -> dict[str, Any]: |
| 141 | """Load and normalize a run config: ``{"order": [...], "collections": {...}}``.""" |
| 142 | config_path = Path(config_path) |
| 143 | try: |
| 144 | payload = json.loads(config_path.read_text(encoding="utf-8")) |
| 145 | except FileNotFoundError as exc: |
| 146 | raise ImportConfigError(f"Run config not found: {config_path}") from exc |
| 147 | except json.JSONDecodeError as exc: |
| 148 | raise ImportConfigError(f"Run config is not valid JSON: {config_path} ({exc})") from exc |
| 149 | |
| 150 | if not isinstance(payload, dict): |
| 151 | raise ImportConfigError("Run config must be a JSON object") |
| 152 | |
| 153 | order = payload.get("order") |
| 154 | if not isinstance(order, list) or not order or not all(isinstance(item, str) and item for item in order): |
| 155 | raise ImportConfigError("Run config 'order' must be a non-empty list of collection names") |
| 156 | |
| 157 | collections = payload.get("collections") |
| 158 | if not isinstance(collections, dict): |
| 159 | raise ImportConfigError("Run config 'collections' must be an object") |
| 160 | |
| 161 | missing = [name for name in order if name not in collections] |
| 162 | if missing: |
| 163 | raise ImportConfigError(f"Run config 'order' names collections with no config: {missing}") |
| 164 | |
| 165 | base = config_path.resolve().parent |
| 166 | normalized = { |
| 167 | name: _normalize_collection_config(name, collections[name], base=base) for name in order |
| 168 | } |
| 169 | return {"order": list(order), "collections": normalized} |
| 170 | |
| 171 | |
| 172 | def _normalize_collection_config(name: str, raw: Any, *, base: Path) -> dict[str, Any]: |
| 173 | if not isinstance(raw, dict): |
| 174 | raise ImportConfigError(f"Collection config for {name!r} must be an object") |
| 175 | |
| 176 | input_value = raw.get("input") |
| 177 | if not isinstance(input_value, str) or not input_value: |
| 178 | raise ImportConfigError(f"Collection config for {name!r} is missing 'input'") |
| 179 | input_path = Path(input_value) |
| 180 | if not input_path.is_absolute(): |
| 181 | input_path = base / input_path |
| 182 | |
| 183 | fmt = raw.get("format") |
| 184 | if fmt is not None and fmt not in SOURCE_FORMATS: |
| 185 | raise ImportConfigError(f"Collection config for {name!r} has an unknown 'format': {fmt!r}") |
| 186 | |
| 187 | mapping = raw.get("mapping") or {} |
| 188 | if not isinstance(mapping, dict) or not all(isinstance(v, str) for v in mapping.values()): |
| 189 | raise ImportConfigError(f"Collection config for {name!r}: 'mapping' must be an object of strings") |
| 190 | |
| 191 | extra_fields = raw.get("extra_fields") or [] |
| 192 | ignore_fields = raw.get("ignore_fields") or [] |
| 193 | value_maps = raw.get("value_maps") or [] |
| 194 | constants = raw.get("constants") or {} |
| 195 | if not isinstance(value_maps, list): |
| 196 | raise ImportConfigError(f"Collection config for {name!r}: 'value_maps' must be a list") |
| 197 | if not isinstance(constants, dict): |
| 198 | raise ImportConfigError(f"Collection config for {name!r}: 'constants' must be an object") |
| 199 | |
| 200 | return { |
| 201 | "collection": name, |
| 202 | "input": input_path, |
| 203 | "format": fmt, |
| 204 | "mapping": dict(mapping), |
| 205 | "extra_fields": list(extra_fields), |
| 206 | "ignore_fields": list(ignore_fields), |
| 207 | "value_maps": list(value_maps), |
| 208 | "constants": dict(constants), |
| 209 | } |
| 210 | |
| 211 | |
| 212 | # -------------------------------------------------------------------------- |
| 213 | # Source loading |
| 214 | # -------------------------------------------------------------------------- |
| 215 | |
| 216 | def load_source_rows(path: Path, fmt: str | None = None) -> list[dict[str, Any]]: |
| 217 | """Return a source file's rows as plain dicts. JSON: a list of objects. CSV/TSV: DictReader rows.""" |
| 218 | path = Path(path) |
| 219 | fmt = fmt or _infer_format(path) |
| 220 | if fmt == "json": |
| 221 | try: |
| 222 | payload = json.loads(path.read_text(encoding="utf-8")) |
| 223 | except FileNotFoundError as exc: |
| 224 | raise ImportConfigError(f"Import source not found: {path}") from exc |
| 225 | except json.JSONDecodeError as exc: |
| 226 | raise ImportConfigError(f"Import source is not valid JSON: {path} ({exc})") from exc |
| 227 | if not isinstance(payload, list): |
| 228 | raise ImportConfigError(f"JSON import source must be a list of records: {path}") |
| 229 | rows = [] |
| 230 | for index, row in enumerate(payload, start=1): |
| 231 | if not isinstance(row, dict): |
| 232 | raise ImportConfigError(f"JSON import source row {index} is not an object: {path}") |
| 233 | rows.append(row) |
| 234 | return rows |
| 235 | |
| 236 | delimiter = "\t" if fmt == "tsv" else "," |
| 237 | try: |
| 238 | with path.open(newline="", encoding="utf-8") as handle: |
| 239 | reader = csv.DictReader(handle, delimiter=delimiter) |
| 240 | return [dict(row) for row in reader] |
| 241 | except FileNotFoundError as exc: |
| 242 | raise ImportConfigError(f"Import source not found: {path}") from exc |
| 243 | |
| 244 | |
| 245 | def _infer_format(path: Path) -> str: |
| 246 | suffix = path.suffix.lower().lstrip(".") |
| 247 | if suffix not in SOURCE_FORMATS: |
| 248 | raise ImportConfigError( |
| 249 | f"Cannot infer source format from extension (expected .json/.csv/.tsv): {path}" |
| 250 | ) |
| 251 | return suffix |
| 252 | |
| 253 | |
| 254 | # -------------------------------------------------------------------------- |
| 255 | # Row mapping |
| 256 | # -------------------------------------------------------------------------- |
| 257 | |
| 258 | def map_row(source_row: dict[str, Any], cfg: dict[str, Any]) -> tuple[dict[str, Any], list[str]]: |
| 259 | """Apply one collection's rename/extra/value-map/constant rules to a source row. |
| 260 | |
| 261 | Returns ``(dest_record, errors)``. ``dest_record`` is the payload |
| 262 | object_records would receive; ``errors`` covers mapping-time problems |
| 263 | (an unmapped column, no resolvable id) -- schema-level checks |
| 264 | (required/enum/relation) happen later in ``validate_row``, once the |
| 265 | target schema is known. |
| 266 | """ |
| 267 | errors: list[str] = [] |
| 268 | mapping: dict[str, str] = cfg["mapping"] |
| 269 | extra_fields = set(cfg["extra_fields"]) |
| 270 | ignore_fields = set(cfg["ignore_fields"]) |
| 271 | |
| 272 | dest: dict[str, Any] = {} |
| 273 | extra_blob: dict[str, Any] = {} |
| 274 | |
| 275 | for source_col, value in source_row.items(): |
| 276 | if source_col in mapping: |
| 277 | dest_name = mapping[source_col] |
| 278 | if dest_name == EXTRA_ROUTE: |
| 279 | extra_blob[source_col] = value |
| 280 | else: |
| 281 | dest[dest_name] = value |
| 282 | elif source_col in extra_fields: |
| 283 | extra_blob[source_col] = value |
| 284 | elif source_col in ignore_fields: |
| 285 | continue |
| 286 | else: |
| 287 | errors.append( |
| 288 | f"unmapped source column {source_col!r} (add it to mapping, " |
| 289 | "extra_fields, or ignore_fields)" |
| 290 | ) |
| 291 | |
| 292 | for rule in cfg["value_maps"]: |
| 293 | if not isinstance(rule, dict): |
| 294 | continue |
| 295 | target_field = rule.get("field") |
| 296 | value_map = rule.get("map") or {} |
| 297 | if target_field in dest: |
| 298 | raw = dest[target_field] |
| 299 | raw_str = "" if raw is None else str(raw) |
| 300 | if raw_str in value_map: |
| 301 | dest[target_field] = value_map[raw_str] |
| 302 | |
| 303 | for const_name, const_value in cfg["constants"].items(): |
| 304 | dest[const_name] = const_value |
| 305 | |
| 306 | if dest.get("id") in (None, ""): |
| 307 | errors.append('row has no resolvable "id" (map a source column to "id" in the mapping)') |
| 308 | else: |
| 309 | dest["id"] = str(dest["id"]) |
| 310 | |
| 311 | if extra_blob: |
| 312 | dest["extra"] = extra_blob |
| 313 | |
| 314 | return dest, errors |
| 315 | |
| 316 | |
| 317 | # -------------------------------------------------------------------------- |
| 318 | # Row validation (dry-run and real-run share this) |
| 319 | # -------------------------------------------------------------------------- |
| 320 | |
| 321 | def _is_computed_field(field: dict[str, Any]) -> bool: |
| 322 | field_type = str(field.get("type", "")).lower() |
| 323 | return bool(field_type == "computed" or field.get("computed")) |
| 324 | |
| 325 | |
| 326 | def _is_empty(value: str) -> bool: |
| 327 | return value is None or value == "" |
| 328 | |
| 329 | |
| 330 | def validate_row( |
| 331 | collection: str, |
| 332 | dest: dict[str, Any], |
| 333 | *, |
| 334 | base_dir: Path, |
| 335 | roots: Iterable[Path] | None, |
| 336 | known_ids: dict[str, set[str]], |
| 337 | ) -> list[str]: |
| 338 | """Validate one mapped row against the target schema's field rules. |
| 339 | |
| 340 | Reuses object_records' own private per-field checks (type/enum/ |
| 341 | validation-rule/relation) directly, rather than re-deriving them -- |
| 342 | there is no public dry-run validator yet (13-import-export-spec.md's |
| 343 | own Open Questions section names this exact gap: "Dry-run validation |
| 344 | needs a new public entry point"). Reusing the private functions means |
| 345 | a validation error here is worded identically to what a real write |
| 346 | would raise, same as the spec wants; a future public |
| 347 | ``validate_record_payload`` can replace this call site without |
| 348 | changing behavior. |
| 349 | |
| 350 | Relation checks are widened for "target existence within the run": a |
| 351 | relation to a row from an earlier (or the same) collection in this run |
| 352 | that has already validated is accepted even though nothing has been |
| 353 | written yet -- ``known_ids`` accumulates as the run proceeds, seeded |
| 354 | per collection with whatever already exists on disk. |
| 355 | """ |
| 356 | errors: list[str] = [] |
| 357 | record_id = str(dest.get("id", "")) |
| 358 | if not object_records.validate_record_id(record_id): |
| 359 | errors.append(f"invalid record id: {record_id!r}") |
| 360 | |
| 361 | fields = object_records._schema_fields(collection, base_dir=base_dir, roots=roots) |
| 362 | submitted = frozenset(dest) |
| 363 | |
| 364 | for schema_field in fields: |
| 365 | name = schema_field["name"] |
| 366 | if name == "id": |
| 367 | continue |
| 368 | raw_value = dest.get(name, "") |
| 369 | value = "" if raw_value is None else str(raw_value) |
| 370 | read_only = object_records._is_computed_or_read_only(schema_field) |
| 371 | computed = _is_computed_field(schema_field) |
| 372 | |
| 373 | if name in submitted and read_only: |
| 374 | if computed: |
| 375 | errors.append(f"field '{name}' is computed and cannot be imported") |
| 376 | continue |
| 377 | # Plain read-only (not computed): allowed here. A real run |
| 378 | # carries it through via create_collection_record(..., |
| 379 | # preserve_read_only=True) -- see the module docstring. |
| 380 | |
| 381 | if object_records._field_is_required(schema_field) and not read_only and _is_empty(value): |
| 382 | errors.append(f"field '{name}' is required") |
| 383 | continue |
| 384 | if _is_empty(value): |
| 385 | continue |
| 386 | |
| 387 | try: |
| 388 | object_records._validate_field_type(schema_field, value) |
| 389 | object_records._validate_field_enum(schema_field, value) |
| 390 | object_records._validate_field_rules(schema_field, value) |
| 391 | except object_records.InvalidRecordPayloadError as exc: |
| 392 | errors.append(str(exc)) |
| 393 | continue |
| 394 | |
| 395 | relation_error = _check_relation(schema_field, value, known_ids=known_ids, base_dir=base_dir) |
| 396 | if relation_error: |
| 397 | errors.append(relation_error) |
| 398 | |
| 399 | return errors |
| 400 | |
| 401 | |
| 402 | def _check_relation( |
| 403 | schema_field: dict[str, Any], |
| 404 | value: str, |
| 405 | *, |
| 406 | known_ids: dict[str, set[str]], |
| 407 | base_dir: Path, |
| 408 | ) -> str | None: |
| 409 | relation = schema_field.get("relation") |
| 410 | if relation is None: |
| 411 | return None |
| 412 | target = relation.get("collection") if isinstance(relation, dict) else relation |
| 413 | if isinstance(target, str) and value in known_ids.get(target, ()): |
| 414 | return None |
| 415 | try: |
| 416 | object_records._validate_field_relation(schema_field, value, base_dir=base_dir) |
| 417 | except object_records.InvalidRecordPayloadError as exc: |
| 418 | return str(exc) |
| 419 | return None |
| 420 | |
| 421 | |
| 422 | # -------------------------------------------------------------------------- |
| 423 | # Reporting |
| 424 | # -------------------------------------------------------------------------- |
| 425 | |
| 426 | @dataclass |
| 427 | class RowOutcome: |
| 428 | row_number: int |
| 429 | id: str |
| 430 | status: str # "ok" | "exists" | "error" -- validation-time classification, never mutated |
| 431 | reasons: list[str] |
| 432 | action: str | None = None # "created" | "updated" | "skipped" -- set only on a real run |
| 433 | |
| 434 | |
| 435 | @dataclass |
| 436 | class CollectionReport: |
| 437 | collection: str |
| 438 | total: int = 0 |
| 439 | ok: int = 0 |
| 440 | exists: int = 0 |
| 441 | errors: int = 0 |
| 442 | created: int = 0 |
| 443 | updated: int = 0 |
| 444 | skipped: int = 0 |
| 445 | rows: list[RowOutcome] = dataclass_field(default_factory=list) |
| 446 | |
| 447 | def to_dict(self) -> dict[str, Any]: |
| 448 | return { |
| 449 | "collection": self.collection, |
| 450 | "total": self.total, |
| 451 | "ok": self.ok, |
| 452 | "exists": self.exists, |
| 453 | "errors": self.errors, |
| 454 | "created": self.created, |
| 455 | "updated": self.updated, |
| 456 | "skipped": self.skipped, |
| 457 | "rows": [ |
| 458 | { |
| 459 | "row": row.row_number, |
| 460 | "id": row.id, |
| 461 | "status": row.status, |
| 462 | "reasons": row.reasons, |
| 463 | "action": row.action, |
| 464 | } |
| 465 | for row in self.rows |
| 466 | ], |
| 467 | } |
| 468 | |
| 469 | |
| 470 | # -------------------------------------------------------------------------- |
| 471 | # Run orchestration |
| 472 | # -------------------------------------------------------------------------- |
| 473 | |
| 474 | def run_import( |
| 475 | plan: dict[str, Any], |
| 476 | *, |
| 477 | base_dir: Path, |
| 478 | roots: Iterable[Path] | None = None, |
| 479 | dry_run: bool, |
| 480 | actor: str | None = None, |
| 481 | update_existing: bool = False, |
| 482 | ) -> list[CollectionReport]: |
| 483 | """Validate (and, unless ``dry_run``, write) every collection in a run's order. |
| 484 | |
| 485 | Nothing is written for a collection until every row in it has been |
| 486 | mapped and validated; only rows without errors are ever passed to |
| 487 | create/update. Duplicate ids are checked twice, at different scopes: |
| 488 | within the source file itself (two rows claiming the same id is |
| 489 | always an error) and against the target collection's existing ids |
| 490 | (idempotency -- ``exists``, resolved to skip or update below). |
| 491 | """ |
| 492 | if not dry_run and not actor: |
| 493 | raise ImportRunError("actor is required for a real run (pass --actor, or use --dry-run)") |
| 494 | |
| 495 | known_ids: dict[str, set[str]] = {} |
| 496 | reports: list[CollectionReport] = [] |
| 497 | |
| 498 | for collection in plan["order"]: |
| 499 | cfg = plan["collections"][collection] |
| 500 | source_rows = load_source_rows(cfg["input"], cfg["format"]) |
| 501 | |
| 502 | existing_ids = _existing_ids(collection, base_dir=base_dir, roots=roots) |
| 503 | known_ids.setdefault(collection, set()).update(existing_ids) |
| 504 | |
| 505 | report = CollectionReport(collection=collection, total=len(source_rows)) |
| 506 | seen_in_file: set[str] = set() |
| 507 | valid_rows: dict[int, dict[str, Any]] = {} |
| 508 | |
| 509 | for row_number, source_row in enumerate(source_rows, start=1): |
| 510 | dest, reasons = map_row(source_row, cfg) |
| 511 | record_id = str(dest.get("id", "")) |
| 512 | |
| 513 | if reasons: |
| 514 | report.rows.append(RowOutcome(row_number, record_id, "error", reasons)) |
| 515 | report.errors += 1 |
| 516 | continue |
| 517 | |
| 518 | if record_id in seen_in_file: |
| 519 | report.rows.append( |
| 520 | RowOutcome( |
| 521 | row_number, record_id, "error", |
| 522 | [f"duplicate id within import file: {record_id}"], |
| 523 | ) |
| 524 | ) |
| 525 | report.errors += 1 |
| 526 | continue |
| 527 | |
| 528 | row_errors = validate_row( |
| 529 | collection, dest, base_dir=base_dir, roots=roots, known_ids=known_ids |
| 530 | ) |
| 531 | if row_errors: |
| 532 | report.rows.append(RowOutcome(row_number, record_id, "error", row_errors)) |
| 533 | report.errors += 1 |
| 534 | continue |
| 535 | |
| 536 | seen_in_file.add(record_id) |
| 537 | known_ids[collection].add(record_id) |
| 538 | status = "exists" if record_id in existing_ids else "ok" |
| 539 | report.rows.append(RowOutcome(row_number, record_id, status, [])) |
| 540 | if status == "exists": |
| 541 | report.exists += 1 |
| 542 | else: |
| 543 | report.ok += 1 |
| 544 | valid_rows[row_number] = dest |
| 545 | |
| 546 | if not dry_run: |
| 547 | _execute_collection( |
| 548 | collection, |
| 549 | report, |
| 550 | valid_rows, |
| 551 | existing_ids=existing_ids, |
| 552 | base_dir=base_dir, |
| 553 | roots=roots, |
| 554 | actor=actor, |
| 555 | update_existing=update_existing, |
| 556 | ) |
| 557 | |
| 558 | reports.append(report) |
| 559 | |
| 560 | return reports |
| 561 | |
| 562 | |
| 563 | def _existing_ids(collection: str, *, base_dir: Path, roots: Iterable[Path] | None) -> set[str]: |
| 564 | try: |
| 565 | rows = object_records.read_collection_records(collection, base_dir=base_dir, roots=roots) |
| 566 | except (object_collections.CollectionNotFoundError, object_collections.InvalidCollectionNameError): |
| 567 | return set() |
| 568 | return {row["id"] for row in rows} |
| 569 | |
| 570 | |
| 571 | def _execute_collection( |
| 572 | collection: str, |
| 573 | report: CollectionReport, |
| 574 | valid_rows: dict[int, dict[str, Any]], |
| 575 | *, |
| 576 | existing_ids: set[str], |
| 577 | base_dir: Path, |
| 578 | roots: Iterable[Path] | None, |
| 579 | actor: str, |
| 580 | update_existing: bool, |
| 581 | ) -> None: |
| 582 | for outcome in report.rows: |
| 583 | if outcome.status == "error": |
| 584 | continue |
| 585 | dest = valid_rows[outcome.row_number] |
| 586 | record_id = dest["id"] |
| 587 | |
| 588 | if outcome.status == "exists": |
| 589 | if update_existing: |
| 590 | changes = {k: v for k, v in dest.items() if k != "id"} |
| 591 | changes = _strip_read_only(collection, changes, base_dir=base_dir, roots=roots) |
| 592 | object_records.update_collection_record( |
| 593 | collection, record_id, changes, base_dir=base_dir, roots=roots, actor=actor |
| 594 | ) |
| 595 | outcome.action = "updated" |
| 596 | report.updated += 1 |
| 597 | else: |
| 598 | outcome.action = "skipped" |
| 599 | report.skipped += 1 |
| 600 | continue |
| 601 | |
| 602 | object_records.create_collection_record( |
| 603 | collection, dest, base_dir=base_dir, roots=roots, actor=actor, preserve_read_only=True |
| 604 | ) |
| 605 | outcome.action = "created" |
| 606 | report.created += 1 |
| 607 | |
| 608 | |
| 609 | def _strip_read_only( |
| 610 | collection: str, changes: dict[str, Any], *, base_dir: Path, roots: Iterable[Path] | None |
| 611 | ) -> dict[str, Any]: |
| 612 | fields = object_records._schema_fields(collection, base_dir=base_dir, roots=roots) |
| 613 | read_only_names = {f["name"] for f in fields if object_records._is_computed_or_read_only(f)} |
| 614 | return {name: value for name, value in changes.items() if name not in read_only_names} |
| 615 | |
| 616 | |
| 617 | # -------------------------------------------------------------------------- |
| 618 | # Export |
| 619 | # -------------------------------------------------------------------------- |
| 620 | |
| 621 | def export_collection( |
| 622 | collection: str, *, base_dir: Path, roots: Iterable[Path] | None = None |
| 623 | ) -> list[dict[str, str]]: |
| 624 | """Return every record in a collection exactly as the read API returns it.""" |
| 625 | return object_records.read_collection_records(collection, base_dir=base_dir, roots=roots) |
| 626 | |
| 627 | |
| 628 | # -------------------------------------------------------------------------- |
| 629 | # CLI |
| 630 | # -------------------------------------------------------------------------- |
| 631 | |
| 632 | def main(argv: list[str] | None = None) -> int: |
| 633 | parser = argparse.ArgumentParser( |
| 634 | description="Import records into, or export them out of, TSV-backed collections." |
| 635 | ) |
| 636 | parser.add_argument( |
| 637 | "--data-dir", |
| 638 | type=Path, |
| 639 | default=None, |
| 640 | help=f"runtime data directory (default: ${DATA_DIR_ENV} or ./data)", |
| 641 | ) |
| 642 | subcommands = parser.add_subparsers(dest="command", required=True) |
| 643 | |
| 644 | run = subcommands.add_parser("run", help="import records per a run config") |
| 645 | run.add_argument("config", type=Path, help="path to the run config JSON file") |
| 646 | run.add_argument("--dry-run", action="store_true", help="validate only; write nothing") |
| 647 | run.add_argument("--actor", help="required for a real run; attributed on every write") |
| 648 | run.add_argument( |
| 649 | "--update-existing", |
| 650 | action="store_true", |
| 651 | help="upsert rows whose id already exists in the target collection (default: skip them)", |
| 652 | ) |
| 653 | run.add_argument("--json", action="store_true", help="print the report as JSON") |
| 654 | |
| 655 | export = subcommands.add_parser("export", help="dump one collection to JSON") |
| 656 | export.add_argument("collection") |
| 657 | export.add_argument( |
| 658 | "--output", type=Path, default=None, help="write to this file instead of stdout" |
| 659 | ) |
| 660 | |
| 661 | args = parser.parse_args(argv) |
| 662 | base_dir = _base_dir(args.data_dir) |
| 663 | |
| 664 | try: |
| 665 | if args.command == "run": |
| 666 | plan = load_plan(args.config) |
| 667 | reports = run_import( |
| 668 | plan, |
| 669 | base_dir=base_dir, |
| 670 | dry_run=args.dry_run, |
| 671 | actor=args.actor, |
| 672 | update_existing=args.update_existing, |
| 673 | ) |
| 674 | _print_run_report(reports, json_output=args.json, dry_run=args.dry_run) |
| 675 | return 1 if any(report.errors for report in reports) else 0 |
| 676 | |
| 677 | if args.command == "export": |
| 678 | records = export_collection(args.collection, base_dir=base_dir) |
| 679 | payload = json.dumps(records, indent=2) |
| 680 | if args.output: |
| 681 | args.output.write_text(payload + "\n", encoding="utf-8") |
| 682 | print(f"Exported {len(records)} record(s) from {args.collection} to {args.output}") |
| 683 | else: |
| 684 | print(payload) |
| 685 | return 0 |
| 686 | except (ImportConfigError, ImportRunError) as exc: |
| 687 | print(f"Error: {exc}", file=sys.stderr) |
| 688 | return 1 |
| 689 | except (object_collections.CollectionNotFoundError, object_collections.InvalidCollectionNameError) as exc: |
| 690 | print(f"Error: {exc}", file=sys.stderr) |
| 691 | return 1 |
| 692 | |
| 693 | parser.error(f"Unknown command: {args.command}") |
| 694 | return 2 |
| 695 | |
| 696 | |
| 697 | def _base_dir(data_dir: Path | None) -> Path: |
| 698 | if data_dir is not None: |
| 699 | return data_dir |
| 700 | return Path(os.environ.get(DATA_DIR_ENV, "data")) |
| 701 | |
| 702 | |
| 703 | def _print_run_report(reports: list[CollectionReport], *, json_output: bool, dry_run: bool) -> None: |
| 704 | if json_output: |
| 705 | print(json.dumps([report.to_dict() for report in reports], indent=2)) |
| 706 | return |
| 707 | |
| 708 | for report in reports: |
| 709 | summary = f"{report.collection}: {report.ok} ok, {report.exists} exists, {report.errors} errors" |
| 710 | if not dry_run: |
| 711 | summary += f" -- {report.created} created, {report.updated} updated, {report.skipped} skipped" |
| 712 | print(summary) |
| 713 | for row in report.rows: |
| 714 | if row.status == "error": |
| 715 | print(f" row {row.row_number} (id={row.id or '?'}): " + "; ".join(row.reasons)) |
| 716 | |
| 717 | |
| 718 | if __name__ == "__main__": |
| 719 | raise SystemExit(main()) |