askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
scan_ingest.py · 4.9 KB · 126 lines
| 1 | """action_scan_ingest -- get the paper in, fast, and get out of the way. |
| 2 | |
| 3 | POST {content_base64 | text, filename?, content_type?, source?, |
| 4 | category_hint?, notes?} |
| 5 | |
| 6 | Ingest is deliberately dumb and deliberately quick. It stores the bytes, |
| 7 | records that a document arrived, and returns -- no OCR, no model call, no |
| 8 | waiting. The reason is behavioural rather than technical: capture has to |
| 9 | happen at the moment the paper exists, standing at a till with a phone, |
| 10 | and anything that makes that moment slower is a receipt that ends up in a |
| 11 | coat pocket instead. Reading it is the processor's problem, later. |
| 12 | |
| 13 | Deduplication is by content hash and lives in the hook, so the second |
| 14 | photo of the same crumpled receipt lands on the scan that already exists |
| 15 | rather than becoming a second expense. That is the normal case, not an |
| 16 | edge case. |
| 17 | |
| 18 | `text` is accepted alongside bytes for documents that arrive as text |
| 19 | already -- a forwarded email receipt, an MCP agent pasting what it read. |
| 20 | Same pipeline, no image, and the free extractor works on it directly. |
| 21 | """ |
| 22 | |
| 23 | import base64 |
| 24 | import hashlib |
| 25 | import os |
| 26 | |
| 27 | import object_ids |
| 28 | import object_records |
| 29 | import object_user_files |
| 30 | |
| 31 | ACTOR = "action_scan_ingest" |
| 32 | |
| 33 | MAX_BYTES = 50 * 1024 * 1024 # the predecessor's cap, and a sane one |
| 34 | |
| 35 | SOURCES = ("web", "phone", "email", "mcp", "api") |
| 36 | HINTS = ("receipt", "bill", "invoice", "statement", "document", "photo") |
| 37 | |
| 38 | |
| 39 | def _base_dir(): |
| 40 | return os.environ.get("DBBASIC_DATA_DIR", "data") |
| 41 | |
| 42 | |
| 43 | def _text(value): |
| 44 | return str(value if value is not None else "").strip() |
| 45 | |
| 46 | |
| 47 | def POST(request): |
| 48 | identity = request.get("_identity") or {} |
| 49 | owner = (_text(request.get("owner_id")) |
| 50 | or _text(identity.get("user_id")) |
| 51 | or _text(identity.get("id"))) |
| 52 | if not owner: |
| 53 | return {"status": 401, |
| 54 | "error": "Ingest needs an owner: a scan belongs to somebody."} |
| 55 | |
| 56 | raw = request.get("content_base64") |
| 57 | body = None |
| 58 | if raw: |
| 59 | try: |
| 60 | body = base64.b64decode(_text(raw), validate=True) |
| 61 | except Exception: |
| 62 | return {"status": 400, "error": "content_base64 is not valid base64."} |
| 63 | elif _text(request.get("text")): |
| 64 | body = _text(request.get("text")).encode("utf-8") |
| 65 | |
| 66 | if not body: |
| 67 | return {"status": 400, |
| 68 | "error": "Nothing to ingest: send content_base64 or text."} |
| 69 | if len(body) > MAX_BYTES: |
| 70 | return {"status": 413, |
| 71 | "error": f"Document exceeds the {MAX_BYTES // (1024 * 1024)}MB cap."} |
| 72 | |
| 73 | digest = hashlib.sha256(body).hexdigest() |
| 74 | source = _text(request.get("source")) or "web" |
| 75 | hint = _text(request.get("category_hint")) or "receipt" |
| 76 | if source not in SOURCES: |
| 77 | return {"status": 400, "error": f"source must be one of {', '.join(SOURCES)}"} |
| 78 | if hint not in HINTS: |
| 79 | return {"status": 400, "error": f"category_hint must be one of {', '.join(HINTS)}"} |
| 80 | |
| 81 | base = _base_dir() |
| 82 | # The hook is the authority on duplicates, but checking here too means |
| 83 | # the bytes of a re-snapped receipt are never written a second time. |
| 84 | try: |
| 85 | for row in object_records.read_collection_records("scans", base_dir=base): |
| 86 | if (row.get("content_sha256") == digest |
| 87 | and _text(row.get("owner_id")) == owner): |
| 88 | return {"ok": True, "scan_id": row["id"], "duplicate": True, |
| 89 | "status_of_scan": row.get("status"), |
| 90 | "note": "this document was already received"} |
| 91 | except Exception: |
| 92 | return {"status": 503, |
| 93 | "error": "Scan history unreadable; refusing to ingest against " |
| 94 | "an unknown history rather than risk a duplicate."} |
| 95 | |
| 96 | scan_id = object_ids.new_uuid4() |
| 97 | file_id = f"scan-{scan_id}" |
| 98 | try: |
| 99 | size = object_user_files.save_file(owner, file_id, body, base_dir=base) |
| 100 | except Exception as exc: |
| 101 | return {"status": 500, "error": f"Could not store the document: {str(exc)[:120]}"} |
| 102 | |
| 103 | is_text = _text(request.get("text")) and not raw |
| 104 | object_records.create_collection_record( |
| 105 | "scans", |
| 106 | { |
| 107 | "id": scan_id, |
| 108 | "filename": _text(request.get("filename")) or f"{hint}-{scan_id[:8]}", |
| 109 | "content_type": (_text(request.get("content_type")) |
| 110 | or ("text/plain" if is_text else "application/octet-stream")), |
| 111 | "size": str(size), |
| 112 | "content_sha256": digest, |
| 113 | "file_id": file_id, |
| 114 | "source": source, |
| 115 | "category_hint": hint, |
| 116 | "status": "pending", |
| 117 | # Text that arrived as text needs no OCR pass to have text. |
| 118 | "ocr_text": body.decode("utf-8", "replace") if is_text else "", |
| 119 | "notes": _text(request.get("notes")), |
| 120 | "owner_id": owner, |
| 121 | "entity_id": _text(request.get("entity_id")), |
| 122 | }, |
| 123 | base_dir=base, actor=ACTOR) |
| 124 | |
| 125 | return {"ok": True, "scan_id": scan_id, "size": size, "status_of_scan": "pending", |
| 126 | "note": "stored; reading happens on the next intake pass"} |