askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_intake.py · 9.7 KB · 267 lines
| 1 | """Reading a receipt: text in, a SUGGESTION out. |
| 2 | |
| 3 | The pure half of document intake, testable without an OCR engine, a |
| 4 | vision model, or a data directory. |
| 5 | |
| 6 | The load-bearing idea in this file is that everything it produces is a |
| 7 | suggestion and nothing it produces is a fact. A scan is evidence and is |
| 8 | never edited (docs/logic-decisions.md #8); the expense is the record, and |
| 9 | it is created by a human confirming, with their name on it. That single |
| 10 | boundary is what makes the whole feature safe to ship with a bad |
| 11 | extractor: a weak reading costs somebody thirty seconds of typing, while |
| 12 | a confident wrong reading that posted straight to the books would cost |
| 13 | them an audit. Every design choice below follows from refusing the second |
| 14 | failure. |
| 15 | |
| 16 | It also makes the free path real. `guess_from_text` is regex over |
| 17 | whatever text an OCR engine produced -- no model, no key, no per-page |
| 18 | cost -- and it degrades to "extracted nothing, here is your image, type |
| 19 | the total" rather than to wrong numbers. An AI extractor plugs into the |
| 20 | same shape and simply guesses better; `normalize_extraction` takes its |
| 21 | loose JSON and clamps it into the same suggestion the regex path |
| 22 | produces, so the confirm step cannot tell them apart and does not care. |
| 23 | """ |
| 24 | |
| 25 | from __future__ import annotations |
| 26 | |
| 27 | import json |
| 28 | import re |
| 29 | from datetime import date |
| 30 | from typing import Any |
| 31 | |
| 32 | import object_banking |
| 33 | |
| 34 | KINDS = ("receipt", "bill", "invoice", "statement", "other") |
| 35 | |
| 36 | # Ordered by how strongly the label implies "this is the number that was |
| 37 | # actually charged". A receipt often shows subtotal, tax and total within |
| 38 | # a few lines of each other, and picking the largest number on the page |
| 39 | # is how intake systems bill a client for the tip line. |
| 40 | _TOTAL_LABELS = ( |
| 41 | (r"grand\s*total", 100), |
| 42 | (r"total\s*due", 95), |
| 43 | (r"amount\s*due", 95), |
| 44 | (r"balance\s*due", 90), |
| 45 | (r"\btotal\b", 80), |
| 46 | (r"amount\s*paid", 75), |
| 47 | (r"\bpaid\b", 60), |
| 48 | ) |
| 49 | |
| 50 | _MONEY = r"[-(]?\s*[$€£]?\s*\d[\d,. ]*\d|\d" |
| 51 | |
| 52 | _DATE_PATTERNS = ( |
| 53 | # ISO first: unambiguous, so it never has to be guessed at. |
| 54 | (re.compile(r"\b(\d{4})-(\d{2})-(\d{2})\b"), ("y", "m", "d")), |
| 55 | (re.compile(r"\b(\d{1,2})/(\d{1,2})/(\d{4})\b"), ("m", "d", "y")), |
| 56 | (re.compile(r"\b(\d{1,2})/(\d{1,2})/(\d{2})\b"), ("m", "d", "yy")), |
| 57 | (re.compile(r"\b(\d{1,2})-(\d{1,2})-(\d{4})\b"), ("d", "m", "y")), |
| 58 | ) |
| 59 | |
| 60 | _NOISE = re.compile(r"^[\W_\d]+$") |
| 61 | |
| 62 | |
| 63 | def _text(value: Any) -> str: |
| 64 | return str(value if value is not None else "").strip() |
| 65 | |
| 66 | |
| 67 | def _clamp_confidence(value: Any) -> float: |
| 68 | try: |
| 69 | number = float(value) |
| 70 | except (TypeError, ValueError): |
| 71 | return 0.0 |
| 72 | if number > 1: # a model that answered in percent |
| 73 | number = number / 100 |
| 74 | return max(0.0, min(1.0, round(number, 3))) |
| 75 | |
| 76 | |
| 77 | def _iso_date(parts: tuple[str, ...], order: tuple[str, ...]) -> str: |
| 78 | values = dict(zip(order, parts)) |
| 79 | year = values.get("y") or values.get("yy") or "" |
| 80 | if "yy" in values: |
| 81 | year = f"20{values['yy']}" |
| 82 | try: |
| 83 | return date(int(year), int(values["m"]), int(values["d"])).isoformat() |
| 84 | except (ValueError, KeyError, TypeError): |
| 85 | return "" |
| 86 | |
| 87 | |
| 88 | def find_date(text: str) -> str: |
| 89 | """The first date the document states, in ISO form. |
| 90 | |
| 91 | Ambiguous formats are read US-first (M/D/Y) because that is what the |
| 92 | receipts this was built for print, and the confirm step shows the |
| 93 | parsed date next to the image precisely so a human catches the day a |
| 94 | machine cannot know. |
| 95 | """ |
| 96 | for pattern, order in _DATE_PATTERNS: |
| 97 | match = pattern.search(text or "") |
| 98 | if match: |
| 99 | parsed = _iso_date(match.groups(), order) |
| 100 | if parsed: |
| 101 | return parsed |
| 102 | return "" |
| 103 | |
| 104 | |
| 105 | def find_total_cents(text: str) -> int: |
| 106 | """The amount most likely to be what was charged. |
| 107 | |
| 108 | Scored by the LABEL beside a number rather than by size. "Take the |
| 109 | biggest figure on the receipt" is the shortcut that bills a client |
| 110 | for the suggested-tip line, and a wrong total is worse than no total |
| 111 | because a plausible one gets confirmed without looking. |
| 112 | """ |
| 113 | best_score = 0 |
| 114 | best_cents = 0 |
| 115 | for line in (text or "").splitlines(): |
| 116 | lowered = line.lower() |
| 117 | for pattern, score in _TOTAL_LABELS: |
| 118 | if not re.search(pattern, lowered): |
| 119 | continue |
| 120 | amounts = re.findall(_MONEY, line) |
| 121 | if not amounts: |
| 122 | continue |
| 123 | cents = abs(object_banking.parse_cents(amounts[-1])) |
| 124 | if cents <= 0: |
| 125 | continue |
| 126 | if score > best_score: |
| 127 | best_score, best_cents = score, cents |
| 128 | break |
| 129 | return best_cents |
| 130 | |
| 131 | |
| 132 | def find_vendor(text: str) -> str: |
| 133 | """Whoever the money went to, guessed from the top of the page. |
| 134 | |
| 135 | Receipts put the trading name first and the useful detail later, so |
| 136 | the first line that is not punctuation, a number or a bare date is a |
| 137 | better guess than anything cleverer -- and being wrong here is |
| 138 | harmless, because the vendor is a description a human is reading |
| 139 | anyway. |
| 140 | """ |
| 141 | for line in (text or "").splitlines(): |
| 142 | candidate = line.strip() |
| 143 | if len(candidate) < 3 or len(candidate) > 60: |
| 144 | continue |
| 145 | if _NOISE.match(candidate): |
| 146 | continue |
| 147 | if find_date(candidate): |
| 148 | continue |
| 149 | return candidate |
| 150 | return "" |
| 151 | |
| 152 | |
| 153 | def guess_from_text(text: str, *, kind_hint: str = "") -> dict: |
| 154 | """The free extractor: no model, no key, no per-page cost. |
| 155 | |
| 156 | Confidence here is deliberately modest and is computed from how much |
| 157 | was actually found, not asserted. It exists so a UI can sort "probably |
| 158 | fine" above "look at this one", never so anything can be posted |
| 159 | unattended. |
| 160 | """ |
| 161 | body = text or "" |
| 162 | total = find_total_cents(body) |
| 163 | when = find_date(body) |
| 164 | vendor = find_vendor(body) |
| 165 | found = sum(1 for value in (total, when, vendor) if value) |
| 166 | return { |
| 167 | "kind": kind_hint if kind_hint in KINDS else "receipt", |
| 168 | "vendor": vendor, |
| 169 | "date": when, |
| 170 | "total_cents": total, |
| 171 | "tax_cents": 0, |
| 172 | "currency": "", |
| 173 | "line_items": [], |
| 174 | "confidence": _clamp_confidence(found / 3 * 0.6), |
| 175 | "engine": "text_rules", |
| 176 | } |
| 177 | |
| 178 | |
| 179 | def _minor_units(payload: dict, minor_key: str, money_key: str) -> int: |
| 180 | """Read an amount from whichever key the extractor used. |
| 181 | |
| 182 | `minor_key` is already in minor units and is only coerced to an int; |
| 183 | `money_key` is what a human would type and goes through the money |
| 184 | parser. Never both, since re-parsing 899 as money yields 89900. |
| 185 | """ |
| 186 | raw = payload.get(minor_key) |
| 187 | if raw not in (None, ""): |
| 188 | try: |
| 189 | return abs(int(str(raw).strip())) |
| 190 | except (TypeError, ValueError): |
| 191 | pass |
| 192 | if payload.get(money_key) not in (None, ""): |
| 193 | return abs(object_banking.parse_cents(payload.get(money_key))) |
| 194 | return 0 |
| 195 | |
| 196 | |
| 197 | def normalize_extraction(payload: Any, *, engine: str = "") -> dict: |
| 198 | """Clamp a model's loose JSON into the same suggestion shape. |
| 199 | |
| 200 | Anything unparseable becomes an EMPTY suggestion rather than an |
| 201 | exception: an extractor having a bad day must leave a scan waiting for |
| 202 | a human, never break the pass that is reading a hundred other |
| 203 | documents. |
| 204 | """ |
| 205 | if isinstance(payload, str): |
| 206 | try: |
| 207 | payload = json.loads(payload) |
| 208 | except (TypeError, ValueError): |
| 209 | payload = None |
| 210 | if not isinstance(payload, dict): |
| 211 | return {"kind": "other", "vendor": "", "date": "", "total_cents": 0, |
| 212 | "tax_cents": 0, "currency": "", "line_items": [], |
| 213 | "confidence": 0.0, "engine": engine or "unknown", |
| 214 | "note": "extractor returned nothing usable"} |
| 215 | |
| 216 | kind = _text(payload.get("kind")).lower() |
| 217 | # A model may answer in minor units ("total_cents": 899) or in the |
| 218 | # money a human would type ("total": "$8.99"). Each is parsed by its |
| 219 | # own rule and never by both -- running a cents figure back through |
| 220 | # the money parser is a hundredfold overcharge that looks plausible. |
| 221 | total = _minor_units(payload, "total_cents", "total") |
| 222 | tax = _minor_units(payload, "tax_cents", "tax") |
| 223 | |
| 224 | items = payload.get("line_items") |
| 225 | clean_items = [] |
| 226 | if isinstance(items, list): |
| 227 | for item in items[:50]: |
| 228 | if not isinstance(item, dict): |
| 229 | continue |
| 230 | clean_items.append({ |
| 231 | "description": _text(item.get("description"))[:200], |
| 232 | "amount_cents": _minor_units(item, "amount_cents", "amount"), |
| 233 | }) |
| 234 | |
| 235 | return { |
| 236 | "kind": kind if kind in KINDS else "other", |
| 237 | "vendor": _text(payload.get("vendor"))[:120], |
| 238 | "date": _text(payload.get("date"))[:10], |
| 239 | "total_cents": total, |
| 240 | "tax_cents": tax, |
| 241 | "currency": _text(payload.get("currency"))[:8].upper(), |
| 242 | "line_items": clean_items, |
| 243 | "confidence": _clamp_confidence(payload.get("confidence")), |
| 244 | "engine": engine or _text(payload.get("engine")) or "ai_vision", |
| 245 | } |
| 246 | |
| 247 | |
| 248 | def expense_draft_from(scan: dict, extraction: dict) -> dict: |
| 249 | """The DRAFT an operator is about to confirm -- fields, not a write. |
| 250 | |
| 251 | Returns what the confirm step would create, so the same mapping can |
| 252 | be shown on screen before anyone commits to it. A draft with no |
| 253 | amount is still returned: the point of confirmation is that a person |
| 254 | can fill in what the machine could not read, with the image in front |
| 255 | of them. |
| 256 | """ |
| 257 | vendor = _text(extraction.get("vendor")) |
| 258 | when = _text(extraction.get("date")) or _text(scan.get("created_at"))[:10] |
| 259 | return { |
| 260 | "description": vendor or _text(scan.get("filename")) or "Scanned receipt", |
| 261 | "incurred_on": when[:10], |
| 262 | "amount_cents": int(extraction.get("total_cents") or 0), |
| 263 | "currency": _text(extraction.get("currency")) or "USD", |
| 264 | "receipt_ref": f"scans/{_text(scan.get('id'))}", |
| 265 | "status": "draft", |
| 266 | "notes": _text(extraction.get("note")), |
| 267 | } |