Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_intake.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_intake.py · 9.7 KB · 267 lines
1"""Reading a receipt: text in, a SUGGESTION out.
2
3The pure half of document intake, testable without an OCR engine, a
4vision model, or a data directory.
5
6The load-bearing idea in this file is that everything it produces is a
7suggestion and nothing it produces is a fact. A scan is evidence and is
8never edited (docs/logic-decisions.md #8); the expense is the record, and
9it is created by a human confirming, with their name on it. That single
10boundary is what makes the whole feature safe to ship with a bad
11extractor: a weak reading costs somebody thirty seconds of typing, while
12a confident wrong reading that posted straight to the books would cost
13them an audit. Every design choice below follows from refusing the second
14failure.
15
16It also makes the free path real. `guess_from_text` is regex over
17whatever text an OCR engine produced -- no model, no key, no per-page
18cost -- and it degrades to "extracted nothing, here is your image, type
19the total" rather than to wrong numbers. An AI extractor plugs into the
20same shape and simply guesses better; `normalize_extraction` takes its
21loose JSON and clamps it into the same suggestion the regex path
22produces, so the confirm step cannot tell them apart and does not care.
23"""
24
25from __future__ import annotations
26
27import json
28import re
29from datetime import date
30from typing import Any
31
32import object_banking
33
34KINDS = ("receipt", "bill", "invoice", "statement", "other")
35
36# Ordered by how strongly the label implies "this is the number that was
37# actually charged". A receipt often shows subtotal, tax and total within
38# a few lines of each other, and picking the largest number on the page
39# is how intake systems bill a client for the tip line.
40_TOTAL_LABELS = (
41 (r"grand\s*total", 100),
42 (r"total\s*due", 95),
43 (r"amount\s*due", 95),
44 (r"balance\s*due", 90),
45 (r"\btotal\b", 80),
46 (r"amount\s*paid", 75),
47 (r"\bpaid\b", 60),
48)
49
50_MONEY = r"[-(]?\s*[$€£]?\s*\d[\d,. ]*\d|\d"
51
52_DATE_PATTERNS = (
53 # ISO first: unambiguous, so it never has to be guessed at.
54 (re.compile(r"\b(\d{4})-(\d{2})-(\d{2})\b"), ("y", "m", "d")),
55 (re.compile(r"\b(\d{1,2})/(\d{1,2})/(\d{4})\b"), ("m", "d", "y")),
56 (re.compile(r"\b(\d{1,2})/(\d{1,2})/(\d{2})\b"), ("m", "d", "yy")),
57 (re.compile(r"\b(\d{1,2})-(\d{1,2})-(\d{4})\b"), ("d", "m", "y")),
58)
59
60_NOISE = re.compile(r"^[\W_\d]+$")
61
62
63def _text(value: Any) -> str:
64 return str(value if value is not None else "").strip()
65
66
67def _clamp_confidence(value: Any) -> float:
68 try:
69 number = float(value)
70 except (TypeError, ValueError):
71 return 0.0
72 if number > 1: # a model that answered in percent
73 number = number / 100
74 return max(0.0, min(1.0, round(number, 3)))
75
76
77def _iso_date(parts: tuple[str, ...], order: tuple[str, ...]) -> str:
78 values = dict(zip(order, parts))
79 year = values.get("y") or values.get("yy") or ""
80 if "yy" in values:
81 year = f"20{values['yy']}"
82 try:
83 return date(int(year), int(values["m"]), int(values["d"])).isoformat()
84 except (ValueError, KeyError, TypeError):
85 return ""
86
87
88def find_date(text: str) -> str:
89 """The first date the document states, in ISO form.
90
91 Ambiguous formats are read US-first (M/D/Y) because that is what the
92 receipts this was built for print, and the confirm step shows the
93 parsed date next to the image precisely so a human catches the day a
94 machine cannot know.
95 """
96 for pattern, order in _DATE_PATTERNS:
97 match = pattern.search(text or "")
98 if match:
99 parsed = _iso_date(match.groups(), order)
100 if parsed:
101 return parsed
102 return ""
103
104
105def find_total_cents(text: str) -> int:
106 """The amount most likely to be what was charged.
107
108 Scored by the LABEL beside a number rather than by size. "Take the
109 biggest figure on the receipt" is the shortcut that bills a client
110 for the suggested-tip line, and a wrong total is worse than no total
111 because a plausible one gets confirmed without looking.
112 """
113 best_score = 0
114 best_cents = 0
115 for line in (text or "").splitlines():
116 lowered = line.lower()
117 for pattern, score in _TOTAL_LABELS:
118 if not re.search(pattern, lowered):
119 continue
120 amounts = re.findall(_MONEY, line)
121 if not amounts:
122 continue
123 cents = abs(object_banking.parse_cents(amounts[-1]))
124 if cents <= 0:
125 continue
126 if score > best_score:
127 best_score, best_cents = score, cents
128 break
129 return best_cents
130
131
132def find_vendor(text: str) -> str:
133 """Whoever the money went to, guessed from the top of the page.
134
135 Receipts put the trading name first and the useful detail later, so
136 the first line that is not punctuation, a number or a bare date is a
137 better guess than anything cleverer -- and being wrong here is
138 harmless, because the vendor is a description a human is reading
139 anyway.
140 """
141 for line in (text or "").splitlines():
142 candidate = line.strip()
143 if len(candidate) < 3 or len(candidate) > 60:
144 continue
145 if _NOISE.match(candidate):
146 continue
147 if find_date(candidate):
148 continue
149 return candidate
150 return ""
151
152
153def guess_from_text(text: str, *, kind_hint: str = "") -> dict:
154 """The free extractor: no model, no key, no per-page cost.
155
156 Confidence here is deliberately modest and is computed from how much
157 was actually found, not asserted. It exists so a UI can sort "probably
158 fine" above "look at this one", never so anything can be posted
159 unattended.
160 """
161 body = text or ""
162 total = find_total_cents(body)
163 when = find_date(body)
164 vendor = find_vendor(body)
165 found = sum(1 for value in (total, when, vendor) if value)
166 return {
167 "kind": kind_hint if kind_hint in KINDS else "receipt",
168 "vendor": vendor,
169 "date": when,
170 "total_cents": total,
171 "tax_cents": 0,
172 "currency": "",
173 "line_items": [],
174 "confidence": _clamp_confidence(found / 3 * 0.6),
175 "engine": "text_rules",
176 }
177
178
179def _minor_units(payload: dict, minor_key: str, money_key: str) -> int:
180 """Read an amount from whichever key the extractor used.
181
182 `minor_key` is already in minor units and is only coerced to an int;
183 `money_key` is what a human would type and goes through the money
184 parser. Never both, since re-parsing 899 as money yields 89900.
185 """
186 raw = payload.get(minor_key)
187 if raw not in (None, ""):
188 try:
189 return abs(int(str(raw).strip()))
190 except (TypeError, ValueError):
191 pass
192 if payload.get(money_key) not in (None, ""):
193 return abs(object_banking.parse_cents(payload.get(money_key)))
194 return 0
195
196
197def normalize_extraction(payload: Any, *, engine: str = "") -> dict:
198 """Clamp a model's loose JSON into the same suggestion shape.
199
200 Anything unparseable becomes an EMPTY suggestion rather than an
201 exception: an extractor having a bad day must leave a scan waiting for
202 a human, never break the pass that is reading a hundred other
203 documents.
204 """
205 if isinstance(payload, str):
206 try:
207 payload = json.loads(payload)
208 except (TypeError, ValueError):
209 payload = None
210 if not isinstance(payload, dict):
211 return {"kind": "other", "vendor": "", "date": "", "total_cents": 0,
212 "tax_cents": 0, "currency": "", "line_items": [],
213 "confidence": 0.0, "engine": engine or "unknown",
214 "note": "extractor returned nothing usable"}
215
216 kind = _text(payload.get("kind")).lower()
217 # A model may answer in minor units ("total_cents": 899) or in the
218 # money a human would type ("total": "$8.99"). Each is parsed by its
219 # own rule and never by both -- running a cents figure back through
220 # the money parser is a hundredfold overcharge that looks plausible.
221 total = _minor_units(payload, "total_cents", "total")
222 tax = _minor_units(payload, "tax_cents", "tax")
223
224 items = payload.get("line_items")
225 clean_items = []
226 if isinstance(items, list):
227 for item in items[:50]:
228 if not isinstance(item, dict):
229 continue
230 clean_items.append({
231 "description": _text(item.get("description"))[:200],
232 "amount_cents": _minor_units(item, "amount_cents", "amount"),
233 })
234
235 return {
236 "kind": kind if kind in KINDS else "other",
237 "vendor": _text(payload.get("vendor"))[:120],
238 "date": _text(payload.get("date"))[:10],
239 "total_cents": total,
240 "tax_cents": tax,
241 "currency": _text(payload.get("currency"))[:8].upper(),
242 "line_items": clean_items,
243 "confidence": _clamp_confidence(payload.get("confidence")),
244 "engine": engine or _text(payload.get("engine")) or "ai_vision",
245 }
246
247
248def expense_draft_from(scan: dict, extraction: dict) -> dict:
249 """The DRAFT an operator is about to confirm -- fields, not a write.
250
251 Returns what the confirm step would create, so the same mapping can
252 be shown on screen before anyone commits to it. A draft with no
253 amount is still returned: the point of confirmation is that a person
254 can fill in what the machine could not read, with the image in front
255 of them.
256 """
257 vendor = _text(extraction.get("vendor"))
258 when = _text(extraction.get("date")) or _text(scan.get("created_at"))[:10]
259 return {
260 "description": vendor or _text(scan.get("filename")) or "Scanned receipt",
261 "incurred_on": when[:10],
262 "amount_cents": int(extraction.get("total_cents") or 0),
263 "currency": _text(extraction.get("currency")) or "USD",
264 "receipt_ref": f"scans/{_text(scan.get('id'))}",
265 "status": "draft",
266 "notes": _text(extraction.get("note")),
267 }