Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
scan_processor.py · 8.1 KB · 200 lines
1"""system_scan_processor -- read what arrived, guess, and stop there.
2
3POST {limit?, engine?, today?} -- the scheduled pass that turns pending
4scans into suggestions. It never posts anything; producing a confirmable
5draft is the whole of its job (docs/logic-decisions.md #6 -- extraction
6is a suggestion, the write that matters is human-confirmed).
7
8Engine selection is configuration, not code (app_settings
9intake.ocr_engine):
10
11 text_rules the default and the FREE path -- regex over whatever text is
12 already on the scan. No model, no key, no per-page cost, and
13 it degrades to "found nothing, here is your image" rather
14 than to wrong numbers.
15 tesseract local OCR when pytesseract and PIL are installed on the host.
16 Absent, the pass says so and leaves the scan pending rather
17 than failing it: a missing system package is an operator's
18 problem to fix, not a document to give up on.
19 ai_vision object_ai with the owner's own key. Better guesses, and the
20 only path with a real per-page cost -- which is why it is
21 opt-in and why the key is theirs.
22
23Whatever the engine, the output lands in the same shape
24(object_intake.normalize_extraction), so the confirm step cannot tell
25which one read the document and does not care. That is what lets the
26cheap path be the default without the expensive path being a rewrite.
27
28Failure posture: attempts are counted and a document that keeps failing
29stops being retried and starts being VISIBLE. Churning a paid extractor
30forever against a corrupt PDF is how an intake queue silently spends
31money.
32"""
33
34import json
35import os
36from datetime import date
37
38import object_intake
39import object_records
40import object_user_files
41
42ACTOR = "system_scan_processor"
43
44DEFAULT_LIMIT = 25
45MAX_ATTEMPTS = 3
46
47
48def _base_dir():
49 return os.environ.get("DBBASIC_DATA_DIR", "data")
50
51
52def _setting(base, key, default=""):
53 try:
54 for row in object_records.read_collection_records("app_settings", base_dir=base):
55 if row.get("key") == key and str(row.get("value") or "").strip():
56 return row["value"].strip()
57 except Exception:
58 pass
59 return default
60
61
62def _int(value, default=0):
63 try:
64 return int(str(value or "").strip() or default)
65 except (TypeError, ValueError):
66 return default
67
68
69def _text(value):
70 return str(value if value is not None else "").strip()
71
72
73def _read_text(base, scan, engine):
74 """Whatever text we can get, and how we got it.
75
76 Returns (text, engine_used, error, recoverable). `recoverable` is
77 carried explicitly rather than sniffed out of the error message
78 later, because the two failures need opposite treatment and telling
79 them apart by substring is the kind of thing that silently stops
80 working when somebody rewords a message:
81
82 recoverable -- the HOST is not set up (a package missing, an engine
83 not built in). The document is fine; leave it pending so fixing
84 the host drains the backlog.
85 not recoverable -- THIS document could not be read. Count the
86 attempt against it, so a corrupt file surfaces instead of
87 churning an extractor forever.
88
89 Text already on the scan wins outright: a forwarded email receipt
90 needs no OCR, and re-reading an image we already read is pure cost.
91 """
92 existing = _text(scan.get("ocr_text"))
93 if existing:
94 return existing, _text(scan.get("ocr_engine")) or "supplied", "", False
95
96 if engine == "tesseract":
97 try:
98 import pytesseract # noqa: F401
99 from PIL import Image # noqa: F401
100 except ImportError:
101 return "", "", ("tesseract engine selected but pytesseract/PIL are "
102 "not installed on this host"), True
103 try:
104 import io
105
106 content = object_user_files.read_file(
107 _text(scan.get("owner_id")), _text(scan.get("file_id")), base_dir=base)
108 image = Image.open(io.BytesIO(content))
109 return pytesseract.image_to_string(image), "tesseract", "", False
110 except Exception as exc:
111 return "", "", f"tesseract failed: {str(exc)[:120]}", False
112
113 if engine == "ai_vision":
114 try:
115 import object_ai
116 except ImportError:
117 return "", "", "ai_vision engine selected but object_ai is unavailable", True
118 reader = getattr(object_ai, "read_document", None)
119 if reader is None:
120 return "", "", ("ai_vision engine selected but object_ai has no "
121 "document reader on this build"), True
122 try:
123 content = object_user_files.read_file(
124 _text(scan.get("owner_id")), _text(scan.get("file_id")), base_dir=base)
125 except Exception as exc:
126 return "", "", f"could not read the stored document: {str(exc)[:120]}", False
127 try:
128 return (_text(reader(content, kind=_text(scan.get("category_hint")))),
129 "ai_vision", "", False)
130 except Exception as exc:
131 return "", "", f"ai_vision failed: {str(exc)[:120]}", False
132
133 # text_rules and anything unrecognised: we read what is there, which
134 # for an image with no OCR engine is nothing -- and nothing is an
135 # honest answer that still leaves a confirmable draft.
136 return "", "text_rules", "", False
137
138
139def POST(request):
140 base = _base_dir()
141 limit = _int(request.get("limit"), DEFAULT_LIMIT) or DEFAULT_LIMIT
142 engine = (_text(request.get("engine"))
143 or _setting(base, "intake.ocr_engine", "text_rules"))
144
145 try:
146 scans = object_records.read_collection_records("scans", base_dir=base)
147 except Exception:
148 return {"ok": True, "skipped": "intake not installed (scans absent)"}
149
150 pending = [row for row in scans
151 if _text(row.get("status")) in ("pending", "processing")
152 and _int(row.get("attempts")) < MAX_ATTEMPTS]
153 pending.sort(key=lambda row: _text(row.get("created_at")))
154
155 read = extracted = failed = 0
156 results = []
157 for scan in pending[:limit]:
158 attempts = _int(scan.get("attempts")) + 1
159 text, engine_used, error, recoverable = _read_text(base, scan, engine)
160 if error:
161 # A host misconfiguration leaves the document PENDING (fix the
162 # host, run again); a genuine read failure counts against it.
163 object_records.update_collection_record(
164 "scans", scan["id"],
165 {"status": "pending" if recoverable else "error",
166 "attempts": str(attempts), "error": error},
167 base_dir=base, actor=ACTOR)
168 failed += 1
169 results.append({"scan": scan["id"], "error": error})
170 continue
171
172 suggestion = object_intake.guess_from_text(
173 text, kind_hint=_text(scan.get("category_hint")))
174 if engine_used and engine_used not in ("text_rules", "supplied"):
175 suggestion["engine"] = engine_used
176 object_records.update_collection_record(
177 "scans", scan["id"],
178 {
179 "status": "extracted",
180 "ocr_text": text,
181 "ocr_engine": engine_used or "text_rules",
182 "extracted": json.dumps(suggestion),
183 "confidence": str(suggestion["confidence"]),
184 "attempts": str(attempts),
185 "error": "",
186 },
187 base_dir=base, actor=ACTOR)
188 read += 1 if text else 0
189 extracted += 1
190 results.append({"scan": scan["id"], "engine": engine_used,
191 "total_cents": suggestion["total_cents"],
192 "confidence": suggestion["confidence"]})
193
194 stuck = [row["id"] for row in scans
195 if _int(row.get("attempts")) >= MAX_ATTEMPTS
196 and _text(row.get("status")) == "error"]
197 return {"ok": True, "today": _text(request.get("today")) or date.today().isoformat(),
198 "engine": engine, "considered": len(pending), "extracted": extracted,
199 "text_read": read, "failed": failed,
200 "needing_a_human": stuck, "results": results}