askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
scan_processor.py · 8.1 KB · 200 lines
| 1 | """system_scan_processor -- read what arrived, guess, and stop there. |
| 2 | |
| 3 | POST {limit?, engine?, today?} -- the scheduled pass that turns pending |
| 4 | scans into suggestions. It never posts anything; producing a confirmable |
| 5 | draft is the whole of its job (docs/logic-decisions.md #6 -- extraction |
| 6 | is a suggestion, the write that matters is human-confirmed). |
| 7 | |
| 8 | Engine selection is configuration, not code (app_settings |
| 9 | intake.ocr_engine): |
| 10 | |
| 11 | text_rules the default and the FREE path -- regex over whatever text is |
| 12 | already on the scan. No model, no key, no per-page cost, and |
| 13 | it degrades to "found nothing, here is your image" rather |
| 14 | than to wrong numbers. |
| 15 | tesseract local OCR when pytesseract and PIL are installed on the host. |
| 16 | Absent, the pass says so and leaves the scan pending rather |
| 17 | than failing it: a missing system package is an operator's |
| 18 | problem to fix, not a document to give up on. |
| 19 | ai_vision object_ai with the owner's own key. Better guesses, and the |
| 20 | only path with a real per-page cost -- which is why it is |
| 21 | opt-in and why the key is theirs. |
| 22 | |
| 23 | Whatever the engine, the output lands in the same shape |
| 24 | (object_intake.normalize_extraction), so the confirm step cannot tell |
| 25 | which one read the document and does not care. That is what lets the |
| 26 | cheap path be the default without the expensive path being a rewrite. |
| 27 | |
| 28 | Failure posture: attempts are counted and a document that keeps failing |
| 29 | stops being retried and starts being VISIBLE. Churning a paid extractor |
| 30 | forever against a corrupt PDF is how an intake queue silently spends |
| 31 | money. |
| 32 | """ |
| 33 | |
| 34 | import json |
| 35 | import os |
| 36 | from datetime import date |
| 37 | |
| 38 | import object_intake |
| 39 | import object_records |
| 40 | import object_user_files |
| 41 | |
| 42 | ACTOR = "system_scan_processor" |
| 43 | |
| 44 | DEFAULT_LIMIT = 25 |
| 45 | MAX_ATTEMPTS = 3 |
| 46 | |
| 47 | |
| 48 | def _base_dir(): |
| 49 | return os.environ.get("DBBASIC_DATA_DIR", "data") |
| 50 | |
| 51 | |
| 52 | def _setting(base, key, default=""): |
| 53 | try: |
| 54 | for row in object_records.read_collection_records("app_settings", base_dir=base): |
| 55 | if row.get("key") == key and str(row.get("value") or "").strip(): |
| 56 | return row["value"].strip() |
| 57 | except Exception: |
| 58 | pass |
| 59 | return default |
| 60 | |
| 61 | |
| 62 | def _int(value, default=0): |
| 63 | try: |
| 64 | return int(str(value or "").strip() or default) |
| 65 | except (TypeError, ValueError): |
| 66 | return default |
| 67 | |
| 68 | |
| 69 | def _text(value): |
| 70 | return str(value if value is not None else "").strip() |
| 71 | |
| 72 | |
| 73 | def _read_text(base, scan, engine): |
| 74 | """Whatever text we can get, and how we got it. |
| 75 | |
| 76 | Returns (text, engine_used, error, recoverable). `recoverable` is |
| 77 | carried explicitly rather than sniffed out of the error message |
| 78 | later, because the two failures need opposite treatment and telling |
| 79 | them apart by substring is the kind of thing that silently stops |
| 80 | working when somebody rewords a message: |
| 81 | |
| 82 | recoverable -- the HOST is not set up (a package missing, an engine |
| 83 | not built in). The document is fine; leave it pending so fixing |
| 84 | the host drains the backlog. |
| 85 | not recoverable -- THIS document could not be read. Count the |
| 86 | attempt against it, so a corrupt file surfaces instead of |
| 87 | churning an extractor forever. |
| 88 | |
| 89 | Text already on the scan wins outright: a forwarded email receipt |
| 90 | needs no OCR, and re-reading an image we already read is pure cost. |
| 91 | """ |
| 92 | existing = _text(scan.get("ocr_text")) |
| 93 | if existing: |
| 94 | return existing, _text(scan.get("ocr_engine")) or "supplied", "", False |
| 95 | |
| 96 | if engine == "tesseract": |
| 97 | try: |
| 98 | import pytesseract # noqa: F401 |
| 99 | from PIL import Image # noqa: F401 |
| 100 | except ImportError: |
| 101 | return "", "", ("tesseract engine selected but pytesseract/PIL are " |
| 102 | "not installed on this host"), True |
| 103 | try: |
| 104 | import io |
| 105 | |
| 106 | content = object_user_files.read_file( |
| 107 | _text(scan.get("owner_id")), _text(scan.get("file_id")), base_dir=base) |
| 108 | image = Image.open(io.BytesIO(content)) |
| 109 | return pytesseract.image_to_string(image), "tesseract", "", False |
| 110 | except Exception as exc: |
| 111 | return "", "", f"tesseract failed: {str(exc)[:120]}", False |
| 112 | |
| 113 | if engine == "ai_vision": |
| 114 | try: |
| 115 | import object_ai |
| 116 | except ImportError: |
| 117 | return "", "", "ai_vision engine selected but object_ai is unavailable", True |
| 118 | reader = getattr(object_ai, "read_document", None) |
| 119 | if reader is None: |
| 120 | return "", "", ("ai_vision engine selected but object_ai has no " |
| 121 | "document reader on this build"), True |
| 122 | try: |
| 123 | content = object_user_files.read_file( |
| 124 | _text(scan.get("owner_id")), _text(scan.get("file_id")), base_dir=base) |
| 125 | except Exception as exc: |
| 126 | return "", "", f"could not read the stored document: {str(exc)[:120]}", False |
| 127 | try: |
| 128 | return (_text(reader(content, kind=_text(scan.get("category_hint")))), |
| 129 | "ai_vision", "", False) |
| 130 | except Exception as exc: |
| 131 | return "", "", f"ai_vision failed: {str(exc)[:120]}", False |
| 132 | |
| 133 | # text_rules and anything unrecognised: we read what is there, which |
| 134 | # for an image with no OCR engine is nothing -- and nothing is an |
| 135 | # honest answer that still leaves a confirmable draft. |
| 136 | return "", "text_rules", "", False |
| 137 | |
| 138 | |
| 139 | def POST(request): |
| 140 | base = _base_dir() |
| 141 | limit = _int(request.get("limit"), DEFAULT_LIMIT) or DEFAULT_LIMIT |
| 142 | engine = (_text(request.get("engine")) |
| 143 | or _setting(base, "intake.ocr_engine", "text_rules")) |
| 144 | |
| 145 | try: |
| 146 | scans = object_records.read_collection_records("scans", base_dir=base) |
| 147 | except Exception: |
| 148 | return {"ok": True, "skipped": "intake not installed (scans absent)"} |
| 149 | |
| 150 | pending = [row for row in scans |
| 151 | if _text(row.get("status")) in ("pending", "processing") |
| 152 | and _int(row.get("attempts")) < MAX_ATTEMPTS] |
| 153 | pending.sort(key=lambda row: _text(row.get("created_at"))) |
| 154 | |
| 155 | read = extracted = failed = 0 |
| 156 | results = [] |
| 157 | for scan in pending[:limit]: |
| 158 | attempts = _int(scan.get("attempts")) + 1 |
| 159 | text, engine_used, error, recoverable = _read_text(base, scan, engine) |
| 160 | if error: |
| 161 | # A host misconfiguration leaves the document PENDING (fix the |
| 162 | # host, run again); a genuine read failure counts against it. |
| 163 | object_records.update_collection_record( |
| 164 | "scans", scan["id"], |
| 165 | {"status": "pending" if recoverable else "error", |
| 166 | "attempts": str(attempts), "error": error}, |
| 167 | base_dir=base, actor=ACTOR) |
| 168 | failed += 1 |
| 169 | results.append({"scan": scan["id"], "error": error}) |
| 170 | continue |
| 171 | |
| 172 | suggestion = object_intake.guess_from_text( |
| 173 | text, kind_hint=_text(scan.get("category_hint"))) |
| 174 | if engine_used and engine_used not in ("text_rules", "supplied"): |
| 175 | suggestion["engine"] = engine_used |
| 176 | object_records.update_collection_record( |
| 177 | "scans", scan["id"], |
| 178 | { |
| 179 | "status": "extracted", |
| 180 | "ocr_text": text, |
| 181 | "ocr_engine": engine_used or "text_rules", |
| 182 | "extracted": json.dumps(suggestion), |
| 183 | "confidence": str(suggestion["confidence"]), |
| 184 | "attempts": str(attempts), |
| 185 | "error": "", |
| 186 | }, |
| 187 | base_dir=base, actor=ACTOR) |
| 188 | read += 1 if text else 0 |
| 189 | extracted += 1 |
| 190 | results.append({"scan": scan["id"], "engine": engine_used, |
| 191 | "total_cents": suggestion["total_cents"], |
| 192 | "confidence": suggestion["confidence"]}) |
| 193 | |
| 194 | stuck = [row["id"] for row in scans |
| 195 | if _int(row.get("attempts")) >= MAX_ATTEMPTS |
| 196 | and _text(row.get("status")) == "error"] |
| 197 | return {"ok": True, "today": _text(request.get("today")) or date.today().isoformat(), |
| 198 | "engine": engine, "considered": len(pending), "extracted": extracted, |
| 199 | "text_read": read, "failed": failed, |
| 200 | "needing_a_human": stuck, "results": results} |