askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_mail.py · 36.8 KB · 921 lines
| 1 | """Inbound mail, the pure half: bytes and IMAP answers in, app-messaging rows |
| 2 | and cursor decisions out. No network, no clock, no data directory. |
| 3 | |
| 4 | The I/O lives in packages/app-mail-sync (``system_mail_sync``), which opens |
| 5 | the IMAP connection, asks the questions this module plans, and writes the |
| 6 | rows this module shapes. Everything that can be wrong about mail without a |
| 7 | server being involved is decided here, where a test can reach it. |
| 8 | |
| 9 | WHAT THE PREDECESSOR GOT WRONG, and where each is answered in this file. |
| 10 | The mail sync this replaces taught four lessons, each paid for: |
| 11 | |
| 12 | - **It marked mail read.** It searched UNSEEN and fetched RFC822, and a |
| 13 | plain FETCH of RFC822 sets \\Seen. So syncing made every message look |
| 14 | read in the person's own mail program, and anything they had already |
| 15 | read elsewhere was never synced at all because it was no longer unseen. |
| 16 | Here the cursor is by UID, not by flag (``next_uids``), the fetch is |
| 17 | BODY.PEEK (the object's job), and ``is_read`` is read off the server's |
| 18 | FLAGS rather than decided by us. |
| 19 | - **It leaked across tenants.** Dedupe and threading were keyed on the |
| 20 | Message-ID alone, so the same message in two people's mailboxes became |
| 21 | one row, and a reply could join a thread owned by somebody else. |
| 22 | ``stable_id`` folds the owner and the account into every id this module |
| 23 | mints, and ``ThreadIndex`` refuses rows that belong to anyone but the |
| 24 | owner it was built for. |
| 25 | - **It joined threads by subject.** Two strangers both writing "Invoice" |
| 26 | ended up in one conversation. ``match_thread`` uses In-Reply-To and |
| 27 | References only; a message with neither starts its own thread. |
| 28 | - **It swallowed failures.** ``except: pass`` around the fetch meant a box |
| 29 | whose password had changed looked exactly like a quiet inbox. Nothing |
| 30 | here catches and hides; the object records every failure on the account |
| 31 | (sync_attempts, sync_error) and the attention count shows it. |
| 32 | |
| 33 | And one thing it never did and this must never start: fetch a URL found in |
| 34 | a message. List-Unsubscribe is PARSED and STORED so a person (or a later |
| 35 | package they choose) can act on it; nothing in this module or the object |
| 36 | opens it. A server that fetches links out of mail is a server any sender |
| 37 | can make issue requests from inside the network. |
| 38 | |
| 39 | Dates are written in the server's own timestamp spelling |
| 40 | (``2026-09-26T10:00:00Z``: UTC, ``Z``, microseconds only when present), |
| 41 | because two spellings of one instant do not sort together (AGENTS.md). |
| 42 | """ |
| 43 | |
| 44 | from __future__ import annotations |
| 45 | |
| 46 | import email |
| 47 | import email.header |
| 48 | import email.policy |
| 49 | import email.utils |
| 50 | import hashlib |
| 51 | import html |
| 52 | import html.parser |
| 53 | import ipaddress |
| 54 | import re |
| 55 | import uuid |
| 56 | from datetime import datetime, timedelta, timezone |
| 57 | from typing import Any, Iterable, Mapping |
| 58 | |
| 59 | # --- the gate's names (read by the object; named here so docs and tests share them) |
| 60 | |
| 61 | ENABLE_ENV = "DBBASIC_ENABLE_MAIL_SYNC" |
| 62 | ALLOW_PRIVATE_HOSTS_ENV = "DBBASIC_MAIL_SYNC_ALLOW_PRIVATE_HOSTS" |
| 63 | KILL_SWITCH_FLAG = "mail_sync_enabled" |
| 64 | PASSWORD_SERVICE_PREFIX = "imap-" |
| 65 | # Failures in a row before an account is counted on the home page. Shared |
| 66 | # by the sync (which counts them) and its attention provider (which reads |
| 67 | # them), so the two cannot disagree about what "keeps failing" means. |
| 68 | ATTENTION_AFTER = 3 |
| 69 | |
| 70 | # --- field bounds (the app-messaging schema's own, plus this package's) ------- |
| 71 | |
| 72 | MAX_SUBJECT = 300 |
| 73 | MAX_ADDRESS = 254 |
| 74 | MAX_MESSAGE_ID = 998 |
| 75 | MAX_SUMMARY = 300 |
| 76 | MAX_REFERENCES = 8000 |
| 77 | MAX_HEADER_VALUE = 2000 |
| 78 | MAX_BODY_CHARS = 200_000 |
| 79 | |
| 80 | NO_SENDER = "(no sender)" |
| 81 | |
| 82 | # A fixed namespace for uuid5 ids. Changing it would re-import every mailbox |
| 83 | # as new rows beside the old ones, so it never changes. |
| 84 | _ID_NAMESPACE = uuid.UUID("6f1d4b0e-3c1a-5b8e-9a61-0d5c3e2f7a90") |
| 85 | |
| 86 | _MSGID_RE = re.compile(r"<[^<>\s]+>") |
| 87 | _CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") |
| 88 | _BLANK_LINES_RE = re.compile(r"\n{3,}") |
| 89 | _MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", |
| 90 | "Jul", "Aug", "Sep", "Oct", "Nov", "Dec") |
| 91 | |
| 92 | |
| 93 | # --- small text helpers -------------------------------------------------------- |
| 94 | |
| 95 | def clean_text(value: Any) -> str: |
| 96 | """Text that the TSV substrate will store: valid UTF-8, no NUL or other |
| 97 | control characters except tab and newline, newlines as \\n. |
| 98 | |
| 99 | Headers with raw 8-bit bytes come out of the email package carrying |
| 100 | surrogate escapes, which cannot be encoded as UTF-8 at all; they are |
| 101 | turned back into bytes and decoded with replacement, so a badly encoded |
| 102 | subject shows a replacement character instead of failing the write. |
| 103 | """ |
| 104 | text = "" if value is None else str(value) |
| 105 | text = text.encode("utf-8", "surrogateescape").decode("utf-8", "replace") |
| 106 | text = text.replace("\r\n", "\n").replace("\r", "\n") |
| 107 | return _CONTROL_RE.sub("", text) |
| 108 | |
| 109 | |
| 110 | def one_line(value: Any) -> str: |
| 111 | """A header value as one line: folded whitespace collapsed.""" |
| 112 | return " ".join(clean_text(value).split()) |
| 113 | |
| 114 | |
| 115 | def cap(value: str, limit: int) -> str: |
| 116 | """At most ``limit`` characters, marking a cut with an ellipsis.""" |
| 117 | if len(value) <= limit: |
| 118 | return value |
| 119 | return value[: max(0, limit - 1)] + "…" |
| 120 | |
| 121 | |
| 122 | def format_timestamp(moment: datetime) -> str: |
| 123 | """The server's timestamp spelling. A naive datetime is taken as UTC.""" |
| 124 | if moment.tzinfo is None: |
| 125 | moment = moment.replace(tzinfo=timezone.utc) |
| 126 | return moment.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") |
| 127 | |
| 128 | |
| 129 | def parse_timestamp(value: Any) -> datetime | None: |
| 130 | """A stored timestamp back to an aware datetime, or None. Compare |
| 131 | these, never the strings (AGENTS.md: two spellings do not sort).""" |
| 132 | raw = str(value or "").strip() |
| 133 | if not raw: |
| 134 | return None |
| 135 | try: |
| 136 | moment = datetime.fromisoformat(raw.replace("Z", "+00:00")) |
| 137 | except ValueError: |
| 138 | return None |
| 139 | return moment if moment.tzinfo else moment.replace(tzinfo=timezone.utc) |
| 140 | |
| 141 | |
| 142 | def parse_date_header(value: str) -> str: |
| 143 | """An RFC 5322 Date header as a server timestamp, or "" when it cannot |
| 144 | be read. A sender's clock is not evidence of much, so a missing or |
| 145 | broken Date is left blank rather than guessed.""" |
| 146 | raw = one_line(value) |
| 147 | if not raw: |
| 148 | return "" |
| 149 | try: |
| 150 | moment = email.utils.parsedate_to_datetime(raw) |
| 151 | except (TypeError, ValueError, IndexError, OverflowError): |
| 152 | return "" |
| 153 | if moment is None: |
| 154 | return "" |
| 155 | try: |
| 156 | return format_timestamp(moment) |
| 157 | except (OverflowError, ValueError): |
| 158 | return "" |
| 159 | |
| 160 | |
| 161 | def parse_internaldate(value: str) -> str: |
| 162 | """An IMAP INTERNALDATE (``26-Sep-2026 10:00:00 +0000``) as a server |
| 163 | timestamp, or "". Parsed by hand rather than with strptime's %b, which |
| 164 | follows the process locale.""" |
| 165 | match = re.fullmatch( |
| 166 | r"\s*(\d{1,2})-([A-Za-z]{3})-(\d{4}) (\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2})\s*", |
| 167 | str(value or "")) |
| 168 | if not match: |
| 169 | return "" |
| 170 | day, month, year, hour, minute, second, sign, off_h, off_m = match.groups() |
| 171 | try: |
| 172 | month_number = [m.lower() for m in _MONTHS].index(month.lower()) + 1 |
| 173 | offset = timedelta(hours=int(off_h), minutes=int(off_m)) |
| 174 | zone = timezone(offset if sign == "+" else -offset) |
| 175 | moment = datetime(int(year), month_number, int(day), |
| 176 | int(hour), int(minute), int(second), tzinfo=zone) |
| 177 | except ValueError: |
| 178 | return "" |
| 179 | return format_timestamp(moment) |
| 180 | |
| 181 | |
| 182 | def imap_date(moment: datetime) -> str: |
| 183 | """A date as IMAP SEARCH wants it (``26-Sep-2026``), locale-free.""" |
| 184 | return f"{moment.day}-{_MONTHS[moment.month - 1]}-{moment.year}" |
| 185 | |
| 186 | |
| 187 | def extract_message_ids(value: str) -> list[str]: |
| 188 | """Every ``<id>`` in a header, in order, without repeats.""" |
| 189 | seen: list[str] = [] |
| 190 | for found in _MSGID_RE.findall(clean_text(value)): |
| 191 | if found not in seen and len(found) <= MAX_MESSAGE_ID: |
| 192 | seen.append(found) |
| 193 | return seen |
| 194 | |
| 195 | |
| 196 | def stable_id(*parts: Any) -> str: |
| 197 | """A deterministic record id for the same facts. |
| 198 | |
| 199 | Every id the sync writes is minted here from the owner, the account and |
| 200 | the message's dedupe key. That is what makes a replayed batch safe: the |
| 201 | second attempt at the same message produces the same id, the create is |
| 202 | refused as a duplicate, and nothing is written twice. It is also the |
| 203 | tenant boundary for ids -- the same Message-ID in two people's |
| 204 | mailboxes yields two different ids, because the owner is part of the |
| 205 | name. |
| 206 | """ |
| 207 | name = "\x1f".join(str(part) for part in parts) |
| 208 | return str(uuid.uuid5(_ID_NAMESPACE, name)) |
| 209 | |
| 210 | |
| 211 | # --- HTML to text --------------------------------------------------------------- |
| 212 | |
| 213 | class _TextExtractor(html.parser.HTMLParser): |
| 214 | _SKIP = {"script", "style", "head", "title", "template", "noscript"} |
| 215 | _BREAK = {"br", "p", "div", "tr", "li", "h1", "h2", "h3", "h4", "h5", "h6", |
| 216 | "blockquote", "pre", "table", "ul", "ol", "hr", "section", "article"} |
| 217 | |
| 218 | def __init__(self) -> None: |
| 219 | super().__init__(convert_charrefs=True) |
| 220 | self.parts: list[str] = [] |
| 221 | self.skipping = 0 |
| 222 | |
| 223 | def handle_starttag(self, tag, attrs): |
| 224 | if tag in self._SKIP: |
| 225 | self.skipping += 1 |
| 226 | elif tag in self._BREAK: |
| 227 | self.parts.append("\n") |
| 228 | if tag == "li": |
| 229 | self.parts.append("- ") |
| 230 | |
| 231 | def handle_endtag(self, tag): |
| 232 | if tag in self._SKIP: |
| 233 | self.skipping = max(0, self.skipping - 1) |
| 234 | elif tag in self._BREAK: |
| 235 | self.parts.append("\n") |
| 236 | |
| 237 | def handle_data(self, data): |
| 238 | if not self.skipping: |
| 239 | self.parts.append(data) |
| 240 | |
| 241 | |
| 242 | def html_to_text(markup: str) -> str: |
| 243 | """Readable text from an HTML body: scripts and styles dropped, blocks |
| 244 | as line breaks, entities decoded, runs of blank lines collapsed. No |
| 245 | link is followed and no image is loaded; this only reads the string.""" |
| 246 | parser = _TextExtractor() |
| 247 | try: |
| 248 | parser.feed(markup) |
| 249 | parser.close() |
| 250 | except Exception: # noqa: BLE001 -- html.parser is lenient; a crash still leaves text |
| 251 | return clean_text(re.sub(r"<[^>]*>", " ", html.unescape(markup))) |
| 252 | lines = [" ".join(line.split()) for line in "".join(parser.parts).split("\n")] |
| 253 | text = "\n".join(lines).strip() |
| 254 | return _BLANK_LINES_RE.sub("\n\n", text) |
| 255 | |
| 256 | |
| 257 | # --- parsing one message --------------------------------------------------------- |
| 258 | |
| 259 | def _raw_header(message, name: str) -> list[str]: |
| 260 | """Header values the slow, forgiving way: the raw text, decoded with |
| 261 | email.header. Used when the policy's structured parse raises.""" |
| 262 | values = [] |
| 263 | for key, value in message.raw_items(): |
| 264 | if key.lower() != name.lower(): |
| 265 | continue |
| 266 | try: |
| 267 | values.append(str(email.header.make_header(email.header.decode_header(str(value))))) |
| 268 | except Exception: # noqa: BLE001 -- keep the undecoded text rather than nothing |
| 269 | values.append(str(value)) |
| 270 | return values |
| 271 | |
| 272 | |
| 273 | def _header_values(message, name: str) -> list: |
| 274 | """Every value of one header, parsed by the default policy (which is |
| 275 | where a malformed header raises), else decoded from the raw text.""" |
| 276 | try: |
| 277 | return list(message.get_all(name) or []) |
| 278 | except Exception: # noqa: BLE001 -- fall back to the raw text, never drop the header |
| 279 | return _raw_header(message, name) |
| 280 | |
| 281 | |
| 282 | def header_text(message, name: str) -> str: |
| 283 | values = _header_values(message, name) |
| 284 | return one_line(values[0]) if values else "" |
| 285 | |
| 286 | |
| 287 | def header_addresses(message, name: str) -> list[tuple[str, str]]: |
| 288 | """(display name, address) pairs from every instance of one header.""" |
| 289 | pairs: list[tuple[str, str]] = [] |
| 290 | for value in _header_values(message, name): |
| 291 | found = None |
| 292 | try: |
| 293 | found = [(a.display_name, a.addr_spec) for a in value.addresses] |
| 294 | except Exception: # noqa: BLE001 -- not a structured header, or it would not parse |
| 295 | found = None |
| 296 | if found is None: |
| 297 | found = email.utils.getaddresses([str(value)]) |
| 298 | for display, address in found: |
| 299 | address = one_line(address).strip("<>") |
| 300 | if "@" in address and len(address) <= MAX_ADDRESS: |
| 301 | pairs.append((one_line(display), address)) |
| 302 | return pairs |
| 303 | |
| 304 | |
| 305 | def _decode_part(part) -> str: |
| 306 | try: |
| 307 | return part.get_content() |
| 308 | except (LookupError, UnicodeError, AssertionError, ValueError): |
| 309 | payload = part.get_payload(decode=True) or b"" |
| 310 | charset = part.get_content_charset() or "utf-8" |
| 311 | try: |
| 312 | return payload.decode(charset, "replace") |
| 313 | except LookupError: |
| 314 | return payload.decode("utf-8", "replace") |
| 315 | |
| 316 | |
| 317 | def _attachment_names(message) -> list[str]: |
| 318 | names = [] |
| 319 | try: |
| 320 | parts = list(message.iter_attachments()) if message.is_multipart() else [] |
| 321 | except Exception: # noqa: BLE001 -- a malformed tree still has a body worth keeping |
| 322 | parts = [] |
| 323 | for part in parts: |
| 324 | try: |
| 325 | name = part.get_filename() or part.get_content_type() |
| 326 | except Exception: # noqa: BLE001 |
| 327 | name = "attachment" |
| 328 | names.append(one_line(name) or "attachment") |
| 329 | return names |
| 330 | |
| 331 | |
| 332 | def body_text(message) -> str: |
| 333 | """The message's words: the text/plain body, else its HTML body as text, |
| 334 | with a closing line naming any attachments that were not copied.""" |
| 335 | text = "" |
| 336 | try: |
| 337 | part = message.get_body(preferencelist=("plain", "html")) |
| 338 | except Exception: # noqa: BLE001 |
| 339 | part = None |
| 340 | if part is not None: |
| 341 | content = _decode_part(part) |
| 342 | if part.get_content_type() == "text/html": |
| 343 | content = html_to_text(content) |
| 344 | text = clean_text(content).strip() |
| 345 | names = _attachment_names(message) |
| 346 | if names: |
| 347 | note = "[Attachments not copied: " + ", ".join(names[:20]) |
| 348 | note += f" and {len(names) - 20} more]" if len(names) > 20 else "]" |
| 349 | text = f"{text}\n\n{note}" if text else note |
| 350 | if len(text) > MAX_BODY_CHARS: |
| 351 | text = text[:MAX_BODY_CHARS] + ( |
| 352 | f"\n\n[Cut at {MAX_BODY_CHARS} characters; the whole message is on the mail server.]") |
| 353 | return text |
| 354 | |
| 355 | |
| 356 | def parse_flags(flags: Iterable[str] | str) -> list[str]: |
| 357 | if isinstance(flags, str): |
| 358 | flags = flags.split() |
| 359 | return [str(flag) for flag in flags if str(flag)] |
| 360 | |
| 361 | |
| 362 | def parse_message( |
| 363 | raw: bytes, |
| 364 | *, |
| 365 | flags: Iterable[str] | str = (), |
| 366 | internaldate: str = "", |
| 367 | header_only_size: int | None = None, |
| 368 | ) -> dict[str, Any]: |
| 369 | """RFC 5322 bytes to the facts the rows need. |
| 370 | |
| 371 | ``header_only_size`` says the object fetched only the header because |
| 372 | the message was too big to download; the body then says exactly that, |
| 373 | with the size, rather than looking empty. |
| 374 | """ |
| 375 | message = email.message_from_bytes(raw, policy=email.policy.default) |
| 376 | senders = header_addresses(message, "From") |
| 377 | from_name, from_address = senders[0] if senders else ("", "") |
| 378 | |
| 379 | message_ids = extract_message_ids(header_text(message, "Message-ID")) |
| 380 | message_id = message_ids[0] if message_ids else "" |
| 381 | if not message_id: |
| 382 | bare = header_text(message, "Message-ID").strip() |
| 383 | if bare and " " not in bare and len(bare) + 2 <= MAX_MESSAGE_ID: |
| 384 | message_id = f"<{bare}>" |
| 385 | replies = extract_message_ids(header_text(message, "In-Reply-To")) |
| 386 | references = extract_message_ids(" ".join( |
| 387 | one_line(value) for value in _header_values(message, "References"))) |
| 388 | |
| 389 | if header_only_size is not None: |
| 390 | megabytes = header_only_size / (1024 * 1024) |
| 391 | body = (f"[This message is {megabytes:.1f} MB, larger than this server " |
| 392 | f"downloads, so only its headers were copied. Open it in your " |
| 393 | f"mail program to read it.]") |
| 394 | else: |
| 395 | body = body_text(message) |
| 396 | |
| 397 | flag_list = parse_flags(flags) |
| 398 | date_raw = header_text(message, "Date") |
| 399 | return { |
| 400 | "message_id": message_id, |
| 401 | "in_reply_to": replies[0] if replies else "", |
| 402 | "references": references, |
| 403 | "subject": header_text(message, "Subject"), |
| 404 | "from_name": from_name, |
| 405 | "from_address": from_address, |
| 406 | "to": [address for _, address in header_addresses(message, "To")], |
| 407 | "cc": [address for _, address in header_addresses(message, "Cc")], |
| 408 | "bcc": [address for _, address in header_addresses(message, "Bcc")], |
| 409 | "date_raw": date_raw, |
| 410 | "date": parse_date_header(date_raw), |
| 411 | "internaldate": parse_internaldate(internaldate), |
| 412 | "body_text": body, |
| 413 | "flags": flag_list, |
| 414 | "is_read": any(flag.lower() == "\\seen" for flag in flag_list), |
| 415 | "list_unsubscribe": cap(header_text(message, "List-Unsubscribe"), MAX_HEADER_VALUE), |
| 416 | "list_unsubscribe_post": cap(header_text(message, "List-Unsubscribe-Post"), MAX_HEADER_VALUE), |
| 417 | "list_id": cap(header_text(message, "List-Id"), MAX_HEADER_VALUE), |
| 418 | "precedence": cap(header_text(message, "Precedence"), 100), |
| 419 | } |
| 420 | |
| 421 | |
| 422 | def dedupe_key(parsed: Mapping[str, Any]) -> str: |
| 423 | """What makes two copies the same message: its Message-ID, or, for the |
| 424 | rare message without one, a hash of who sent it, when they said, the |
| 425 | subject and the start of the body. The key is compared only inside one |
| 426 | owner's one account (the object scopes the lookup, ``stable_id`` scopes |
| 427 | the ids), so it never needs to be globally unique.""" |
| 428 | if parsed.get("message_id"): |
| 429 | return "mid:" + str(parsed["message_id"]) |
| 430 | digest = hashlib.sha256("\x1f".join([ |
| 431 | str(parsed.get("from_address") or ""), |
| 432 | str(parsed.get("date_raw") or ""), |
| 433 | str(parsed.get("subject") or ""), |
| 434 | str(parsed.get("body_text") or "")[:4096], |
| 435 | ]).encode("utf-8", "replace")).hexdigest() |
| 436 | return "sha256:" + digest |
| 437 | |
| 438 | |
| 439 | def direction_for(parsed: Mapping[str, Any], *, account_address: str, folder_role: str) -> str: |
| 440 | """outbound for what this mailbox sent (the Sent folder, or From is the |
| 441 | account's own address), inbound for everything else.""" |
| 442 | if folder_role == "sent": |
| 443 | return "outbound" |
| 444 | sender = str(parsed.get("from_address") or "").casefold() |
| 445 | if sender and sender == str(account_address or "").casefold(): |
| 446 | return "outbound" |
| 447 | return "inbound" |
| 448 | |
| 449 | |
| 450 | # --- rows ---------------------------------------------------------------------- |
| 451 | |
| 452 | def _participants(parsed: Mapping[str, Any], account_address: str) -> list[str]: |
| 453 | own = str(account_address or "").casefold() |
| 454 | seen: list[str] = [] |
| 455 | for address in [parsed.get("from_address") or ""] + list(parsed.get("to") or []) + list(parsed.get("cc") or []): |
| 456 | if address and address.casefold() != own and address.casefold() not in {s.casefold() for s in seen}: |
| 457 | seen.append(address) |
| 458 | return seen |
| 459 | |
| 460 | |
| 461 | def participant_summary(existing: str, parsed: Mapping[str, Any], account_address: str) -> str: |
| 462 | names = [name.strip() for name in str(existing or "").split(",") if name.strip()] |
| 463 | for address in _participants(parsed, account_address): |
| 464 | if address.casefold() not in {name.casefold() for name in names}: |
| 465 | names.append(address) |
| 466 | return cap(", ".join(names), MAX_SUMMARY) |
| 467 | |
| 468 | |
| 469 | def message_rows( |
| 470 | parsed: Mapping[str, Any], |
| 471 | *, |
| 472 | owner_id: str, |
| 473 | message_id: str, |
| 474 | thread_id: str, |
| 475 | direction: str, |
| 476 | ) -> tuple[dict[str, str], list[dict[str, str]]]: |
| 477 | """The ``messages`` row and its ``message_recipients`` rows.""" |
| 478 | row = { |
| 479 | "id": message_id, |
| 480 | "thread_id": thread_id, |
| 481 | "direction": direction, |
| 482 | "from_address": parsed.get("from_address") or NO_SENDER, |
| 483 | "subject": cap(str(parsed.get("subject") or ""), MAX_SUBJECT), |
| 484 | "body_text": str(parsed.get("body_text") or ""), |
| 485 | "is_read": "true" if direction == "outbound" or parsed.get("is_read") else "false", |
| 486 | "in_reply_to": str(parsed.get("in_reply_to") or ""), |
| 487 | "external_message_id": str(parsed.get("message_id") or ""), |
| 488 | "owner_id": owner_id, |
| 489 | } |
| 490 | if direction == "outbound": |
| 491 | stamp = parsed.get("date") or parsed.get("internaldate") |
| 492 | if stamp: |
| 493 | row["sent_at"] = stamp |
| 494 | else: |
| 495 | stamp = parsed.get("internaldate") or parsed.get("date") |
| 496 | if stamp: |
| 497 | row["received_at"] = stamp |
| 498 | |
| 499 | recipients = [] |
| 500 | for kind in ("to", "cc", "bcc"): |
| 501 | seen: set[str] = set() |
| 502 | for address in parsed.get(kind) or []: |
| 503 | if address.casefold() in seen: |
| 504 | continue |
| 505 | seen.add(address.casefold()) |
| 506 | recipients.append({ |
| 507 | "id": stable_id("recipient", message_id, kind, address.casefold()), |
| 508 | "message_id": message_id, |
| 509 | "kind": kind, |
| 510 | "address": address, |
| 511 | "owner_id": owner_id, |
| 512 | }) |
| 513 | return row, recipients |
| 514 | |
| 515 | |
| 516 | def message_time(parsed: Mapping[str, Any], direction: str) -> str: |
| 517 | if direction == "outbound": |
| 518 | return parsed.get("date") or parsed.get("internaldate") or "" |
| 519 | return parsed.get("internaldate") or parsed.get("date") or "" |
| 520 | |
| 521 | |
| 522 | def new_thread_row( |
| 523 | parsed: Mapping[str, Any], *, owner_id: str, thread_id: str, |
| 524 | direction: str, account_address: str, |
| 525 | ) -> dict[str, str]: |
| 526 | row = { |
| 527 | "id": thread_id, |
| 528 | "thread_type": "email", |
| 529 | "subject": cap(str(parsed.get("subject") or ""), MAX_SUBJECT), |
| 530 | "participant_summary": participant_summary("", parsed, account_address), |
| 531 | "message_count": "1", |
| 532 | "is_read": "true" if direction == "outbound" or parsed.get("is_read") else "false", |
| 533 | "owner_id": owner_id, |
| 534 | } |
| 535 | stamp = message_time(parsed, direction) |
| 536 | if stamp: |
| 537 | row["last_message_at"] = stamp |
| 538 | return row |
| 539 | |
| 540 | |
| 541 | def thread_changes( |
| 542 | thread: Mapping[str, Any], parsed: Mapping[str, Any], *, |
| 543 | direction: str, account_address: str, |
| 544 | ) -> dict[str, str]: |
| 545 | """What one more message does to the thread it joined: the count goes up, |
| 546 | the latest time moves forward only (an old message arriving late does |
| 547 | not make a thread look fresh), an unread arrival makes the thread |
| 548 | unread, and new participants are added.""" |
| 549 | try: |
| 550 | count = int(float(str(thread.get("message_count") or "0"))) |
| 551 | except ValueError: |
| 552 | count = 0 |
| 553 | changes = {"message_count": str(count + 1)} |
| 554 | stamp = message_time(parsed, direction) |
| 555 | current = parse_timestamp(thread.get("last_message_at")) |
| 556 | incoming = parse_timestamp(stamp) |
| 557 | if incoming is not None and (current is None or incoming > current): |
| 558 | changes["last_message_at"] = stamp |
| 559 | if direction != "outbound" and not parsed.get("is_read"): |
| 560 | changes["is_read"] = "false" |
| 561 | summary = participant_summary(str(thread.get("participant_summary") or ""), parsed, account_address) |
| 562 | if summary != str(thread.get("participant_summary") or ""): |
| 563 | changes["participant_summary"] = summary |
| 564 | return changes |
| 565 | |
| 566 | |
| 567 | def synced_row( |
| 568 | parsed: Mapping[str, Any], *, row_id: str, owner_id: str, account_id: str, |
| 569 | folder: str, uid: int, uidvalidity: str, message_id: str, key: str, |
| 570 | ) -> dict[str, str]: |
| 571 | """The transport facts app-messaging deliberately does not carry.""" |
| 572 | return { |
| 573 | "id": row_id, |
| 574 | "account_id": account_id, |
| 575 | "folder": folder, |
| 576 | "uid": str(uid), |
| 577 | "uidvalidity": str(uidvalidity), |
| 578 | "message_id": message_id, |
| 579 | "dedupe_key": key, |
| 580 | "references": cap(" ".join(parsed.get("references") or []), MAX_REFERENCES), |
| 581 | "list_unsubscribe": str(parsed.get("list_unsubscribe") or ""), |
| 582 | "list_unsubscribe_post": str(parsed.get("list_unsubscribe_post") or ""), |
| 583 | "list_id": str(parsed.get("list_id") or ""), |
| 584 | "precedence": str(parsed.get("precedence") or ""), |
| 585 | "owner_id": owner_id, |
| 586 | } |
| 587 | |
| 588 | |
| 589 | # --- threading ----------------------------------------------------------------- |
| 590 | |
| 591 | class ThreadIndex: |
| 592 | """Which thread a Message-ID belongs to, for ONE owner. |
| 593 | |
| 594 | Built from that owner's rows only: a row whose owner_id is anyone |
| 595 | else's is skipped even if the caller passed it in, so a caller that |
| 596 | forgets to filter cannot make a thread leak across mailboxes. |
| 597 | """ |
| 598 | |
| 599 | def __init__(self, owner_id: str) -> None: |
| 600 | self.owner_id = owner_id |
| 601 | self.by_message_id: dict[str, str] = {} |
| 602 | self.by_parent: dict[str, str] = {} |
| 603 | |
| 604 | @classmethod |
| 605 | def from_rows( |
| 606 | cls, owner_id: str, messages: Iterable[Mapping[str, Any]], |
| 607 | synced: Iterable[Mapping[str, Any]] = (), |
| 608 | ) -> "ThreadIndex": |
| 609 | index = cls(owner_id) |
| 610 | thread_of_message: dict[str, str] = {} |
| 611 | for row in messages: |
| 612 | if str(row.get("owner_id") or "") != owner_id or not row.get("thread_id"): |
| 613 | continue |
| 614 | thread_id = str(row["thread_id"]) |
| 615 | thread_of_message[str(row.get("id") or "")] = thread_id |
| 616 | if row.get("external_message_id"): |
| 617 | index.by_message_id.setdefault(str(row["external_message_id"]), thread_id) |
| 618 | if row.get("in_reply_to"): |
| 619 | index.by_parent.setdefault(str(row["in_reply_to"]), thread_id) |
| 620 | for row in synced: |
| 621 | if str(row.get("owner_id") or "") != owner_id: |
| 622 | continue |
| 623 | thread_id = thread_of_message.get(str(row.get("message_id") or "")) |
| 624 | if not thread_id: |
| 625 | continue |
| 626 | for reference in str(row.get("references") or "").split(): |
| 627 | index.by_parent.setdefault(reference, thread_id) |
| 628 | return index |
| 629 | |
| 630 | def add(self, parsed: Mapping[str, Any], thread_id: str) -> None: |
| 631 | if parsed.get("message_id"): |
| 632 | self.by_message_id.setdefault(str(parsed["message_id"]), thread_id) |
| 633 | for parent in [parsed.get("in_reply_to")] + list(parsed.get("references") or []): |
| 634 | if parent: |
| 635 | self.by_parent.setdefault(str(parent), thread_id) |
| 636 | |
| 637 | |
| 638 | def match_thread(parsed: Mapping[str, Any], index: ThreadIndex) -> str | None: |
| 639 | """The owner's existing thread this message belongs to, or None. |
| 640 | |
| 641 | In order: the message it says it replies to; then its References, the |
| 642 | nearest ancestor first; then a reply to it that arrived first (Sent |
| 643 | and INBOX are separate folders, so a reply can be synced before its |
| 644 | parent). Never the subject: two people writing "Invoice" are not one |
| 645 | conversation. |
| 646 | """ |
| 647 | candidates = [parsed.get("in_reply_to")] + list(reversed(parsed.get("references") or [])) |
| 648 | for candidate in candidates: |
| 649 | if candidate and candidate in index.by_message_id: |
| 650 | return index.by_message_id[candidate] |
| 651 | own = parsed.get("message_id") |
| 652 | if own and own in index.by_parent: |
| 653 | return index.by_parent[own] |
| 654 | return None |
| 655 | |
| 656 | |
| 657 | # --- cursor planning ------------------------------------------------------------- |
| 658 | |
| 659 | def plan_folder(state: Mapping[str, Any] | None, *, uidvalidity: str) -> str: |
| 660 | """``fresh`` (never synced), ``reset`` (the server renumbered the folder: |
| 661 | UIDVALIDITY changed, so every stored UID is meaningless) or |
| 662 | ``continue``. A fresh or reset folder starts from a recent window, and |
| 663 | dedupe makes re-meeting old messages harmless.""" |
| 664 | stored = str((state or {}).get("uidvalidity") or "").strip() |
| 665 | if not stored: |
| 666 | return "fresh" |
| 667 | if stored != str(uidvalidity).strip(): |
| 668 | return "reset" |
| 669 | return "continue" |
| 670 | |
| 671 | |
| 672 | def window_baseline(window_uids: Iterable[int], *, first_window: int, highest_uid: int) -> int: |
| 673 | """Where a fresh cursor starts: just below the oldest of the newest |
| 674 | ``first_window`` recent messages, so the first sync takes a window and |
| 675 | not the whole history. Nothing recent means start at the top: the next |
| 676 | message to arrive is the first one synced.""" |
| 677 | recent = sorted({int(uid) for uid in window_uids})[-max(1, first_window):] |
| 678 | if not recent: |
| 679 | return max(0, int(highest_uid)) |
| 680 | return recent[0] - 1 |
| 681 | |
| 682 | |
| 683 | def next_uids(found: Iterable[int], *, last_uid: int, limit: int) -> list[int]: |
| 684 | """The next UIDs to import, oldest first, at most ``limit``. |
| 685 | |
| 686 | Filters ``> last_uid`` itself because ``UID SEARCH UID n:*`` answers |
| 687 | with the highest message even when n is beyond it -- a classic way for |
| 688 | a sync to import its newest message forever. |
| 689 | """ |
| 690 | return sorted({int(uid) for uid in found if int(uid) > int(last_uid)})[: max(0, limit)] |
| 691 | |
| 692 | |
| 693 | def plan_downloads( |
| 694 | metas: Iterable[Mapping[str, Any]], *, max_message_bytes: int, byte_budget: int, |
| 695 | ) -> list[tuple[int, str]]: |
| 696 | """(uid, "full" | "header") for this tick, oldest first. |
| 697 | |
| 698 | A message over ``max_message_bytes`` gets its header only (the body |
| 699 | then says so). The byte budget bounds one tick's download because the |
| 700 | sync runs inside the daemon's loop; it always admits the first message, |
| 701 | so a big message cannot stall the folder, and it stops before the one |
| 702 | that would overrun, which the next tick picks up. |
| 703 | """ |
| 704 | plan: list[tuple[int, str]] = [] |
| 705 | spent = 0 |
| 706 | for meta in sorted(metas, key=lambda m: int(m["uid"])): |
| 707 | size = int(meta.get("size") or 0) |
| 708 | mode = "full" if size <= max_message_bytes else "header" |
| 709 | cost = size if mode == "full" else 0 |
| 710 | if plan and spent + cost > byte_budget: |
| 711 | break |
| 712 | plan.append((int(meta["uid"]), mode)) |
| 713 | spent += cost |
| 714 | return plan |
| 715 | |
| 716 | |
| 717 | # --- account scheduling ----------------------------------------------------------- |
| 718 | |
| 719 | def account_due( |
| 720 | account: Mapping[str, Any], now: datetime, *, |
| 721 | claim_ttl_seconds: int, retry_base_seconds: int, retry_max_seconds: int, |
| 722 | ) -> tuple[bool, str]: |
| 723 | """(due, reason-not). Disabled accounts, accounts another pass is |
| 724 | working on, and failing accounts inside their backoff are not due. |
| 725 | |
| 726 | Backoff because a wrong password tried every two minutes forever is |
| 727 | how a mail provider locks an account. It doubles from |
| 728 | ``retry_base_seconds`` and stops growing at ``retry_max_seconds``; the |
| 729 | owner can zero sync_attempts to try again at once. |
| 730 | """ |
| 731 | if str(account.get("enabled") or "true").strip().lower() in {"false", "0", "no", "off"}: |
| 732 | return False, "disabled" |
| 733 | started = parse_timestamp(account.get("sync_started_at")) |
| 734 | finished = parse_timestamp(account.get("sync_finished_at")) |
| 735 | if started is not None and (finished is None or started > finished): |
| 736 | if (now - started).total_seconds() < claim_ttl_seconds: |
| 737 | return False, "another pass is syncing it" |
| 738 | try: |
| 739 | attempts = int(str(account.get("sync_attempts") or "0")) |
| 740 | except ValueError: |
| 741 | attempts = 0 |
| 742 | if attempts > 0 and finished is not None: |
| 743 | try: |
| 744 | wait = retry_base_seconds * (2 ** (attempts - 1)) |
| 745 | except OverflowError: |
| 746 | wait = retry_max_seconds |
| 747 | wait = min(wait, retry_max_seconds) |
| 748 | resume = finished + timedelta(seconds=wait) |
| 749 | if now < resume: |
| 750 | return False, f"waiting after {attempts} failed tries, until {format_timestamp(resume)}" |
| 751 | return True, "" |
| 752 | |
| 753 | |
| 754 | def password_service(account_id: str) -> str: |
| 755 | """The identity-vault service name holding an account's password.""" |
| 756 | return PASSWORD_SERVICE_PREFIX + str(account_id) |
| 757 | |
| 758 | |
| 759 | def host_refusal( |
| 760 | host: str, addresses: Iterable[str], *, security: str, allow_private: bool, |
| 761 | ) -> str | None: |
| 762 | """Why the server must not connect to this host, or None. |
| 763 | |
| 764 | The account's host is typed by a user, and the server is the one that |
| 765 | connects, so without this check any member could make the server knock |
| 766 | on addresses inside its own network (a database port on localhost, a |
| 767 | metadata service) and read reachability out of the error. Private, |
| 768 | loopback and link-local addresses are refused unless the operator set |
| 769 | DBBASIC_MAIL_SYNC_ALLOW_PRIVATE_HOSTS (a desk with its own mail server). |
| 770 | ``plain`` (no TLS) is allowed only to this machine itself: a password |
| 771 | sent in the clear over any real network is a password given away. |
| 772 | """ |
| 773 | resolved = [] |
| 774 | for address in addresses: |
| 775 | try: |
| 776 | resolved.append(ipaddress.ip_address(str(address).split("%", 1)[0])) |
| 777 | except ValueError: |
| 778 | return f"{host!r} resolved to {address!r}, which is not an address" |
| 779 | if not resolved: |
| 780 | return f"{host!r} did not resolve to any address" |
| 781 | if security == "plain" and not all(ip.is_loopback for ip in resolved): |
| 782 | return ("security 'plain' sends the password unencrypted, so it is only " |
| 783 | "allowed to this machine itself (127.0.0.1 or ::1); use ssl or starttls") |
| 784 | if not allow_private: |
| 785 | for ip in resolved: |
| 786 | if not ip.is_global: |
| 787 | return (f"{host!r} is a private or local address ({ip}); set " |
| 788 | f"{ALLOW_PRIVATE_HOSTS_ENV}=true if this server is meant to " |
| 789 | f"reach mail servers on its own network") |
| 790 | return None |
| 791 | |
| 792 | |
| 793 | # --- reading IMAP answers ---------------------------------------------------------- |
| 794 | |
| 795 | def _as_text(item: Any) -> str: |
| 796 | if isinstance(item, tuple): |
| 797 | item = item[0] |
| 798 | if isinstance(item, bytes): |
| 799 | return item.decode("utf-8", "replace") |
| 800 | return str(item or "") |
| 801 | |
| 802 | |
| 803 | def parse_search(data: Iterable[Any]) -> list[int]: |
| 804 | """UIDs from a UID SEARCH answer.""" |
| 805 | found: list[int] = [] |
| 806 | for item in data or []: |
| 807 | for token in _as_text(item).split(): |
| 808 | if token.isdigit(): |
| 809 | found.append(int(token)) |
| 810 | return found |
| 811 | |
| 812 | |
| 813 | def parse_fetch_meta(data: Iterable[Any]) -> list[dict[str, Any]]: |
| 814 | """uid, flags, size and internaldate from a ``UID FETCH (UID FLAGS |
| 815 | RFC822.SIZE INTERNALDATE)`` answer, one dict per message.""" |
| 816 | metas = [] |
| 817 | for item in data or []: |
| 818 | text = _as_text(item) |
| 819 | uid = re.search(r"\bUID (\d+)", text) |
| 820 | if not uid: |
| 821 | continue |
| 822 | size = re.search(r"\bRFC822\.SIZE (\d+)", text) |
| 823 | flags = re.search(r"\bFLAGS \(([^)]*)\)", text) |
| 824 | internal = re.search(r'\bINTERNALDATE "([^"]+)"', text) |
| 825 | metas.append({ |
| 826 | "uid": int(uid.group(1)), |
| 827 | "size": int(size.group(1)) if size else 0, |
| 828 | "flags": flags.group(1).split() if flags else [], |
| 829 | "internaldate": internal.group(1) if internal else "", |
| 830 | }) |
| 831 | return metas |
| 832 | |
| 833 | |
| 834 | def fetch_literal(data: Iterable[Any]) -> bytes | None: |
| 835 | """The message bytes from a ``UID FETCH (BODY.PEEK[...])`` answer.""" |
| 836 | for item in data or []: |
| 837 | if isinstance(item, tuple) and len(item) >= 2 and "BODY[" in _as_text(item[0]).upper(): |
| 838 | payload = item[1] |
| 839 | return payload if isinstance(payload, bytes) else str(payload).encode("utf-8") |
| 840 | return None |
| 841 | |
| 842 | |
| 843 | _LIST_RE = re.compile(r'^\((?P<flags>[^)]*)\)\s+(?P<delim>"(?:[^"\\]|\\.)*"|NIL)\s+(?P<name>.+)$') |
| 844 | _LIST_LITERAL_RE = re.compile(r'^\((?P<flags>[^)]*)\)\s+(?P<delim>"(?:[^"\\]|\\.)*"|NIL)\s+\{\d+\}$') |
| 845 | |
| 846 | |
| 847 | def parse_list(data: Iterable[Any]) -> list[tuple[list[str], str]]: |
| 848 | """(attributes, mailbox name) from a LIST answer. Names stay in the |
| 849 | server's own (modified UTF-7) spelling, which is what SELECT needs.""" |
| 850 | entries = [] |
| 851 | for item in data or []: |
| 852 | if item is None: |
| 853 | continue |
| 854 | if isinstance(item, tuple): |
| 855 | # A name sent as a literal: the attributes line, then its bytes. |
| 856 | match = _LIST_LITERAL_RE.match(_as_text(item[0]).strip()) |
| 857 | if match and len(item) >= 2: |
| 858 | entries.append((match.group("flags").split(), _as_text(item[1]))) |
| 859 | continue |
| 860 | match = _LIST_RE.match(_as_text(item).strip()) |
| 861 | if not match: |
| 862 | continue |
| 863 | name = match.group("name").strip() |
| 864 | if name.startswith('"') and name.endswith('"') and len(name) >= 2: |
| 865 | name = re.sub(r'\\(.)', r'\1', name[1:-1]) |
| 866 | entries.append((match.group("flags").split(), name)) |
| 867 | return entries |
| 868 | |
| 869 | |
| 870 | _SENT_NAMES = ("sent", "sent items", "sent messages", "sent mail", "inbox.sent", |
| 871 | "[gmail]/sent mail") |
| 872 | |
| 873 | |
| 874 | def choose_folders(entries: Iterable[tuple[list[str], str]]) -> list[tuple[str, str]]: |
| 875 | """(name, role) for a new account: INBOX, and the Sent folder when the |
| 876 | server has one -- by its \\Sent attribute (RFC 6154) first, then by the |
| 877 | names servers commonly use.""" |
| 878 | entries = list(entries) |
| 879 | chosen = [("INBOX", "inbox")] |
| 880 | sent = next((name for attrs, name in entries |
| 881 | if any(a.lower() == "\\sent" for a in attrs)), None) |
| 882 | if sent is None: |
| 883 | sent = next((name for attrs, name in entries |
| 884 | if name.lower() in _SENT_NAMES |
| 885 | and not any(a.lower() == "\\noselect" for a in attrs)), None) |
| 886 | if sent and sent.upper() != "INBOX": |
| 887 | chosen.append((sent, "sent")) |
| 888 | return chosen |
| 889 | |
| 890 | |
| 891 | def folder_role(name: str, entries: Iterable[tuple[list[str], str]]) -> str: |
| 892 | """inbox, sent or other, for a folder the owner named themselves.""" |
| 893 | if str(name).upper() == "INBOX": |
| 894 | return "inbox" |
| 895 | for attrs, listed in entries: |
| 896 | if listed == name and any(a.lower() == "\\sent" for a in attrs): |
| 897 | return "sent" |
| 898 | return "sent" if str(name).lower() in _SENT_NAMES else "other" |
| 899 | |
| 900 | |
| 901 | def split_folders(value: str) -> list[str]: |
| 902 | """The account's folders field: names separated by commas.""" |
| 903 | names = [] |
| 904 | for name in str(value or "").split(","): |
| 905 | name = name.strip() |
| 906 | if name and name not in names: |
| 907 | names.append(name) |
| 908 | return names |
| 909 | |
| 910 | |
| 911 | def quote_mailbox(name: str) -> str: |
| 912 | """A mailbox name as an IMAP quoted string (imaplib does not quote). |
| 913 | |
| 914 | A line break inside a quoted string would end the command and let the |
| 915 | rest of the name run as a second one on the person's mailbox (a |
| 916 | DELETE, an EXPUNGE), so a name carrying one is refused, not quoted. |
| 917 | """ |
| 918 | name = str(name) |
| 919 | if "\r" in name or "\n" in name or "\x00" in name: |
| 920 | raise ValueError("a mailbox name cannot contain a line break") |
| 921 | return '"' + name.replace("\\", "\\\\").replace('"', '\\"') + '"' |