Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_mail.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_mail.py · 36.8 KB · 921 lines
1"""Inbound mail, the pure half: bytes and IMAP answers in, app-messaging rows
2and cursor decisions out. No network, no clock, no data directory.
3
4The I/O lives in packages/app-mail-sync (``system_mail_sync``), which opens
5the IMAP connection, asks the questions this module plans, and writes the
6rows this module shapes. Everything that can be wrong about mail without a
7server being involved is decided here, where a test can reach it.
8
9WHAT THE PREDECESSOR GOT WRONG, and where each is answered in this file.
10The mail sync this replaces taught four lessons, each paid for:
11
12- **It marked mail read.** It searched UNSEEN and fetched RFC822, and a
13 plain FETCH of RFC822 sets \\Seen. So syncing made every message look
14 read in the person's own mail program, and anything they had already
15 read elsewhere was never synced at all because it was no longer unseen.
16 Here the cursor is by UID, not by flag (``next_uids``), the fetch is
17 BODY.PEEK (the object's job), and ``is_read`` is read off the server's
18 FLAGS rather than decided by us.
19- **It leaked across tenants.** Dedupe and threading were keyed on the
20 Message-ID alone, so the same message in two people's mailboxes became
21 one row, and a reply could join a thread owned by somebody else.
22 ``stable_id`` folds the owner and the account into every id this module
23 mints, and ``ThreadIndex`` refuses rows that belong to anyone but the
24 owner it was built for.
25- **It joined threads by subject.** Two strangers both writing "Invoice"
26 ended up in one conversation. ``match_thread`` uses In-Reply-To and
27 References only; a message with neither starts its own thread.
28- **It swallowed failures.** ``except: pass`` around the fetch meant a box
29 whose password had changed looked exactly like a quiet inbox. Nothing
30 here catches and hides; the object records every failure on the account
31 (sync_attempts, sync_error) and the attention count shows it.
32
33And one thing it never did and this must never start: fetch a URL found in
34a message. List-Unsubscribe is PARSED and STORED so a person (or a later
35package they choose) can act on it; nothing in this module or the object
36opens it. A server that fetches links out of mail is a server any sender
37can make issue requests from inside the network.
38
39Dates are written in the server's own timestamp spelling
40(``2026-09-26T10:00:00Z``: UTC, ``Z``, microseconds only when present),
41because two spellings of one instant do not sort together (AGENTS.md).
42"""
43
44from __future__ import annotations
45
46import email
47import email.header
48import email.policy
49import email.utils
50import hashlib
51import html
52import html.parser
53import ipaddress
54import re
55import uuid
56from datetime import datetime, timedelta, timezone
57from typing import Any, Iterable, Mapping
58
59# --- the gate's names (read by the object; named here so docs and tests share them)
60
61ENABLE_ENV = "DBBASIC_ENABLE_MAIL_SYNC"
62ALLOW_PRIVATE_HOSTS_ENV = "DBBASIC_MAIL_SYNC_ALLOW_PRIVATE_HOSTS"
63KILL_SWITCH_FLAG = "mail_sync_enabled"
64PASSWORD_SERVICE_PREFIX = "imap-"
65# Failures in a row before an account is counted on the home page. Shared
66# by the sync (which counts them) and its attention provider (which reads
67# them), so the two cannot disagree about what "keeps failing" means.
68ATTENTION_AFTER = 3
69
70# --- field bounds (the app-messaging schema's own, plus this package's) -------
71
72MAX_SUBJECT = 300
73MAX_ADDRESS = 254
74MAX_MESSAGE_ID = 998
75MAX_SUMMARY = 300
76MAX_REFERENCES = 8000
77MAX_HEADER_VALUE = 2000
78MAX_BODY_CHARS = 200_000
79
80NO_SENDER = "(no sender)"
81
82# A fixed namespace for uuid5 ids. Changing it would re-import every mailbox
83# as new rows beside the old ones, so it never changes.
84_ID_NAMESPACE = uuid.UUID("6f1d4b0e-3c1a-5b8e-9a61-0d5c3e2f7a90")
85
86_MSGID_RE = re.compile(r"<[^<>\s]+>")
87_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]")
88_BLANK_LINES_RE = re.compile(r"\n{3,}")
89_MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun",
90 "Jul", "Aug", "Sep", "Oct", "Nov", "Dec")
91
92
93# --- small text helpers --------------------------------------------------------
94
95def clean_text(value: Any) -> str:
96 """Text that the TSV substrate will store: valid UTF-8, no NUL or other
97 control characters except tab and newline, newlines as \\n.
98
99 Headers with raw 8-bit bytes come out of the email package carrying
100 surrogate escapes, which cannot be encoded as UTF-8 at all; they are
101 turned back into bytes and decoded with replacement, so a badly encoded
102 subject shows a replacement character instead of failing the write.
103 """
104 text = "" if value is None else str(value)
105 text = text.encode("utf-8", "surrogateescape").decode("utf-8", "replace")
106 text = text.replace("\r\n", "\n").replace("\r", "\n")
107 return _CONTROL_RE.sub("", text)
108
109
110def one_line(value: Any) -> str:
111 """A header value as one line: folded whitespace collapsed."""
112 return " ".join(clean_text(value).split())
113
114
115def cap(value: str, limit: int) -> str:
116 """At most ``limit`` characters, marking a cut with an ellipsis."""
117 if len(value) <= limit:
118 return value
119 return value[: max(0, limit - 1)] + "…"
120
121
122def format_timestamp(moment: datetime) -> str:
123 """The server's timestamp spelling. A naive datetime is taken as UTC."""
124 if moment.tzinfo is None:
125 moment = moment.replace(tzinfo=timezone.utc)
126 return moment.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
127
128
129def parse_timestamp(value: Any) -> datetime | None:
130 """A stored timestamp back to an aware datetime, or None. Compare
131 these, never the strings (AGENTS.md: two spellings do not sort)."""
132 raw = str(value or "").strip()
133 if not raw:
134 return None
135 try:
136 moment = datetime.fromisoformat(raw.replace("Z", "+00:00"))
137 except ValueError:
138 return None
139 return moment if moment.tzinfo else moment.replace(tzinfo=timezone.utc)
140
141
142def parse_date_header(value: str) -> str:
143 """An RFC 5322 Date header as a server timestamp, or "" when it cannot
144 be read. A sender's clock is not evidence of much, so a missing or
145 broken Date is left blank rather than guessed."""
146 raw = one_line(value)
147 if not raw:
148 return ""
149 try:
150 moment = email.utils.parsedate_to_datetime(raw)
151 except (TypeError, ValueError, IndexError, OverflowError):
152 return ""
153 if moment is None:
154 return ""
155 try:
156 return format_timestamp(moment)
157 except (OverflowError, ValueError):
158 return ""
159
160
161def parse_internaldate(value: str) -> str:
162 """An IMAP INTERNALDATE (``26-Sep-2026 10:00:00 +0000``) as a server
163 timestamp, or "". Parsed by hand rather than with strptime's %b, which
164 follows the process locale."""
165 match = re.fullmatch(
166 r"\s*(\d{1,2})-([A-Za-z]{3})-(\d{4}) (\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2})\s*",
167 str(value or ""))
168 if not match:
169 return ""
170 day, month, year, hour, minute, second, sign, off_h, off_m = match.groups()
171 try:
172 month_number = [m.lower() for m in _MONTHS].index(month.lower()) + 1
173 offset = timedelta(hours=int(off_h), minutes=int(off_m))
174 zone = timezone(offset if sign == "+" else -offset)
175 moment = datetime(int(year), month_number, int(day),
176 int(hour), int(minute), int(second), tzinfo=zone)
177 except ValueError:
178 return ""
179 return format_timestamp(moment)
180
181
182def imap_date(moment: datetime) -> str:
183 """A date as IMAP SEARCH wants it (``26-Sep-2026``), locale-free."""
184 return f"{moment.day}-{_MONTHS[moment.month - 1]}-{moment.year}"
185
186
187def extract_message_ids(value: str) -> list[str]:
188 """Every ``<id>`` in a header, in order, without repeats."""
189 seen: list[str] = []
190 for found in _MSGID_RE.findall(clean_text(value)):
191 if found not in seen and len(found) <= MAX_MESSAGE_ID:
192 seen.append(found)
193 return seen
194
195
196def stable_id(*parts: Any) -> str:
197 """A deterministic record id for the same facts.
198
199 Every id the sync writes is minted here from the owner, the account and
200 the message's dedupe key. That is what makes a replayed batch safe: the
201 second attempt at the same message produces the same id, the create is
202 refused as a duplicate, and nothing is written twice. It is also the
203 tenant boundary for ids -- the same Message-ID in two people's
204 mailboxes yields two different ids, because the owner is part of the
205 name.
206 """
207 name = "\x1f".join(str(part) for part in parts)
208 return str(uuid.uuid5(_ID_NAMESPACE, name))
209
210
211# --- HTML to text ---------------------------------------------------------------
212
213class _TextExtractor(html.parser.HTMLParser):
214 _SKIP = {"script", "style", "head", "title", "template", "noscript"}
215 _BREAK = {"br", "p", "div", "tr", "li", "h1", "h2", "h3", "h4", "h5", "h6",
216 "blockquote", "pre", "table", "ul", "ol", "hr", "section", "article"}
217
218 def __init__(self) -> None:
219 super().__init__(convert_charrefs=True)
220 self.parts: list[str] = []
221 self.skipping = 0
222
223 def handle_starttag(self, tag, attrs):
224 if tag in self._SKIP:
225 self.skipping += 1
226 elif tag in self._BREAK:
227 self.parts.append("\n")
228 if tag == "li":
229 self.parts.append("- ")
230
231 def handle_endtag(self, tag):
232 if tag in self._SKIP:
233 self.skipping = max(0, self.skipping - 1)
234 elif tag in self._BREAK:
235 self.parts.append("\n")
236
237 def handle_data(self, data):
238 if not self.skipping:
239 self.parts.append(data)
240
241
242def html_to_text(markup: str) -> str:
243 """Readable text from an HTML body: scripts and styles dropped, blocks
244 as line breaks, entities decoded, runs of blank lines collapsed. No
245 link is followed and no image is loaded; this only reads the string."""
246 parser = _TextExtractor()
247 try:
248 parser.feed(markup)
249 parser.close()
250 except Exception: # noqa: BLE001 -- html.parser is lenient; a crash still leaves text
251 return clean_text(re.sub(r"<[^>]*>", " ", html.unescape(markup)))
252 lines = [" ".join(line.split()) for line in "".join(parser.parts).split("\n")]
253 text = "\n".join(lines).strip()
254 return _BLANK_LINES_RE.sub("\n\n", text)
255
256
257# --- parsing one message ---------------------------------------------------------
258
259def _raw_header(message, name: str) -> list[str]:
260 """Header values the slow, forgiving way: the raw text, decoded with
261 email.header. Used when the policy's structured parse raises."""
262 values = []
263 for key, value in message.raw_items():
264 if key.lower() != name.lower():
265 continue
266 try:
267 values.append(str(email.header.make_header(email.header.decode_header(str(value)))))
268 except Exception: # noqa: BLE001 -- keep the undecoded text rather than nothing
269 values.append(str(value))
270 return values
271
272
273def _header_values(message, name: str) -> list:
274 """Every value of one header, parsed by the default policy (which is
275 where a malformed header raises), else decoded from the raw text."""
276 try:
277 return list(message.get_all(name) or [])
278 except Exception: # noqa: BLE001 -- fall back to the raw text, never drop the header
279 return _raw_header(message, name)
280
281
282def header_text(message, name: str) -> str:
283 values = _header_values(message, name)
284 return one_line(values[0]) if values else ""
285
286
287def header_addresses(message, name: str) -> list[tuple[str, str]]:
288 """(display name, address) pairs from every instance of one header."""
289 pairs: list[tuple[str, str]] = []
290 for value in _header_values(message, name):
291 found = None
292 try:
293 found = [(a.display_name, a.addr_spec) for a in value.addresses]
294 except Exception: # noqa: BLE001 -- not a structured header, or it would not parse
295 found = None
296 if found is None:
297 found = email.utils.getaddresses([str(value)])
298 for display, address in found:
299 address = one_line(address).strip("<>")
300 if "@" in address and len(address) <= MAX_ADDRESS:
301 pairs.append((one_line(display), address))
302 return pairs
303
304
305def _decode_part(part) -> str:
306 try:
307 return part.get_content()
308 except (LookupError, UnicodeError, AssertionError, ValueError):
309 payload = part.get_payload(decode=True) or b""
310 charset = part.get_content_charset() or "utf-8"
311 try:
312 return payload.decode(charset, "replace")
313 except LookupError:
314 return payload.decode("utf-8", "replace")
315
316
317def _attachment_names(message) -> list[str]:
318 names = []
319 try:
320 parts = list(message.iter_attachments()) if message.is_multipart() else []
321 except Exception: # noqa: BLE001 -- a malformed tree still has a body worth keeping
322 parts = []
323 for part in parts:
324 try:
325 name = part.get_filename() or part.get_content_type()
326 except Exception: # noqa: BLE001
327 name = "attachment"
328 names.append(one_line(name) or "attachment")
329 return names
330
331
332def body_text(message) -> str:
333 """The message's words: the text/plain body, else its HTML body as text,
334 with a closing line naming any attachments that were not copied."""
335 text = ""
336 try:
337 part = message.get_body(preferencelist=("plain", "html"))
338 except Exception: # noqa: BLE001
339 part = None
340 if part is not None:
341 content = _decode_part(part)
342 if part.get_content_type() == "text/html":
343 content = html_to_text(content)
344 text = clean_text(content).strip()
345 names = _attachment_names(message)
346 if names:
347 note = "[Attachments not copied: " + ", ".join(names[:20])
348 note += f" and {len(names) - 20} more]" if len(names) > 20 else "]"
349 text = f"{text}\n\n{note}" if text else note
350 if len(text) > MAX_BODY_CHARS:
351 text = text[:MAX_BODY_CHARS] + (
352 f"\n\n[Cut at {MAX_BODY_CHARS} characters; the whole message is on the mail server.]")
353 return text
354
355
356def parse_flags(flags: Iterable[str] | str) -> list[str]:
357 if isinstance(flags, str):
358 flags = flags.split()
359 return [str(flag) for flag in flags if str(flag)]
360
361
362def parse_message(
363 raw: bytes,
364 *,
365 flags: Iterable[str] | str = (),
366 internaldate: str = "",
367 header_only_size: int | None = None,
368) -> dict[str, Any]:
369 """RFC 5322 bytes to the facts the rows need.
370
371 ``header_only_size`` says the object fetched only the header because
372 the message was too big to download; the body then says exactly that,
373 with the size, rather than looking empty.
374 """
375 message = email.message_from_bytes(raw, policy=email.policy.default)
376 senders = header_addresses(message, "From")
377 from_name, from_address = senders[0] if senders else ("", "")
378
379 message_ids = extract_message_ids(header_text(message, "Message-ID"))
380 message_id = message_ids[0] if message_ids else ""
381 if not message_id:
382 bare = header_text(message, "Message-ID").strip()
383 if bare and " " not in bare and len(bare) + 2 <= MAX_MESSAGE_ID:
384 message_id = f"<{bare}>"
385 replies = extract_message_ids(header_text(message, "In-Reply-To"))
386 references = extract_message_ids(" ".join(
387 one_line(value) for value in _header_values(message, "References")))
388
389 if header_only_size is not None:
390 megabytes = header_only_size / (1024 * 1024)
391 body = (f"[This message is {megabytes:.1f} MB, larger than this server "
392 f"downloads, so only its headers were copied. Open it in your "
393 f"mail program to read it.]")
394 else:
395 body = body_text(message)
396
397 flag_list = parse_flags(flags)
398 date_raw = header_text(message, "Date")
399 return {
400 "message_id": message_id,
401 "in_reply_to": replies[0] if replies else "",
402 "references": references,
403 "subject": header_text(message, "Subject"),
404 "from_name": from_name,
405 "from_address": from_address,
406 "to": [address for _, address in header_addresses(message, "To")],
407 "cc": [address for _, address in header_addresses(message, "Cc")],
408 "bcc": [address for _, address in header_addresses(message, "Bcc")],
409 "date_raw": date_raw,
410 "date": parse_date_header(date_raw),
411 "internaldate": parse_internaldate(internaldate),
412 "body_text": body,
413 "flags": flag_list,
414 "is_read": any(flag.lower() == "\\seen" for flag in flag_list),
415 "list_unsubscribe": cap(header_text(message, "List-Unsubscribe"), MAX_HEADER_VALUE),
416 "list_unsubscribe_post": cap(header_text(message, "List-Unsubscribe-Post"), MAX_HEADER_VALUE),
417 "list_id": cap(header_text(message, "List-Id"), MAX_HEADER_VALUE),
418 "precedence": cap(header_text(message, "Precedence"), 100),
419 }
420
421
422def dedupe_key(parsed: Mapping[str, Any]) -> str:
423 """What makes two copies the same message: its Message-ID, or, for the
424 rare message without one, a hash of who sent it, when they said, the
425 subject and the start of the body. The key is compared only inside one
426 owner's one account (the object scopes the lookup, ``stable_id`` scopes
427 the ids), so it never needs to be globally unique."""
428 if parsed.get("message_id"):
429 return "mid:" + str(parsed["message_id"])
430 digest = hashlib.sha256("\x1f".join([
431 str(parsed.get("from_address") or ""),
432 str(parsed.get("date_raw") or ""),
433 str(parsed.get("subject") or ""),
434 str(parsed.get("body_text") or "")[:4096],
435 ]).encode("utf-8", "replace")).hexdigest()
436 return "sha256:" + digest
437
438
439def direction_for(parsed: Mapping[str, Any], *, account_address: str, folder_role: str) -> str:
440 """outbound for what this mailbox sent (the Sent folder, or From is the
441 account's own address), inbound for everything else."""
442 if folder_role == "sent":
443 return "outbound"
444 sender = str(parsed.get("from_address") or "").casefold()
445 if sender and sender == str(account_address or "").casefold():
446 return "outbound"
447 return "inbound"
448
449
450# --- rows ----------------------------------------------------------------------
451
452def _participants(parsed: Mapping[str, Any], account_address: str) -> list[str]:
453 own = str(account_address or "").casefold()
454 seen: list[str] = []
455 for address in [parsed.get("from_address") or ""] + list(parsed.get("to") or []) + list(parsed.get("cc") or []):
456 if address and address.casefold() != own and address.casefold() not in {s.casefold() for s in seen}:
457 seen.append(address)
458 return seen
459
460
461def participant_summary(existing: str, parsed: Mapping[str, Any], account_address: str) -> str:
462 names = [name.strip() for name in str(existing or "").split(",") if name.strip()]
463 for address in _participants(parsed, account_address):
464 if address.casefold() not in {name.casefold() for name in names}:
465 names.append(address)
466 return cap(", ".join(names), MAX_SUMMARY)
467
468
469def message_rows(
470 parsed: Mapping[str, Any],
471 *,
472 owner_id: str,
473 message_id: str,
474 thread_id: str,
475 direction: str,
476) -> tuple[dict[str, str], list[dict[str, str]]]:
477 """The ``messages`` row and its ``message_recipients`` rows."""
478 row = {
479 "id": message_id,
480 "thread_id": thread_id,
481 "direction": direction,
482 "from_address": parsed.get("from_address") or NO_SENDER,
483 "subject": cap(str(parsed.get("subject") or ""), MAX_SUBJECT),
484 "body_text": str(parsed.get("body_text") or ""),
485 "is_read": "true" if direction == "outbound" or parsed.get("is_read") else "false",
486 "in_reply_to": str(parsed.get("in_reply_to") or ""),
487 "external_message_id": str(parsed.get("message_id") or ""),
488 "owner_id": owner_id,
489 }
490 if direction == "outbound":
491 stamp = parsed.get("date") or parsed.get("internaldate")
492 if stamp:
493 row["sent_at"] = stamp
494 else:
495 stamp = parsed.get("internaldate") or parsed.get("date")
496 if stamp:
497 row["received_at"] = stamp
498
499 recipients = []
500 for kind in ("to", "cc", "bcc"):
501 seen: set[str] = set()
502 for address in parsed.get(kind) or []:
503 if address.casefold() in seen:
504 continue
505 seen.add(address.casefold())
506 recipients.append({
507 "id": stable_id("recipient", message_id, kind, address.casefold()),
508 "message_id": message_id,
509 "kind": kind,
510 "address": address,
511 "owner_id": owner_id,
512 })
513 return row, recipients
514
515
516def message_time(parsed: Mapping[str, Any], direction: str) -> str:
517 if direction == "outbound":
518 return parsed.get("date") or parsed.get("internaldate") or ""
519 return parsed.get("internaldate") or parsed.get("date") or ""
520
521
522def new_thread_row(
523 parsed: Mapping[str, Any], *, owner_id: str, thread_id: str,
524 direction: str, account_address: str,
525) -> dict[str, str]:
526 row = {
527 "id": thread_id,
528 "thread_type": "email",
529 "subject": cap(str(parsed.get("subject") or ""), MAX_SUBJECT),
530 "participant_summary": participant_summary("", parsed, account_address),
531 "message_count": "1",
532 "is_read": "true" if direction == "outbound" or parsed.get("is_read") else "false",
533 "owner_id": owner_id,
534 }
535 stamp = message_time(parsed, direction)
536 if stamp:
537 row["last_message_at"] = stamp
538 return row
539
540
541def thread_changes(
542 thread: Mapping[str, Any], parsed: Mapping[str, Any], *,
543 direction: str, account_address: str,
544) -> dict[str, str]:
545 """What one more message does to the thread it joined: the count goes up,
546 the latest time moves forward only (an old message arriving late does
547 not make a thread look fresh), an unread arrival makes the thread
548 unread, and new participants are added."""
549 try:
550 count = int(float(str(thread.get("message_count") or "0")))
551 except ValueError:
552 count = 0
553 changes = {"message_count": str(count + 1)}
554 stamp = message_time(parsed, direction)
555 current = parse_timestamp(thread.get("last_message_at"))
556 incoming = parse_timestamp(stamp)
557 if incoming is not None and (current is None or incoming > current):
558 changes["last_message_at"] = stamp
559 if direction != "outbound" and not parsed.get("is_read"):
560 changes["is_read"] = "false"
561 summary = participant_summary(str(thread.get("participant_summary") or ""), parsed, account_address)
562 if summary != str(thread.get("participant_summary") or ""):
563 changes["participant_summary"] = summary
564 return changes
565
566
567def synced_row(
568 parsed: Mapping[str, Any], *, row_id: str, owner_id: str, account_id: str,
569 folder: str, uid: int, uidvalidity: str, message_id: str, key: str,
570) -> dict[str, str]:
571 """The transport facts app-messaging deliberately does not carry."""
572 return {
573 "id": row_id,
574 "account_id": account_id,
575 "folder": folder,
576 "uid": str(uid),
577 "uidvalidity": str(uidvalidity),
578 "message_id": message_id,
579 "dedupe_key": key,
580 "references": cap(" ".join(parsed.get("references") or []), MAX_REFERENCES),
581 "list_unsubscribe": str(parsed.get("list_unsubscribe") or ""),
582 "list_unsubscribe_post": str(parsed.get("list_unsubscribe_post") or ""),
583 "list_id": str(parsed.get("list_id") or ""),
584 "precedence": str(parsed.get("precedence") or ""),
585 "owner_id": owner_id,
586 }
587
588
589# --- threading -----------------------------------------------------------------
590
591class ThreadIndex:
592 """Which thread a Message-ID belongs to, for ONE owner.
593
594 Built from that owner's rows only: a row whose owner_id is anyone
595 else's is skipped even if the caller passed it in, so a caller that
596 forgets to filter cannot make a thread leak across mailboxes.
597 """
598
599 def __init__(self, owner_id: str) -> None:
600 self.owner_id = owner_id
601 self.by_message_id: dict[str, str] = {}
602 self.by_parent: dict[str, str] = {}
603
604 @classmethod
605 def from_rows(
606 cls, owner_id: str, messages: Iterable[Mapping[str, Any]],
607 synced: Iterable[Mapping[str, Any]] = (),
608 ) -> "ThreadIndex":
609 index = cls(owner_id)
610 thread_of_message: dict[str, str] = {}
611 for row in messages:
612 if str(row.get("owner_id") or "") != owner_id or not row.get("thread_id"):
613 continue
614 thread_id = str(row["thread_id"])
615 thread_of_message[str(row.get("id") or "")] = thread_id
616 if row.get("external_message_id"):
617 index.by_message_id.setdefault(str(row["external_message_id"]), thread_id)
618 if row.get("in_reply_to"):
619 index.by_parent.setdefault(str(row["in_reply_to"]), thread_id)
620 for row in synced:
621 if str(row.get("owner_id") or "") != owner_id:
622 continue
623 thread_id = thread_of_message.get(str(row.get("message_id") or ""))
624 if not thread_id:
625 continue
626 for reference in str(row.get("references") or "").split():
627 index.by_parent.setdefault(reference, thread_id)
628 return index
629
630 def add(self, parsed: Mapping[str, Any], thread_id: str) -> None:
631 if parsed.get("message_id"):
632 self.by_message_id.setdefault(str(parsed["message_id"]), thread_id)
633 for parent in [parsed.get("in_reply_to")] + list(parsed.get("references") or []):
634 if parent:
635 self.by_parent.setdefault(str(parent), thread_id)
636
637
638def match_thread(parsed: Mapping[str, Any], index: ThreadIndex) -> str | None:
639 """The owner's existing thread this message belongs to, or None.
640
641 In order: the message it says it replies to; then its References, the
642 nearest ancestor first; then a reply to it that arrived first (Sent
643 and INBOX are separate folders, so a reply can be synced before its
644 parent). Never the subject: two people writing "Invoice" are not one
645 conversation.
646 """
647 candidates = [parsed.get("in_reply_to")] + list(reversed(parsed.get("references") or []))
648 for candidate in candidates:
649 if candidate and candidate in index.by_message_id:
650 return index.by_message_id[candidate]
651 own = parsed.get("message_id")
652 if own and own in index.by_parent:
653 return index.by_parent[own]
654 return None
655
656
657# --- cursor planning -------------------------------------------------------------
658
659def plan_folder(state: Mapping[str, Any] | None, *, uidvalidity: str) -> str:
660 """``fresh`` (never synced), ``reset`` (the server renumbered the folder:
661 UIDVALIDITY changed, so every stored UID is meaningless) or
662 ``continue``. A fresh or reset folder starts from a recent window, and
663 dedupe makes re-meeting old messages harmless."""
664 stored = str((state or {}).get("uidvalidity") or "").strip()
665 if not stored:
666 return "fresh"
667 if stored != str(uidvalidity).strip():
668 return "reset"
669 return "continue"
670
671
672def window_baseline(window_uids: Iterable[int], *, first_window: int, highest_uid: int) -> int:
673 """Where a fresh cursor starts: just below the oldest of the newest
674 ``first_window`` recent messages, so the first sync takes a window and
675 not the whole history. Nothing recent means start at the top: the next
676 message to arrive is the first one synced."""
677 recent = sorted({int(uid) for uid in window_uids})[-max(1, first_window):]
678 if not recent:
679 return max(0, int(highest_uid))
680 return recent[0] - 1
681
682
683def next_uids(found: Iterable[int], *, last_uid: int, limit: int) -> list[int]:
684 """The next UIDs to import, oldest first, at most ``limit``.
685
686 Filters ``> last_uid`` itself because ``UID SEARCH UID n:*`` answers
687 with the highest message even when n is beyond it -- a classic way for
688 a sync to import its newest message forever.
689 """
690 return sorted({int(uid) for uid in found if int(uid) > int(last_uid)})[: max(0, limit)]
691
692
693def plan_downloads(
694 metas: Iterable[Mapping[str, Any]], *, max_message_bytes: int, byte_budget: int,
695) -> list[tuple[int, str]]:
696 """(uid, "full" | "header") for this tick, oldest first.
697
698 A message over ``max_message_bytes`` gets its header only (the body
699 then says so). The byte budget bounds one tick's download because the
700 sync runs inside the daemon's loop; it always admits the first message,
701 so a big message cannot stall the folder, and it stops before the one
702 that would overrun, which the next tick picks up.
703 """
704 plan: list[tuple[int, str]] = []
705 spent = 0
706 for meta in sorted(metas, key=lambda m: int(m["uid"])):
707 size = int(meta.get("size") or 0)
708 mode = "full" if size <= max_message_bytes else "header"
709 cost = size if mode == "full" else 0
710 if plan and spent + cost > byte_budget:
711 break
712 plan.append((int(meta["uid"]), mode))
713 spent += cost
714 return plan
715
716
717# --- account scheduling -----------------------------------------------------------
718
719def account_due(
720 account: Mapping[str, Any], now: datetime, *,
721 claim_ttl_seconds: int, retry_base_seconds: int, retry_max_seconds: int,
722) -> tuple[bool, str]:
723 """(due, reason-not). Disabled accounts, accounts another pass is
724 working on, and failing accounts inside their backoff are not due.
725
726 Backoff because a wrong password tried every two minutes forever is
727 how a mail provider locks an account. It doubles from
728 ``retry_base_seconds`` and stops growing at ``retry_max_seconds``; the
729 owner can zero sync_attempts to try again at once.
730 """
731 if str(account.get("enabled") or "true").strip().lower() in {"false", "0", "no", "off"}:
732 return False, "disabled"
733 started = parse_timestamp(account.get("sync_started_at"))
734 finished = parse_timestamp(account.get("sync_finished_at"))
735 if started is not None and (finished is None or started > finished):
736 if (now - started).total_seconds() < claim_ttl_seconds:
737 return False, "another pass is syncing it"
738 try:
739 attempts = int(str(account.get("sync_attempts") or "0"))
740 except ValueError:
741 attempts = 0
742 if attempts > 0 and finished is not None:
743 try:
744 wait = retry_base_seconds * (2 ** (attempts - 1))
745 except OverflowError:
746 wait = retry_max_seconds
747 wait = min(wait, retry_max_seconds)
748 resume = finished + timedelta(seconds=wait)
749 if now < resume:
750 return False, f"waiting after {attempts} failed tries, until {format_timestamp(resume)}"
751 return True, ""
752
753
754def password_service(account_id: str) -> str:
755 """The identity-vault service name holding an account's password."""
756 return PASSWORD_SERVICE_PREFIX + str(account_id)
757
758
759def host_refusal(
760 host: str, addresses: Iterable[str], *, security: str, allow_private: bool,
761) -> str | None:
762 """Why the server must not connect to this host, or None.
763
764 The account's host is typed by a user, and the server is the one that
765 connects, so without this check any member could make the server knock
766 on addresses inside its own network (a database port on localhost, a
767 metadata service) and read reachability out of the error. Private,
768 loopback and link-local addresses are refused unless the operator set
769 DBBASIC_MAIL_SYNC_ALLOW_PRIVATE_HOSTS (a desk with its own mail server).
770 ``plain`` (no TLS) is allowed only to this machine itself: a password
771 sent in the clear over any real network is a password given away.
772 """
773 resolved = []
774 for address in addresses:
775 try:
776 resolved.append(ipaddress.ip_address(str(address).split("%", 1)[0]))
777 except ValueError:
778 return f"{host!r} resolved to {address!r}, which is not an address"
779 if not resolved:
780 return f"{host!r} did not resolve to any address"
781 if security == "plain" and not all(ip.is_loopback for ip in resolved):
782 return ("security 'plain' sends the password unencrypted, so it is only "
783 "allowed to this machine itself (127.0.0.1 or ::1); use ssl or starttls")
784 if not allow_private:
785 for ip in resolved:
786 if not ip.is_global:
787 return (f"{host!r} is a private or local address ({ip}); set "
788 f"{ALLOW_PRIVATE_HOSTS_ENV}=true if this server is meant to "
789 f"reach mail servers on its own network")
790 return None
791
792
793# --- reading IMAP answers ----------------------------------------------------------
794
795def _as_text(item: Any) -> str:
796 if isinstance(item, tuple):
797 item = item[0]
798 if isinstance(item, bytes):
799 return item.decode("utf-8", "replace")
800 return str(item or "")
801
802
803def parse_search(data: Iterable[Any]) -> list[int]:
804 """UIDs from a UID SEARCH answer."""
805 found: list[int] = []
806 for item in data or []:
807 for token in _as_text(item).split():
808 if token.isdigit():
809 found.append(int(token))
810 return found
811
812
813def parse_fetch_meta(data: Iterable[Any]) -> list[dict[str, Any]]:
814 """uid, flags, size and internaldate from a ``UID FETCH (UID FLAGS
815 RFC822.SIZE INTERNALDATE)`` answer, one dict per message."""
816 metas = []
817 for item in data or []:
818 text = _as_text(item)
819 uid = re.search(r"\bUID (\d+)", text)
820 if not uid:
821 continue
822 size = re.search(r"\bRFC822\.SIZE (\d+)", text)
823 flags = re.search(r"\bFLAGS \(([^)]*)\)", text)
824 internal = re.search(r'\bINTERNALDATE "([^"]+)"', text)
825 metas.append({
826 "uid": int(uid.group(1)),
827 "size": int(size.group(1)) if size else 0,
828 "flags": flags.group(1).split() if flags else [],
829 "internaldate": internal.group(1) if internal else "",
830 })
831 return metas
832
833
834def fetch_literal(data: Iterable[Any]) -> bytes | None:
835 """The message bytes from a ``UID FETCH (BODY.PEEK[...])`` answer."""
836 for item in data or []:
837 if isinstance(item, tuple) and len(item) >= 2 and "BODY[" in _as_text(item[0]).upper():
838 payload = item[1]
839 return payload if isinstance(payload, bytes) else str(payload).encode("utf-8")
840 return None
841
842
843_LIST_RE = re.compile(r'^\((?P<flags>[^)]*)\)\s+(?P<delim>"(?:[^"\\]|\\.)*"|NIL)\s+(?P<name>.+)$')
844_LIST_LITERAL_RE = re.compile(r'^\((?P<flags>[^)]*)\)\s+(?P<delim>"(?:[^"\\]|\\.)*"|NIL)\s+\{\d+\}$')
845
846
847def parse_list(data: Iterable[Any]) -> list[tuple[list[str], str]]:
848 """(attributes, mailbox name) from a LIST answer. Names stay in the
849 server's own (modified UTF-7) spelling, which is what SELECT needs."""
850 entries = []
851 for item in data or []:
852 if item is None:
853 continue
854 if isinstance(item, tuple):
855 # A name sent as a literal: the attributes line, then its bytes.
856 match = _LIST_LITERAL_RE.match(_as_text(item[0]).strip())
857 if match and len(item) >= 2:
858 entries.append((match.group("flags").split(), _as_text(item[1])))
859 continue
860 match = _LIST_RE.match(_as_text(item).strip())
861 if not match:
862 continue
863 name = match.group("name").strip()
864 if name.startswith('"') and name.endswith('"') and len(name) >= 2:
865 name = re.sub(r'\\(.)', r'\1', name[1:-1])
866 entries.append((match.group("flags").split(), name))
867 return entries
868
869
870_SENT_NAMES = ("sent", "sent items", "sent messages", "sent mail", "inbox.sent",
871 "[gmail]/sent mail")
872
873
874def choose_folders(entries: Iterable[tuple[list[str], str]]) -> list[tuple[str, str]]:
875 """(name, role) for a new account: INBOX, and the Sent folder when the
876 server has one -- by its \\Sent attribute (RFC 6154) first, then by the
877 names servers commonly use."""
878 entries = list(entries)
879 chosen = [("INBOX", "inbox")]
880 sent = next((name for attrs, name in entries
881 if any(a.lower() == "\\sent" for a in attrs)), None)
882 if sent is None:
883 sent = next((name for attrs, name in entries
884 if name.lower() in _SENT_NAMES
885 and not any(a.lower() == "\\noselect" for a in attrs)), None)
886 if sent and sent.upper() != "INBOX":
887 chosen.append((sent, "sent"))
888 return chosen
889
890
891def folder_role(name: str, entries: Iterable[tuple[list[str], str]]) -> str:
892 """inbox, sent or other, for a folder the owner named themselves."""
893 if str(name).upper() == "INBOX":
894 return "inbox"
895 for attrs, listed in entries:
896 if listed == name and any(a.lower() == "\\sent" for a in attrs):
897 return "sent"
898 return "sent" if str(name).lower() in _SENT_NAMES else "other"
899
900
901def split_folders(value: str) -> list[str]:
902 """The account's folders field: names separated by commas."""
903 names = []
904 for name in str(value or "").split(","):
905 name = name.strip()
906 if name and name not in names:
907 names.append(name)
908 return names
909
910
911def quote_mailbox(name: str) -> str:
912 """A mailbox name as an IMAP quoted string (imaplib does not quote).
913
914 A line break inside a quoted string would end the command and let the
915 rest of the name run as a second one on the person's mailbox (a
916 DELETE, an EXPUNGE), so a name carrying one is refused, not quoted.
917 """
918 name = str(name)
919 if "\r" in name or "\n" in name or "\x00" in name:
920 raise ValueError("a mailbox name cannot contain a line break")
921 return '"' + name.replace("\\", "\\\\").replace('"', '\\"') + '"'