Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/object_email.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
object_email.py · 15.7 KB · 382 lines
1"""01 email adapter -- one outbound-mail queue every block enqueues into.
2
3A block that needs to send mail (submit confirmations, notify's email
4channel, invoice delivery, membership invites, rollup digests) calls
5``enqueue()`` -- a plain in-process write of one ``queued`` row into the
6``email_outbox`` collection -- and returns. It never touches ``smtplib``,
7never blocks on the network, and never loses the message on an exception.
8The daemon's ``process_email_outbox`` pass drains the queue: it renders
9each ``queued`` row that's due, hands it to the configured transport, and
10records the outcome back onto the row (``sent`` / retry / ``dead``). See
11plan/vocabulary/01-email-adapter-spec.md.
12
13Design decisions pinned here (the spec's Open Questions, resolved):
14
15* **States (spec's "State naming" question).** Exactly three:
16 ``queued`` (pending -- including waiting on a backoff between retries),
17 ``sent`` (terminal success), ``dead`` (terminal failure: retries
18 exhausted, or a permanent send-time error like a refused recipient).
19 There is no ``sending`` state (delivery is at-least-once, so a crash
20 mid-send simply retries -- an intermediate marker buys nothing) and no
21 standalone ``failed`` (a retryable error stays ``queued`` with an
22 advanced ``next_attempt_at``; a non-retryable one goes straight to
23 ``dead``).
24* **Transport modes.** ``DBBASIC_SMTP_MODE`` = ``disabled`` (default --
25 everything queues, nothing sends, fully inspectable), ``log`` (write to
26 a log file instead of the network -- dev/CI), or ``live`` (real SMTP).
27* **max_attempts** is captured onto the row at enqueue time; **from_addr**
28 is resolved lazily at send time from ``DBBASIC_SMTP_FROM`` when the row
29 left it blank (so fixing a wrong default From doesn't require draining
30 the backlog).
31* **Packaging.** The schema + permission fragment ship as the ``app-email``
32 package (no pages); the daemon pass and the ``smtplib`` call live in
33 core (here + object_daemon) -- there is no manifest mechanism for a
34 package to register a daemon pass yet.
35
36The module is transport-agnostic and side-effect-injectable: ``enqueue``
37and the pure delivery decision (``plan_delivery``) are unit-testable
38without a network, and ``attempt_delivery`` takes a ``sender`` callable so
39tests drive success/permanent/transient outcomes directly.
40"""
41
42from __future__ import annotations
43
44import json
45import os
46import smtplib
47from datetime import datetime, timedelta, timezone
48from email.message import EmailMessage
49from email.utils import parseaddr
50from pathlib import Path
51from typing import Any, Callable, Mapping
52
53import object_feature_flags
54import object_records
55
56OUTBOX_COLLECTION = "email_outbox"
57FEATURE_FLAGS_COLLECTION = "feature_flags"
58EMAIL_ENABLED_FLAG = "email_enabled"
59DEFAULT_ACTOR = "daemon:email"
60
61STATUS_QUEUED = "queued"
62STATUS_SENT = "sent"
63STATUS_DEAD = "dead"
64
65# SMTP exceptions that will never succeed on retry -- a refused/invalid
66# recipient or sender, or an unsupported command. These skip the retry
67# budget and go straight to `dead`; everything else (connection refused,
68# timeout, transient 4xx) is retried with backoff up to max_attempts.
69_PERMANENT_SMTP_ERRORS = (
70 smtplib.SMTPRecipientsRefused,
71 smtplib.SMTPSenderRefused,
72 smtplib.SMTPNotSupportedError,
73)
74
75
76# --- configuration ---------------------------------------------------------
77
78class SmtpConfig:
79 """Deploy-time SMTP configuration, read from ``DBBASIC_SMTP_*`` env vars
80 (the operator-owned env file, never a record or backup -- same write-only
81 posture object_service_keys uses). Plain attributes so tests can build one
82 without the environment."""
83
84 __slots__ = (
85 "mode", "host", "port", "username", "password", "use_tls", "from_addr",
86 "timeout", "max_attempts", "retry_base", "retry_max", "rate_limit",
87 "batch_size", "log_path",
88 )
89
90 def __init__(
91 self, *, mode="disabled", host="", port=587, username="", password="",
92 use_tls=True, from_addr="", timeout=10, max_attempts=5, retry_base=30,
93 retry_max=3600, rate_limit=30, batch_size=10,
94 log_path="data/logs/email_outbox/log.tsv",
95 ):
96 self.mode = mode
97 self.host = host
98 self.port = port
99 self.username = username
100 self.password = password
101 self.use_tls = use_tls
102 self.from_addr = from_addr
103 self.timeout = timeout
104 self.max_attempts = max_attempts
105 self.retry_base = retry_base
106 self.retry_max = retry_max
107 self.rate_limit = rate_limit
108 self.batch_size = batch_size
109 self.log_path = log_path
110
111 @property
112 def configured(self) -> bool:
113 """Whether the pass has anywhere to send. ``disabled`` never is;
114 ``live`` needs a host; ``log`` is always 'configured' (it writes a
115 file, needs no network)."""
116 if self.mode == "log":
117 return True
118 if self.mode == "live":
119 return bool(self.host)
120 return False
121
122
123def _env_bool(env: Mapping[str, str], key: str, default: bool) -> bool:
124 raw = env.get(key)
125 if raw is None or raw == "":
126 return default
127 return raw.strip().lower() not in {"off", "false", "0", "no"}
128
129
130def _env_int(env: Mapping[str, str], key: str, default: int) -> int:
131 try:
132 return int(str(env.get(key, "")).strip())
133 except (TypeError, ValueError):
134 return default
135
136
137def smtp_config_from_env(env: Mapping[str, str] | None = None) -> SmtpConfig:
138 env = os.environ if env is None else env
139 mode = (env.get("DBBASIC_SMTP_MODE") or "disabled").strip().lower()
140 if mode not in {"disabled", "log", "live"}:
141 mode = "disabled"
142 return SmtpConfig(
143 mode=mode,
144 host=(env.get("DBBASIC_SMTP_HOST") or "").strip(),
145 port=_env_int(env, "DBBASIC_SMTP_PORT", 587),
146 username=(env.get("DBBASIC_SMTP_USERNAME") or "").strip(),
147 password=env.get("DBBASIC_SMTP_PASSWORD") or "",
148 use_tls=_env_bool(env, "DBBASIC_SMTP_USE_TLS", True),
149 from_addr=(env.get("DBBASIC_SMTP_FROM") or "").strip(),
150 timeout=_env_int(env, "DBBASIC_SMTP_TIMEOUT_SECONDS", 10),
151 max_attempts=_env_int(env, "DBBASIC_SMTP_MAX_ATTEMPTS", 5),
152 retry_base=_env_int(env, "DBBASIC_SMTP_RETRY_BASE_SECONDS", 30),
153 retry_max=_env_int(env, "DBBASIC_SMTP_RETRY_MAX_SECONDS", 3600),
154 rate_limit=_env_int(env, "DBBASIC_SMTP_RATE_LIMIT_PER_MINUTE", 30),
155 batch_size=_env_int(env, "DBBASIC_SMTP_BATCH_SIZE", 10),
156 log_path=(env.get("DBBASIC_SMTP_LOG_PATH") or "data/logs/email_outbox/log.tsv").strip(),
157 )
158
159
160def email_pass_enabled(*, base_dir: Any) -> bool:
161 """Brownout kill switch -- a ``feature_flags`` row named ``email_enabled``,
162 default ON. This is the runtime lever an operator flips to stop outbound
163 network calls without a redeploy; it is NOT the "is SMTP configured" gate
164 (that's the env, checked separately). Mirrors notify_pass_enabled."""
165 return object_feature_flags.kill_switch_on(EMAIL_ENABLED_FLAG, base_dir=base_dir)
166
167
168# --- time helpers ----------------------------------------------------------
169
170def _now_iso(now: datetime | None = None) -> str:
171 now = datetime.now(timezone.utc) if now is None else now
172 return now.isoformat().replace("+00:00", "Z")
173
174
175def _parse_iso(value: str) -> datetime | None:
176 raw = (value or "").strip()
177 if not raw:
178 return None
179 try:
180 return datetime.fromisoformat(raw.replace("Z", "+00:00"))
181 except ValueError:
182 return None
183
184
185# --- enqueue (the in-process call surface) ---------------------------------
186
187def enqueue(
188 to: str, subject: str, text_body: str, *,
189 html_body: str | None = None, from_addr: str | None = None,
190 reply_to: str | None = None, source_object_id: str | None = None,
191 extra: Any = None, base_dir: Any, config: SmtpConfig | None = None,
192 now: datetime | None = None,
193) -> str:
194 """Write one ``queued`` row and return its id. A pure local write: no
195 rendering, no network, no blocking on delivery. ``max_attempts`` is
196 captured now (a later env change won't retroactively shrink an in-flight
197 message's budget); ``from_addr`` left blank is resolved lazily at send.
198 ``extra`` may be a dict/list (JSON-encoded) or a string, stashed verbatim
199 for the caller's own correlation id -- the adapter never reads it."""
200 config = smtp_config_from_env() if config is None else config
201 stamp = _now_iso(now)
202 record = {
203 "to": to,
204 "from_addr": from_addr or "",
205 "reply_to": reply_to or "",
206 "subject": subject,
207 "text_body": text_body,
208 "html_body": html_body or "",
209 "status": STATUS_QUEUED,
210 "attempts": "0",
211 "max_attempts": str(config.max_attempts),
212 "last_error": "",
213 "next_attempt_at": stamp,
214 "sent_at": "",
215 "source_object_id": source_object_id or "",
216 }
217 # `extra` is the overflow field (store: extra) -- a JSON object only. Pass a
218 # dict, or a JSON-object string; omit it entirely when there's nothing (an
219 # empty string is not a valid object and the store would reject it).
220 if isinstance(extra, dict):
221 record["extra"] = json.dumps(extra, sort_keys=True)
222 elif isinstance(extra, str) and extra.strip():
223 record["extra"] = extra
224 created = object_records.create_collection_record(
225 OUTBOX_COLLECTION, record, base_dir=base_dir, actor=DEFAULT_ACTOR,
226 )
227 return created["id"]
228
229
230# --- delivery decision (pure) ----------------------------------------------
231
232def is_due(record: Mapping[str, Any], now: datetime | None = None) -> bool:
233 """A row the pass should attempt this tick: still ``queued`` and its
234 ``next_attempt_at`` (blank = immediately) has arrived."""
235 if (record.get("status") or STATUS_QUEUED) != STATUS_QUEUED:
236 return False
237 when = _parse_iso(record.get("next_attempt_at") or "")
238 if when is None:
239 return True
240 now = datetime.now(timezone.utc) if now is None else now
241 return when <= now
242
243
244def backoff_seconds(attempts: int, config: SmtpConfig) -> int:
245 """``retry_base * 2 ** (attempts - 1)`` capped at ``retry_max`` -- the same
246 exponential shape process_queue uses. ``attempts`` is the count AFTER this
247 failed try (>= 1)."""
248 exp = max(0, attempts - 1)
249 try:
250 delay = config.retry_base * (2 ** exp)
251 except OverflowError:
252 delay = config.retry_max
253 return int(min(delay, config.retry_max))
254
255
256def plan_delivery(
257 record: Mapping[str, Any], config: SmtpConfig, *,
258 ok: bool, permanent: bool = False, error: str = "",
259 now: datetime | None = None,
260) -> dict[str, str]:
261 """The record update a delivery outcome implies -- pure, no IO. ``ok``:
262 marked ``sent``. Not ok + ``permanent`` (or retries now exhausted): marked
263 ``dead``. Otherwise: stays ``queued`` with an advanced ``next_attempt_at``
264 and the error captured."""
265 now = datetime.now(timezone.utc) if now is None else now
266 stamp = _now_iso(now)
267 if ok:
268 return {"status": STATUS_SENT, "sent_at": stamp, "last_error": "", "updated_at": stamp}
269
270 attempts = _env_int(record, "attempts", 0) + 1
271 try:
272 max_attempts = int(str(record.get("max_attempts") or config.max_attempts))
273 except (TypeError, ValueError):
274 max_attempts = config.max_attempts
275 update = {"attempts": str(attempts), "last_error": (error or "")[:500], "updated_at": stamp}
276 if permanent or attempts >= max_attempts:
277 update["status"] = STATUS_DEAD
278 else:
279 nxt = now + timedelta(seconds=backoff_seconds(attempts, config))
280 update["status"] = STATUS_QUEUED
281 update["next_attempt_at"] = _now_iso(nxt)
282 return update
283
284
285# --- transports ------------------------------------------------------------
286
287def build_message(record: Mapping[str, Any], config: SmtpConfig) -> EmailMessage:
288 """Render one outbox row into an EmailMessage. ``from_addr`` falls back to
289 ``DBBASIC_SMTP_FROM`` lazily here (not captured at enqueue)."""
290 msg = EmailMessage()
291 msg["To"] = record.get("to") or ""
292 msg["From"] = (record.get("from_addr") or "").strip() or config.from_addr
293 msg["Subject"] = record.get("subject") or ""
294 reply_to = (record.get("reply_to") or "").strip()
295 if reply_to:
296 msg["Reply-To"] = reply_to
297 msg.set_content(record.get("text_body") or "")
298 html = record.get("html_body") or ""
299 if html:
300 msg.add_alternative(html, subtype="html")
301 return msg
302
303
304def _send_live(config: SmtpConfig, record: Mapping[str, Any]) -> None:
305 """Real SMTP send. Raises on any failure (the caller classifies it)."""
306 msg = build_message(record, config)
307 sender = parseaddr(msg["From"])[1]
308 recipients = [addr for _, addr in [parseaddr(r) for r in (record.get("to") or "").split(",")] if addr]
309 if not recipients:
310 raise smtplib.SMTPRecipientsRefused({})
311 with smtplib.SMTP(config.host, config.port, timeout=config.timeout) as server:
312 if config.use_tls:
313 server.starttls()
314 if config.username:
315 server.login(config.username, config.password)
316 server.send_message(msg, from_addr=sender or None, to_addrs=recipients)
317
318
319def _send_log(config: SmtpConfig, record: Mapping[str, Any]) -> None:
320 """`log` mode: append a one-line record to the log file instead of the
321 network -- dev/CI verification that enqueue -> drain works end to end
322 without an SMTP server. Tab-separated, newlines stripped from the fields."""
323 path = Path(config.log_path)
324 path.parent.mkdir(parents=True, exist_ok=True)
325 from_addr = (record.get("from_addr") or "").strip() or config.from_addr
326
327 def _clean(value: Any) -> str:
328 return str(value or "").replace("\t", " ").replace("\n", " ").replace("\r", " ")
329
330 line = "\t".join(_clean(v) for v in (
331 _now_iso(), record.get("to"), from_addr, record.get("subject"),
332 record.get("text_body"),
333 ))
334 with path.open("a", encoding="utf-8") as fh:
335 fh.write(line + "\n")
336
337
338def sender_for(config: SmtpConfig) -> Callable[[SmtpConfig, Mapping[str, Any]], None]:
339 """The transport callable for the configured mode."""
340 if config.mode == "live":
341 return _send_live
342 if config.mode == "log":
343 return _send_log
344 raise ValueError(f"no transport for mode {config.mode!r}")
345
346
347def attempt_delivery(
348 record: Mapping[str, Any], config: SmtpConfig, *,
349 sender: Callable[[SmtpConfig, Mapping[str, Any]], None] | None = None,
350 now: datetime | None = None,
351) -> dict[str, str]:
352 """Try to deliver one row and return the record update its outcome implies.
353 ``sender`` is injectable (tests drive success/permanent/transient without a
354 network); it defaults to the configured transport. A permanent SMTP error
355 (refused recipient/sender) is marked ``dead`` immediately; any other
356 exception is a transient failure and retries with backoff."""
357 sender = sender_for(config) if sender is None else sender
358 try:
359 sender(config, record)
360 except _PERMANENT_SMTP_ERRORS as exc:
361 return plan_delivery(record, config, ok=False, permanent=True, error=str(exc), now=now)
362 except Exception as exc: # noqa: BLE001 -- transient: retry with backoff
363 return plan_delivery(record, config, ok=False, error=str(exc), now=now)
364 return plan_delivery(record, config, ok=True, now=now)
365
366
367# --- rate limiter (rolling per-minute window) ------------------------------
368
369def rate_window_reset(window: Mapping[str, Any] | None, now_ts: float) -> dict[str, float]:
370 """Fold the persisted ``{window_start, sent_count}`` marker: a window older
371 than 60s rolls over to a fresh one. Pure -- the daemon owns the file IO."""
372 start = 0.0
373 count = 0
374 if isinstance(window, Mapping):
375 try:
376 start = float(window.get("window_start") or 0)
377 count = int(window.get("sent_count") or 0)
378 except (TypeError, ValueError):
379 start, count = 0.0, 0
380 if now_ts - start >= 60:
381 return {"window_start": now_ts, "sent_count": 0}
382 return {"window_start": start, "sent_count": count}