askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
object_email.py · 15.7 KB · 382 lines
| 1 | """01 email adapter -- one outbound-mail queue every block enqueues into. |
| 2 | |
| 3 | A block that needs to send mail (submit confirmations, notify's email |
| 4 | channel, invoice delivery, membership invites, rollup digests) calls |
| 5 | ``enqueue()`` -- a plain in-process write of one ``queued`` row into the |
| 6 | ``email_outbox`` collection -- and returns. It never touches ``smtplib``, |
| 7 | never blocks on the network, and never loses the message on an exception. |
| 8 | The daemon's ``process_email_outbox`` pass drains the queue: it renders |
| 9 | each ``queued`` row that's due, hands it to the configured transport, and |
| 10 | records the outcome back onto the row (``sent`` / retry / ``dead``). See |
| 11 | plan/vocabulary/01-email-adapter-spec.md. |
| 12 | |
| 13 | Design decisions pinned here (the spec's Open Questions, resolved): |
| 14 | |
| 15 | * **States (spec's "State naming" question).** Exactly three: |
| 16 | ``queued`` (pending -- including waiting on a backoff between retries), |
| 17 | ``sent`` (terminal success), ``dead`` (terminal failure: retries |
| 18 | exhausted, or a permanent send-time error like a refused recipient). |
| 19 | There is no ``sending`` state (delivery is at-least-once, so a crash |
| 20 | mid-send simply retries -- an intermediate marker buys nothing) and no |
| 21 | standalone ``failed`` (a retryable error stays ``queued`` with an |
| 22 | advanced ``next_attempt_at``; a non-retryable one goes straight to |
| 23 | ``dead``). |
| 24 | * **Transport modes.** ``DBBASIC_SMTP_MODE`` = ``disabled`` (default -- |
| 25 | everything queues, nothing sends, fully inspectable), ``log`` (write to |
| 26 | a log file instead of the network -- dev/CI), or ``live`` (real SMTP). |
| 27 | * **max_attempts** is captured onto the row at enqueue time; **from_addr** |
| 28 | is resolved lazily at send time from ``DBBASIC_SMTP_FROM`` when the row |
| 29 | left it blank (so fixing a wrong default From doesn't require draining |
| 30 | the backlog). |
| 31 | * **Packaging.** The schema + permission fragment ship as the ``app-email`` |
| 32 | package (no pages); the daemon pass and the ``smtplib`` call live in |
| 33 | core (here + object_daemon) -- there is no manifest mechanism for a |
| 34 | package to register a daemon pass yet. |
| 35 | |
| 36 | The module is transport-agnostic and side-effect-injectable: ``enqueue`` |
| 37 | and the pure delivery decision (``plan_delivery``) are unit-testable |
| 38 | without a network, and ``attempt_delivery`` takes a ``sender`` callable so |
| 39 | tests drive success/permanent/transient outcomes directly. |
| 40 | """ |
| 41 | |
| 42 | from __future__ import annotations |
| 43 | |
| 44 | import json |
| 45 | import os |
| 46 | import smtplib |
| 47 | from datetime import datetime, timedelta, timezone |
| 48 | from email.message import EmailMessage |
| 49 | from email.utils import parseaddr |
| 50 | from pathlib import Path |
| 51 | from typing import Any, Callable, Mapping |
| 52 | |
| 53 | import object_feature_flags |
| 54 | import object_records |
| 55 | |
| 56 | OUTBOX_COLLECTION = "email_outbox" |
| 57 | FEATURE_FLAGS_COLLECTION = "feature_flags" |
| 58 | EMAIL_ENABLED_FLAG = "email_enabled" |
| 59 | DEFAULT_ACTOR = "daemon:email" |
| 60 | |
| 61 | STATUS_QUEUED = "queued" |
| 62 | STATUS_SENT = "sent" |
| 63 | STATUS_DEAD = "dead" |
| 64 | |
| 65 | # SMTP exceptions that will never succeed on retry -- a refused/invalid |
| 66 | # recipient or sender, or an unsupported command. These skip the retry |
| 67 | # budget and go straight to `dead`; everything else (connection refused, |
| 68 | # timeout, transient 4xx) is retried with backoff up to max_attempts. |
| 69 | _PERMANENT_SMTP_ERRORS = ( |
| 70 | smtplib.SMTPRecipientsRefused, |
| 71 | smtplib.SMTPSenderRefused, |
| 72 | smtplib.SMTPNotSupportedError, |
| 73 | ) |
| 74 | |
| 75 | |
| 76 | # --- configuration --------------------------------------------------------- |
| 77 | |
| 78 | class SmtpConfig: |
| 79 | """Deploy-time SMTP configuration, read from ``DBBASIC_SMTP_*`` env vars |
| 80 | (the operator-owned env file, never a record or backup -- same write-only |
| 81 | posture object_service_keys uses). Plain attributes so tests can build one |
| 82 | without the environment.""" |
| 83 | |
| 84 | __slots__ = ( |
| 85 | "mode", "host", "port", "username", "password", "use_tls", "from_addr", |
| 86 | "timeout", "max_attempts", "retry_base", "retry_max", "rate_limit", |
| 87 | "batch_size", "log_path", |
| 88 | ) |
| 89 | |
| 90 | def __init__( |
| 91 | self, *, mode="disabled", host="", port=587, username="", password="", |
| 92 | use_tls=True, from_addr="", timeout=10, max_attempts=5, retry_base=30, |
| 93 | retry_max=3600, rate_limit=30, batch_size=10, |
| 94 | log_path="data/logs/email_outbox/log.tsv", |
| 95 | ): |
| 96 | self.mode = mode |
| 97 | self.host = host |
| 98 | self.port = port |
| 99 | self.username = username |
| 100 | self.password = password |
| 101 | self.use_tls = use_tls |
| 102 | self.from_addr = from_addr |
| 103 | self.timeout = timeout |
| 104 | self.max_attempts = max_attempts |
| 105 | self.retry_base = retry_base |
| 106 | self.retry_max = retry_max |
| 107 | self.rate_limit = rate_limit |
| 108 | self.batch_size = batch_size |
| 109 | self.log_path = log_path |
| 110 | |
| 111 | @property |
| 112 | def configured(self) -> bool: |
| 113 | """Whether the pass has anywhere to send. ``disabled`` never is; |
| 114 | ``live`` needs a host; ``log`` is always 'configured' (it writes a |
| 115 | file, needs no network).""" |
| 116 | if self.mode == "log": |
| 117 | return True |
| 118 | if self.mode == "live": |
| 119 | return bool(self.host) |
| 120 | return False |
| 121 | |
| 122 | |
| 123 | def _env_bool(env: Mapping[str, str], key: str, default: bool) -> bool: |
| 124 | raw = env.get(key) |
| 125 | if raw is None or raw == "": |
| 126 | return default |
| 127 | return raw.strip().lower() not in {"off", "false", "0", "no"} |
| 128 | |
| 129 | |
| 130 | def _env_int(env: Mapping[str, str], key: str, default: int) -> int: |
| 131 | try: |
| 132 | return int(str(env.get(key, "")).strip()) |
| 133 | except (TypeError, ValueError): |
| 134 | return default |
| 135 | |
| 136 | |
| 137 | def smtp_config_from_env(env: Mapping[str, str] | None = None) -> SmtpConfig: |
| 138 | env = os.environ if env is None else env |
| 139 | mode = (env.get("DBBASIC_SMTP_MODE") or "disabled").strip().lower() |
| 140 | if mode not in {"disabled", "log", "live"}: |
| 141 | mode = "disabled" |
| 142 | return SmtpConfig( |
| 143 | mode=mode, |
| 144 | host=(env.get("DBBASIC_SMTP_HOST") or "").strip(), |
| 145 | port=_env_int(env, "DBBASIC_SMTP_PORT", 587), |
| 146 | username=(env.get("DBBASIC_SMTP_USERNAME") or "").strip(), |
| 147 | password=env.get("DBBASIC_SMTP_PASSWORD") or "", |
| 148 | use_tls=_env_bool(env, "DBBASIC_SMTP_USE_TLS", True), |
| 149 | from_addr=(env.get("DBBASIC_SMTP_FROM") or "").strip(), |
| 150 | timeout=_env_int(env, "DBBASIC_SMTP_TIMEOUT_SECONDS", 10), |
| 151 | max_attempts=_env_int(env, "DBBASIC_SMTP_MAX_ATTEMPTS", 5), |
| 152 | retry_base=_env_int(env, "DBBASIC_SMTP_RETRY_BASE_SECONDS", 30), |
| 153 | retry_max=_env_int(env, "DBBASIC_SMTP_RETRY_MAX_SECONDS", 3600), |
| 154 | rate_limit=_env_int(env, "DBBASIC_SMTP_RATE_LIMIT_PER_MINUTE", 30), |
| 155 | batch_size=_env_int(env, "DBBASIC_SMTP_BATCH_SIZE", 10), |
| 156 | log_path=(env.get("DBBASIC_SMTP_LOG_PATH") or "data/logs/email_outbox/log.tsv").strip(), |
| 157 | ) |
| 158 | |
| 159 | |
| 160 | def email_pass_enabled(*, base_dir: Any) -> bool: |
| 161 | """Brownout kill switch -- a ``feature_flags`` row named ``email_enabled``, |
| 162 | default ON. This is the runtime lever an operator flips to stop outbound |
| 163 | network calls without a redeploy; it is NOT the "is SMTP configured" gate |
| 164 | (that's the env, checked separately). Mirrors notify_pass_enabled.""" |
| 165 | return object_feature_flags.kill_switch_on(EMAIL_ENABLED_FLAG, base_dir=base_dir) |
| 166 | |
| 167 | |
| 168 | # --- time helpers ---------------------------------------------------------- |
| 169 | |
| 170 | def _now_iso(now: datetime | None = None) -> str: |
| 171 | now = datetime.now(timezone.utc) if now is None else now |
| 172 | return now.isoformat().replace("+00:00", "Z") |
| 173 | |
| 174 | |
| 175 | def _parse_iso(value: str) -> datetime | None: |
| 176 | raw = (value or "").strip() |
| 177 | if not raw: |
| 178 | return None |
| 179 | try: |
| 180 | return datetime.fromisoformat(raw.replace("Z", "+00:00")) |
| 181 | except ValueError: |
| 182 | return None |
| 183 | |
| 184 | |
| 185 | # --- enqueue (the in-process call surface) --------------------------------- |
| 186 | |
| 187 | def enqueue( |
| 188 | to: str, subject: str, text_body: str, *, |
| 189 | html_body: str | None = None, from_addr: str | None = None, |
| 190 | reply_to: str | None = None, source_object_id: str | None = None, |
| 191 | extra: Any = None, base_dir: Any, config: SmtpConfig | None = None, |
| 192 | now: datetime | None = None, |
| 193 | ) -> str: |
| 194 | """Write one ``queued`` row and return its id. A pure local write: no |
| 195 | rendering, no network, no blocking on delivery. ``max_attempts`` is |
| 196 | captured now (a later env change won't retroactively shrink an in-flight |
| 197 | message's budget); ``from_addr`` left blank is resolved lazily at send. |
| 198 | ``extra`` may be a dict/list (JSON-encoded) or a string, stashed verbatim |
| 199 | for the caller's own correlation id -- the adapter never reads it.""" |
| 200 | config = smtp_config_from_env() if config is None else config |
| 201 | stamp = _now_iso(now) |
| 202 | record = { |
| 203 | "to": to, |
| 204 | "from_addr": from_addr or "", |
| 205 | "reply_to": reply_to or "", |
| 206 | "subject": subject, |
| 207 | "text_body": text_body, |
| 208 | "html_body": html_body or "", |
| 209 | "status": STATUS_QUEUED, |
| 210 | "attempts": "0", |
| 211 | "max_attempts": str(config.max_attempts), |
| 212 | "last_error": "", |
| 213 | "next_attempt_at": stamp, |
| 214 | "sent_at": "", |
| 215 | "source_object_id": source_object_id or "", |
| 216 | } |
| 217 | # `extra` is the overflow field (store: extra) -- a JSON object only. Pass a |
| 218 | # dict, or a JSON-object string; omit it entirely when there's nothing (an |
| 219 | # empty string is not a valid object and the store would reject it). |
| 220 | if isinstance(extra, dict): |
| 221 | record["extra"] = json.dumps(extra, sort_keys=True) |
| 222 | elif isinstance(extra, str) and extra.strip(): |
| 223 | record["extra"] = extra |
| 224 | created = object_records.create_collection_record( |
| 225 | OUTBOX_COLLECTION, record, base_dir=base_dir, actor=DEFAULT_ACTOR, |
| 226 | ) |
| 227 | return created["id"] |
| 228 | |
| 229 | |
| 230 | # --- delivery decision (pure) ---------------------------------------------- |
| 231 | |
| 232 | def is_due(record: Mapping[str, Any], now: datetime | None = None) -> bool: |
| 233 | """A row the pass should attempt this tick: still ``queued`` and its |
| 234 | ``next_attempt_at`` (blank = immediately) has arrived.""" |
| 235 | if (record.get("status") or STATUS_QUEUED) != STATUS_QUEUED: |
| 236 | return False |
| 237 | when = _parse_iso(record.get("next_attempt_at") or "") |
| 238 | if when is None: |
| 239 | return True |
| 240 | now = datetime.now(timezone.utc) if now is None else now |
| 241 | return when <= now |
| 242 | |
| 243 | |
| 244 | def backoff_seconds(attempts: int, config: SmtpConfig) -> int: |
| 245 | """``retry_base * 2 ** (attempts - 1)`` capped at ``retry_max`` -- the same |
| 246 | exponential shape process_queue uses. ``attempts`` is the count AFTER this |
| 247 | failed try (>= 1).""" |
| 248 | exp = max(0, attempts - 1) |
| 249 | try: |
| 250 | delay = config.retry_base * (2 ** exp) |
| 251 | except OverflowError: |
| 252 | delay = config.retry_max |
| 253 | return int(min(delay, config.retry_max)) |
| 254 | |
| 255 | |
| 256 | def plan_delivery( |
| 257 | record: Mapping[str, Any], config: SmtpConfig, *, |
| 258 | ok: bool, permanent: bool = False, error: str = "", |
| 259 | now: datetime | None = None, |
| 260 | ) -> dict[str, str]: |
| 261 | """The record update a delivery outcome implies -- pure, no IO. ``ok``: |
| 262 | marked ``sent``. Not ok + ``permanent`` (or retries now exhausted): marked |
| 263 | ``dead``. Otherwise: stays ``queued`` with an advanced ``next_attempt_at`` |
| 264 | and the error captured.""" |
| 265 | now = datetime.now(timezone.utc) if now is None else now |
| 266 | stamp = _now_iso(now) |
| 267 | if ok: |
| 268 | return {"status": STATUS_SENT, "sent_at": stamp, "last_error": "", "updated_at": stamp} |
| 269 | |
| 270 | attempts = _env_int(record, "attempts", 0) + 1 |
| 271 | try: |
| 272 | max_attempts = int(str(record.get("max_attempts") or config.max_attempts)) |
| 273 | except (TypeError, ValueError): |
| 274 | max_attempts = config.max_attempts |
| 275 | update = {"attempts": str(attempts), "last_error": (error or "")[:500], "updated_at": stamp} |
| 276 | if permanent or attempts >= max_attempts: |
| 277 | update["status"] = STATUS_DEAD |
| 278 | else: |
| 279 | nxt = now + timedelta(seconds=backoff_seconds(attempts, config)) |
| 280 | update["status"] = STATUS_QUEUED |
| 281 | update["next_attempt_at"] = _now_iso(nxt) |
| 282 | return update |
| 283 | |
| 284 | |
| 285 | # --- transports ------------------------------------------------------------ |
| 286 | |
| 287 | def build_message(record: Mapping[str, Any], config: SmtpConfig) -> EmailMessage: |
| 288 | """Render one outbox row into an EmailMessage. ``from_addr`` falls back to |
| 289 | ``DBBASIC_SMTP_FROM`` lazily here (not captured at enqueue).""" |
| 290 | msg = EmailMessage() |
| 291 | msg["To"] = record.get("to") or "" |
| 292 | msg["From"] = (record.get("from_addr") or "").strip() or config.from_addr |
| 293 | msg["Subject"] = record.get("subject") or "" |
| 294 | reply_to = (record.get("reply_to") or "").strip() |
| 295 | if reply_to: |
| 296 | msg["Reply-To"] = reply_to |
| 297 | msg.set_content(record.get("text_body") or "") |
| 298 | html = record.get("html_body") or "" |
| 299 | if html: |
| 300 | msg.add_alternative(html, subtype="html") |
| 301 | return msg |
| 302 | |
| 303 | |
| 304 | def _send_live(config: SmtpConfig, record: Mapping[str, Any]) -> None: |
| 305 | """Real SMTP send. Raises on any failure (the caller classifies it).""" |
| 306 | msg = build_message(record, config) |
| 307 | sender = parseaddr(msg["From"])[1] |
| 308 | recipients = [addr for _, addr in [parseaddr(r) for r in (record.get("to") or "").split(",")] if addr] |
| 309 | if not recipients: |
| 310 | raise smtplib.SMTPRecipientsRefused({}) |
| 311 | with smtplib.SMTP(config.host, config.port, timeout=config.timeout) as server: |
| 312 | if config.use_tls: |
| 313 | server.starttls() |
| 314 | if config.username: |
| 315 | server.login(config.username, config.password) |
| 316 | server.send_message(msg, from_addr=sender or None, to_addrs=recipients) |
| 317 | |
| 318 | |
| 319 | def _send_log(config: SmtpConfig, record: Mapping[str, Any]) -> None: |
| 320 | """`log` mode: append a one-line record to the log file instead of the |
| 321 | network -- dev/CI verification that enqueue -> drain works end to end |
| 322 | without an SMTP server. Tab-separated, newlines stripped from the fields.""" |
| 323 | path = Path(config.log_path) |
| 324 | path.parent.mkdir(parents=True, exist_ok=True) |
| 325 | from_addr = (record.get("from_addr") or "").strip() or config.from_addr |
| 326 | |
| 327 | def _clean(value: Any) -> str: |
| 328 | return str(value or "").replace("\t", " ").replace("\n", " ").replace("\r", " ") |
| 329 | |
| 330 | line = "\t".join(_clean(v) for v in ( |
| 331 | _now_iso(), record.get("to"), from_addr, record.get("subject"), |
| 332 | record.get("text_body"), |
| 333 | )) |
| 334 | with path.open("a", encoding="utf-8") as fh: |
| 335 | fh.write(line + "\n") |
| 336 | |
| 337 | |
| 338 | def sender_for(config: SmtpConfig) -> Callable[[SmtpConfig, Mapping[str, Any]], None]: |
| 339 | """The transport callable for the configured mode.""" |
| 340 | if config.mode == "live": |
| 341 | return _send_live |
| 342 | if config.mode == "log": |
| 343 | return _send_log |
| 344 | raise ValueError(f"no transport for mode {config.mode!r}") |
| 345 | |
| 346 | |
| 347 | def attempt_delivery( |
| 348 | record: Mapping[str, Any], config: SmtpConfig, *, |
| 349 | sender: Callable[[SmtpConfig, Mapping[str, Any]], None] | None = None, |
| 350 | now: datetime | None = None, |
| 351 | ) -> dict[str, str]: |
| 352 | """Try to deliver one row and return the record update its outcome implies. |
| 353 | ``sender`` is injectable (tests drive success/permanent/transient without a |
| 354 | network); it defaults to the configured transport. A permanent SMTP error |
| 355 | (refused recipient/sender) is marked ``dead`` immediately; any other |
| 356 | exception is a transient failure and retries with backoff.""" |
| 357 | sender = sender_for(config) if sender is None else sender |
| 358 | try: |
| 359 | sender(config, record) |
| 360 | except _PERMANENT_SMTP_ERRORS as exc: |
| 361 | return plan_delivery(record, config, ok=False, permanent=True, error=str(exc), now=now) |
| 362 | except Exception as exc: # noqa: BLE001 -- transient: retry with backoff |
| 363 | return plan_delivery(record, config, ok=False, error=str(exc), now=now) |
| 364 | return plan_delivery(record, config, ok=True, now=now) |
| 365 | |
| 366 | |
| 367 | # --- rate limiter (rolling per-minute window) ------------------------------ |
| 368 | |
| 369 | def rate_window_reset(window: Mapping[str, Any] | None, now_ts: float) -> dict[str, float]: |
| 370 | """Fold the persisted ``{window_start, sent_count}`` marker: a window older |
| 371 | than 60s rolls over to a fresh one. Pure -- the daemon owns the file IO.""" |
| 372 | start = 0.0 |
| 373 | count = 0 |
| 374 | if isinstance(window, Mapping): |
| 375 | try: |
| 376 | start = float(window.get("window_start") or 0) |
| 377 | count = int(window.get("sent_count") or 0) |
| 378 | except (TypeError, ValueError): |
| 379 | start, count = 0.0, 0 |
| 380 | if now_ts - start >= 60: |
| 381 | return {"window_start": now_ts, "sent_count": 0} |
| 382 | return {"window_start": start, "sent_count": count} |