askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_mail_sync.py · 31.4 KB · 744 lines
| 1 | """system_mail_sync against a fake IMAP server. |
| 2 | |
| 3 | The fake is injected where the real one would be -- imaplib.IMAP4_SSL and |
| 4 | imaplib.IMAP4, plus socket.getaddrinfo for the host check -- the way the |
| 5 | email outbox tests swap object_email.sender_for. It records every command, |
| 6 | so the claims that matter can be asserted as facts about the conversation |
| 7 | rather than inferred from the rows: the folder was EXAMINEd, every body was |
| 8 | fetched with PEEK, nothing was ever STOREd, APPENDed or EXPUNGEd. |
| 9 | |
| 10 | The lessons the predecessor taught are each a test here, named for the |
| 11 | claim: mail is never marked read; the same Message-ID in two people's |
| 12 | mailboxes is two rows; a reply never joins another person's thread; a |
| 13 | failure is written on the account and counted, never swallowed; a link in |
| 14 | a header is stored and never fetched; and a pass that dies half way loses |
| 15 | nothing and duplicates nothing. |
| 16 | |
| 17 | An OPTIONAL test at the bottom talks to a real IMAP server, and is skipped |
| 18 | unless DBBASIC_TEST_IMAP=host:port:user:password names one. |
| 19 | """ |
| 20 | |
| 21 | import imaplib |
| 22 | import json |
| 23 | import os |
| 24 | import shutil |
| 25 | import socket |
| 26 | from datetime import datetime, timedelta, timezone |
| 27 | from email.utils import format_datetime |
| 28 | from pathlib import Path |
| 29 | |
| 30 | import pytest |
| 31 | from conftest import stage_collection |
| 32 | |
| 33 | import object_execution |
| 34 | import object_mail |
| 35 | import object_records |
| 36 | import object_service_keys |
| 37 | import python_object_runtime |
| 38 | |
| 39 | REPO_ROOT = Path(__file__).resolve().parents[1] |
| 40 | PACKAGE = REPO_ROOT / "packages" / "app-mail-sync" |
| 41 | SOURCE = PACKAGE / "objects" / "system" / "mail_sync.py" |
| 42 | RUNTIME = python_object_runtime.PythonObjectRuntime() |
| 43 | |
| 44 | NOW = datetime.now(timezone.utc) |
| 45 | PASSWORD = "correct horse battery" |
| 46 | |
| 47 | |
| 48 | # --- the fake server ------------------------------------------------------------ |
| 49 | |
| 50 | def internaldate(moment): |
| 51 | return object_mail.imap_date(moment) + moment.strftime(" %H:%M:%S +0000") |
| 52 | |
| 53 | |
| 54 | class FakeServer: |
| 55 | """Mailboxes by (user, folder). Each message: uid, raw, flags, date.""" |
| 56 | |
| 57 | def __init__(self): |
| 58 | self.users = {} |
| 59 | self.boxes = {} |
| 60 | self.uidvalidity = {} |
| 61 | self.commands = [] |
| 62 | self.timeouts = [] |
| 63 | self.connections = 0 |
| 64 | self.fail_fetch_uid = None |
| 65 | self.sent_folder = "Sent" |
| 66 | self.send_uidnext = True |
| 67 | |
| 68 | def user(self, name, password=PASSWORD): |
| 69 | self.users[name] = password |
| 70 | self.boxes.setdefault((name, "INBOX"), []) |
| 71 | self.boxes.setdefault((name, self.sent_folder), []) |
| 72 | self.uidvalidity.setdefault((name, "INBOX"), 1) |
| 73 | self.uidvalidity.setdefault((name, self.sent_folder), 1) |
| 74 | |
| 75 | def deliver(self, user, raw, *, folder="INBOX", flags=(), when=None, size=None): |
| 76 | box = self.boxes.setdefault((user, folder), []) |
| 77 | self.uidvalidity.setdefault((user, folder), 1) |
| 78 | uid = (max((m["uid"] for m in box), default=0)) + 1 |
| 79 | box.append({"uid": uid, "raw": raw, "flags": list(flags), |
| 80 | "when": when or NOW - timedelta(hours=1), "size": size}) |
| 81 | return uid |
| 82 | |
| 83 | def connect(self, host, port, timeout=None, **_): |
| 84 | self.connections += 1 |
| 85 | self.timeouts.append(timeout) |
| 86 | return FakeConnection(self) |
| 87 | |
| 88 | |
| 89 | class FakeConnection: |
| 90 | def __init__(self, server): |
| 91 | self.server = server |
| 92 | self.user = None |
| 93 | self.box = None |
| 94 | self.readonly = None |
| 95 | self.pending = {} |
| 96 | |
| 97 | def _log(self, *parts): |
| 98 | self.server.commands.append(" ".join(str(p) for p in parts)) |
| 99 | |
| 100 | def starttls(self, ssl_context=None): |
| 101 | self._log("STARTTLS") |
| 102 | |
| 103 | def login(self, user, password): |
| 104 | self._log("LOGIN", user) |
| 105 | if self.server.users.get(user) != password: |
| 106 | raise imaplib.IMAP4.error(b"[AUTHENTICATIONFAILED] Invalid credentials") |
| 107 | self.user = user |
| 108 | return "OK", [b"logged in"] |
| 109 | |
| 110 | def logout(self): |
| 111 | self._log("LOGOUT") |
| 112 | return "BYE", [b""] |
| 113 | |
| 114 | def list(self, directory='""', pattern="*"): |
| 115 | self._log("LIST") |
| 116 | lines = [] |
| 117 | for (user, folder) in self.server.boxes: |
| 118 | if user != self.user: |
| 119 | continue |
| 120 | attrs = "\\HasNoChildren \\Sent" if folder == self.server.sent_folder else "\\HasNoChildren" |
| 121 | lines.append(f'({attrs}) "/" "{folder}"'.encode()) |
| 122 | return "OK", lines |
| 123 | |
| 124 | def select(self, mailbox, readonly=False): |
| 125 | name = mailbox[1:-1] if mailbox.startswith('"') else mailbox |
| 126 | self._log("EXAMINE" if readonly else "SELECT", name) |
| 127 | key = (self.user, name) |
| 128 | if key not in self.server.boxes: |
| 129 | return "NO", [b"Mailbox does not exist"] |
| 130 | self.box = key |
| 131 | self.readonly = readonly |
| 132 | messages = self.server.boxes[key] |
| 133 | self.pending = {"UIDVALIDITY": [str(self.server.uidvalidity[key]).encode()]} |
| 134 | if self.server.send_uidnext: |
| 135 | self.pending["UIDNEXT"] = [ |
| 136 | str(max((m["uid"] for m in messages), default=0) + 1).encode()] |
| 137 | return "OK", [str(len(messages)).encode()] |
| 138 | |
| 139 | def response(self, code): |
| 140 | return code, self.pending.pop(code, [None]) |
| 141 | |
| 142 | def _messages(self): |
| 143 | return self.server.boxes[self.box] |
| 144 | |
| 145 | def uid(self, command, *args): |
| 146 | self._log("UID", command, *args) |
| 147 | command = command.upper() |
| 148 | if command == "SEARCH": |
| 149 | return "OK", [" ".join(str(u) for u in self._search(args)).encode()] |
| 150 | if command == "FETCH": |
| 151 | return self._fetch(args[0], args[1]) |
| 152 | raise AssertionError(f"the sync issued UID {command}, which it must never do") |
| 153 | |
| 154 | def _search(self, args): |
| 155 | messages = self._messages() |
| 156 | if args[0] == "ALL": |
| 157 | return [m["uid"] for m in messages] |
| 158 | if args[0] == "SINCE": |
| 159 | day, month, year = args[1].split("-") |
| 160 | since = datetime(int(year), object_mail._MONTHS.index(month) + 1, int(day), |
| 161 | tzinfo=timezone.utc) |
| 162 | return [m["uid"] for m in messages if m["when"] >= since] |
| 163 | if args[0] == "UID": |
| 164 | low = int(args[1].split(":")[0]) |
| 165 | found = [m["uid"] for m in messages if m["uid"] >= low] |
| 166 | # Real servers answer n:* with the highest message even when n |
| 167 | # is past it. The sync must not import that message again. |
| 168 | if not found and messages: |
| 169 | found = [messages[-1]["uid"]] |
| 170 | return found |
| 171 | raise AssertionError(f"unexpected SEARCH {args}") |
| 172 | |
| 173 | def _fetch(self, uid_set, items): |
| 174 | wanted = {int(u) for u in uid_set.split(",")} |
| 175 | data = [] |
| 176 | for m in self._messages(): |
| 177 | if m["uid"] not in wanted: |
| 178 | continue |
| 179 | if "BODY.PEEK[" in items: |
| 180 | if self.server.fail_fetch_uid == m["uid"]: |
| 181 | raise OSError("connection reset by peer") |
| 182 | raw = m["raw"] |
| 183 | if "HEADER" in items: |
| 184 | raw = raw.split(b"\r\n\r\n", 1)[0] + b"\r\n\r\n" |
| 185 | data.append((f"{m['uid']} (UID {m['uid']} BODY[] {{{len(raw)}}}".encode(), raw)) |
| 186 | data.append(b")") |
| 187 | elif "BODY[" in items or "RFC822" in items.replace("RFC822.SIZE", ""): |
| 188 | # A non-PEEK body fetch sets \Seen on a real server. |
| 189 | if "\\Seen" not in m["flags"]: |
| 190 | m["flags"].append("\\Seen") |
| 191 | raise AssertionError("the sync fetched a body without PEEK") |
| 192 | else: |
| 193 | size = m["size"] if m["size"] is not None else len(m["raw"]) |
| 194 | flags = " ".join(m["flags"]) |
| 195 | data.append( |
| 196 | f'{m["uid"]} (UID {m["uid"]} FLAGS ({flags}) RFC822.SIZE {size} ' |
| 197 | f'INTERNALDATE "{internaldate(m["when"])}")'.encode()) |
| 198 | return "OK", data |
| 199 | |
| 200 | |
| 201 | def mail(message_id=None, *, sender="alice@example.org", to="me@dove.test", subject="Hello", |
| 202 | body="Hi there.", in_reply_to=None, references=None, extra=""): |
| 203 | lines = [f"From: Alice <{sender}>", f"To: {to}", f"Subject: {subject}", |
| 204 | f"Date: {format_datetime(NOW - timedelta(hours=2))}"] |
| 205 | if message_id: |
| 206 | lines.append(f"Message-ID: {message_id}") |
| 207 | if in_reply_to: |
| 208 | lines.append(f"In-Reply-To: {in_reply_to}") |
| 209 | if references: |
| 210 | lines.append(f"References: {references}") |
| 211 | if extra: |
| 212 | lines.append(extra) |
| 213 | lines += ["MIME-Version: 1.0", "Content-Type: text/plain; charset=utf-8", "", body, ""] |
| 214 | return "\r\n".join(lines).encode("utf-8") |
| 215 | |
| 216 | |
| 217 | # --- the box ------------------------------------------------------------------------- |
| 218 | |
| 219 | @pytest.fixture |
| 220 | def box(tmp_path, monkeypatch): |
| 221 | data_dir = tmp_path / "data" |
| 222 | for name in ("message_threads", "messages", "message_recipients"): |
| 223 | stage_collection(data_dir, "app-messaging", name) |
| 224 | for name in ("mail_accounts", "mail_sync_state", "mail_synced_messages"): |
| 225 | stage_collection(data_dir, "app-mail-sync", name) |
| 226 | objects_root = tmp_path / "objects" |
| 227 | shutil.copytree(PACKAGE / "objects", objects_root) |
| 228 | |
| 229 | server = FakeServer() |
| 230 | |
| 231 | def connect(*args, **kwargs): |
| 232 | return server.connect(*args, **kwargs) |
| 233 | |
| 234 | # The object names imaplib.IMAP4.error / .abort; the stand-in keeps them. |
| 235 | connect.error, connect.abort = imaplib.IMAP4.error, imaplib.IMAP4.abort |
| 236 | monkeypatch.setattr(imaplib, "IMAP4_SSL", connect) |
| 237 | monkeypatch.setattr(imaplib, "IMAP4", connect) |
| 238 | addresses = {"host": "1.1.1.1"} |
| 239 | monkeypatch.setattr(socket, "getaddrinfo", lambda host, port, *a, **k: [ |
| 240 | (socket.AF_INET, socket.SOCK_STREAM, 6, "", (addresses["host"], port))]) |
| 241 | monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir)) |
| 242 | monkeypatch.setenv("DBBASIC_ENABLE_MAIL_SYNC", "true") |
| 243 | monkeypatch.delenv("DBBASIC_MAIL_SYNC_ALLOW_PRIVATE_HOSTS", raising=False) |
| 244 | |
| 245 | class Box: |
| 246 | pass |
| 247 | |
| 248 | b = Box() |
| 249 | b.data, b.objects, b.server, b.addresses = data_dir, objects_root, server, addresses |
| 250 | return b |
| 251 | |
| 252 | |
| 253 | def account(box, account_id="acct-1", *, owner="u1", user="me@dove.test", password=PASSWORD, |
| 254 | **fields): |
| 255 | row = {"id": account_id, "label": "Work", "email_address": user, |
| 256 | "host": "imap.example.test", "security": "ssl", "username": user, |
| 257 | "owner_id": owner} |
| 258 | row.update(fields) |
| 259 | object_records.create_collection_record("mail_accounts", row, base_dir=box.data, actor="test") |
| 260 | if password is not None: |
| 261 | object_service_keys.set_service_key(owner, f"imap-{account_id}", password, |
| 262 | base_dir=box.data) |
| 263 | return row |
| 264 | |
| 265 | |
| 266 | def run(box, method="POST", object_id="system_mail_sync"): |
| 267 | outcome = object_execution.execute_object( |
| 268 | RUNTIME, object_execution.ObjectExecutionRequest(object_id, method=method, payload={}), |
| 269 | roots=[box.objects]) |
| 270 | assert outcome.ok, outcome.error.to_dict() if outcome.error else outcome |
| 271 | return outcome.result |
| 272 | |
| 273 | |
| 274 | def rows(box, collection): |
| 275 | return object_records.read_collection_records(collection, base_dir=box.data) |
| 276 | |
| 277 | |
| 278 | def acct(box, account_id="acct-1"): |
| 279 | return object_records.get_collection_record("mail_accounts", account_id, base_dir=box.data) |
| 280 | |
| 281 | |
| 282 | def retry_now(box, account_id="acct-1"): |
| 283 | """What the owner does after fixing something: zero the failure count.""" |
| 284 | object_records.update_collection_record("mail_accounts", account_id, |
| 285 | {"sync_attempts": "0"}, base_dir=box.data, |
| 286 | actor="u1") |
| 287 | |
| 288 | |
| 289 | def change_count(box, collection): |
| 290 | path = box.data / "record_changes" / collection / "changes.jsonl" |
| 291 | return len(path.read_text().splitlines()) if path.exists() else 0 |
| 292 | |
| 293 | |
| 294 | # --- off unless the operator says so ----------------------------------------------------- |
| 295 | |
| 296 | def test_the_pass_is_off_until_the_operator_opts_in_and_says_which_variable(box, monkeypatch): |
| 297 | monkeypatch.delenv("DBBASIC_ENABLE_MAIL_SYNC") |
| 298 | account(box) |
| 299 | result = run(box) |
| 300 | assert "DBBASIC_ENABLE_MAIL_SYNC" in result["skipped"] |
| 301 | assert box.server.connections == 0 |
| 302 | |
| 303 | |
| 304 | def test_the_kill_switch_stops_it_without_a_redeploy(box): |
| 305 | stage_collection(box.data, "app-settings", "feature_flags") |
| 306 | object_records.create_collection_record( |
| 307 | "feature_flags", {"id": "f1", "flag": "mail_sync_enabled", "value": "off"}, |
| 308 | base_dir=box.data, actor="test") |
| 309 | account(box) |
| 310 | result = run(box) |
| 311 | assert "mail_sync_enabled" in result["skipped"] |
| 312 | assert box.server.connections == 0 |
| 313 | |
| 314 | |
| 315 | def test_an_account_without_a_password_is_skipped_not_failed(box): |
| 316 | """Not set up is not a failure: nothing counts against the account, so |
| 317 | setting the password later just works.""" |
| 318 | account(box, password=None) |
| 319 | result = run(box) |
| 320 | assert result["declined"] == 1 |
| 321 | assert "imap-acct-1" in result["results"][0]["declined"] |
| 322 | assert box.server.connections == 0 |
| 323 | assert acct(box)["sync_attempts"] in ("", "0") |
| 324 | |
| 325 | |
| 326 | # --- copying without touching the mailbox ------------------------------------------------- |
| 327 | |
| 328 | def test_mail_is_copied_and_never_marked_read(box): |
| 329 | """The predecessor's worst habit: syncing marked everything read, and |
| 330 | mail read elsewhere was never synced. Here the folder is EXAMINEd, the |
| 331 | body is PEEKed, and is_read is whatever the server said.""" |
| 332 | box.server.user("me@dove.test") |
| 333 | box.server.deliver("me@dove.test", mail("<m1@x>", subject="Unread one")) |
| 334 | box.server.deliver("me@dove.test", mail("<m2@x>", subject="Read elsewhere"), flags=["\\Seen"]) |
| 335 | account(box) |
| 336 | |
| 337 | result = run(box) |
| 338 | assert result["imported"] == 2, result |
| 339 | |
| 340 | messages = {m["subject"]: m for m in rows(box, "messages")} |
| 341 | assert messages["Unread one"]["is_read"] == "false" |
| 342 | assert messages["Read elsewhere"]["is_read"] == "true" |
| 343 | assert all(m["owner_id"] == "u1" and m["direction"] == "inbound" for m in messages.values()) |
| 344 | flags = [m["flags"] for m in box.server.boxes[("me@dove.test", "INBOX")]] |
| 345 | assert flags == [[], ["\\Seen"]] |
| 346 | assert not any(c.startswith("SELECT") for c in box.server.commands) |
| 347 | fetches = [c for c in box.server.commands if c.startswith("UID FETCH") and "BODY" in c] |
| 348 | assert fetches and all("BODY.PEEK[" in c for c in fetches) |
| 349 | for verb in ("STORE", "APPEND", "EXPUNGE", "COPY", "MOVE"): |
| 350 | assert not any(verb in c.split()[:2] for c in box.server.commands), verb |
| 351 | |
| 352 | |
| 353 | def test_recipients_threads_and_transport_facts_land_in_their_own_places(box): |
| 354 | box.server.user("me@dove.test") |
| 355 | box.server.deliver("me@dove.test", mail( |
| 356 | "<m1@x>", to="me@dove.test, Bob <bob@example.org>", |
| 357 | extra="Cc: carol@example.org\r\nList-Id: <news.example.org>\r\n" |
| 358 | "List-Unsubscribe: <https://example.org/unsub?u=1>")) |
| 359 | account(box) |
| 360 | run(box) |
| 361 | |
| 362 | message = rows(box, "messages")[0] |
| 363 | recipients = sorted((r["kind"], r["address"]) for r in rows(box, "message_recipients")) |
| 364 | assert recipients == [("cc", "carol@example.org"), ("to", "bob@example.org"), |
| 365 | ("to", "me@dove.test")] |
| 366 | thread = rows(box, "message_threads")[0] |
| 367 | assert thread["id"] == message["thread_id"] and thread["message_count"] == "1" |
| 368 | assert thread["is_read"] == "false" and thread["owner_id"] == "u1" |
| 369 | synced = rows(box, "mail_synced_messages")[0] |
| 370 | assert synced["message_id"] == message["id"] and synced["folder"] == "INBOX" |
| 371 | assert synced["list_unsubscribe"] == "<https://example.org/unsub?u=1>" |
| 372 | assert synced["list_id"] == "<news.example.org>" |
| 373 | |
| 374 | |
| 375 | def test_the_first_sync_takes_a_recent_window_not_the_whole_history(box): |
| 376 | box.server.user("me@dove.test") |
| 377 | box.server.deliver("me@dove.test", mail("<old@x>", subject="Old"), when=NOW - timedelta(days=90)) |
| 378 | box.server.deliver("me@dove.test", mail("<new@x>", subject="New")) |
| 379 | account(box) |
| 380 | run(box) |
| 381 | assert [m["subject"] for m in rows(box, "messages")] == ["New"] |
| 382 | |
| 383 | |
| 384 | def test_folders_are_found_on_the_first_sync_and_written_back(box): |
| 385 | box.server.user("me@dove.test") |
| 386 | account(box) |
| 387 | run(box) |
| 388 | assert acct(box)["folders"] == "INBOX, Sent" |
| 389 | roles = {s["folder"]: s["role"] for s in rows(box, "mail_sync_state")} |
| 390 | assert roles == {"INBOX": "inbox", "Sent": "sent"} |
| 391 | |
| 392 | |
| 393 | def test_a_pass_copies_at_most_fifty_messages_and_the_next_one_continues(box): |
| 394 | """The pass runs inside the daemon's loop: a backlog drains over |
| 395 | several passes instead of one pass holding everything else up.""" |
| 396 | box.server.user("me@dove.test") |
| 397 | for n in range(60): |
| 398 | box.server.deliver("me@dove.test", mail(f"<m{n}@x>", subject=f"n{n}")) |
| 399 | account(box) |
| 400 | assert run(box)["imported"] == 50 |
| 401 | assert run(box)["imported"] == 10 |
| 402 | assert len(rows(box, "messages")) == 60 |
| 403 | |
| 404 | |
| 405 | def test_a_huge_message_arrives_as_headers_and_says_so(box): |
| 406 | box.server.user("me@dove.test") |
| 407 | box.server.deliver("me@dove.test", mail("<big@x>", subject="Big", body="body text"), |
| 408 | size=40 * 1024 * 1024) |
| 409 | account(box) |
| 410 | run(box) |
| 411 | message = rows(box, "messages")[0] |
| 412 | assert message["subject"] == "Big" |
| 413 | assert "only its headers were copied" in message["body_text"] |
| 414 | assert any("BODY.PEEK[HEADER]" in c for c in box.server.commands) |
| 415 | |
| 416 | |
| 417 | def test_every_connection_has_a_timeout(box): |
| 418 | box.server.user("me@dove.test") |
| 419 | account(box) |
| 420 | run(box) |
| 421 | assert box.server.timeouts and all(t and t > 0 for t in box.server.timeouts) |
| 422 | |
| 423 | |
| 424 | # --- dedupe and at-least-once ------------------------------------------------------------- |
| 425 | |
| 426 | def test_running_twice_copies_nothing_twice_and_an_idle_pass_writes_nothing(box): |
| 427 | box.server.user("me@dove.test") |
| 428 | box.server.deliver("me@dove.test", mail("<m1@x>")) |
| 429 | account(box) |
| 430 | run(box) |
| 431 | before = {c: change_count(box, c) for c in ("mail_accounts", "messages", "mail_sync_state")} |
| 432 | second = run(box) |
| 433 | assert second["imported"] == 0 and second["results"][0].get("up_to_date") is True |
| 434 | after = {c: change_count(box, c) for c in ("mail_accounts", "messages", "mail_sync_state")} |
| 435 | assert after == before, "an account with nothing new must not rewrite its rows" |
| 436 | assert len(rows(box, "messages")) == 1 |
| 437 | |
| 438 | |
| 439 | def test_a_server_without_uidnext_does_not_hand_back_its_newest_message_forever(box): |
| 440 | """UID SEARCH UID n:* answers with the highest message even when n is |
| 441 | past it. Without the filter every pass would download that message |
| 442 | again (dedupe would drop it, but the download is the cost).""" |
| 443 | box.server.send_uidnext = False |
| 444 | box.server.user("me@dove.test") |
| 445 | box.server.deliver("me@dove.test", mail("<m1@x>")) |
| 446 | account(box) |
| 447 | run(box) |
| 448 | box.server.commands.clear() |
| 449 | run(box) |
| 450 | assert not [c for c in box.server.commands if "BODY.PEEK" in c] |
| 451 | assert len(rows(box, "messages")) == 1 |
| 452 | |
| 453 | |
| 454 | def test_a_pass_that_dies_half_way_loses_nothing_and_duplicates_nothing(box): |
| 455 | """The cursor moves only past rows that are written; the retry |
| 456 | re-meets nothing it already wrote because every id is derived from the |
| 457 | message.""" |
| 458 | box.server.user("me@dove.test") |
| 459 | for n in range(3): |
| 460 | box.server.deliver("me@dove.test", mail(f"<m{n}@x>", subject=f"n{n}")) |
| 461 | account(box) |
| 462 | box.server.fail_fetch_uid = 2 |
| 463 | first = run(box) |
| 464 | assert first["failed"] == 1 |
| 465 | assert [m["subject"] for m in rows(box, "messages")] == ["n0"] |
| 466 | inbox = next(s for s in rows(box, "mail_sync_state") if s["folder"] == "INBOX") |
| 467 | assert inbox["last_uid"] == "1" |
| 468 | assert acct(box)["sync_attempts"] == "1" |
| 469 | |
| 470 | box.server.fail_fetch_uid = None |
| 471 | retry_now(box) |
| 472 | run(box) |
| 473 | assert sorted(m["subject"] for m in rows(box, "messages")) == ["n0", "n1", "n2"] |
| 474 | assert acct(box)["sync_attempts"] == "0" and acct(box)["sync_error"] == "" |
| 475 | |
| 476 | |
| 477 | def test_replaying_a_batch_whose_cursor_never_moved_writes_nothing_new(box): |
| 478 | """The worst crash: every row written, the cursor not. Same ids, so the |
| 479 | replay is refused row by row and nothing is doubled.""" |
| 480 | box.server.user("me@dove.test") |
| 481 | box.server.deliver("me@dove.test", mail("<m1@x>")) |
| 482 | box.server.deliver("me@dove.test", mail("<m2@x>", in_reply_to="<m1@x>")) |
| 483 | account(box) |
| 484 | run(box) |
| 485 | inbox = next(s for s in rows(box, "mail_sync_state") if s["folder"] == "INBOX") |
| 486 | object_records.update_collection_record("mail_sync_state", inbox["id"], {"last_uid": "0"}, |
| 487 | base_dir=box.data, actor="test") |
| 488 | counts = {c: len(rows(box, c)) for c in ("messages", "message_threads", |
| 489 | "message_recipients", "mail_synced_messages")} |
| 490 | run(box) |
| 491 | assert {c: len(rows(box, c)) for c in counts} == counts |
| 492 | assert rows(box, "message_threads")[0]["message_count"] == "2" |
| 493 | |
| 494 | |
| 495 | def test_a_uidvalidity_change_restarts_the_folder_without_duplicates(box): |
| 496 | box.server.user("me@dove.test") |
| 497 | box.server.deliver("me@dove.test", mail("<m1@x>")) |
| 498 | account(box) |
| 499 | run(box) |
| 500 | box.server.uidvalidity[("me@dove.test", "INBOX")] = 99 |
| 501 | box.server.deliver("me@dove.test", mail("<m2@x>")) |
| 502 | run(box) |
| 503 | assert sorted(m["external_message_id"] for m in rows(box, "messages")) == ["<m1@x>", "<m2@x>"] |
| 504 | inbox = next(s for s in rows(box, "mail_sync_state") if s["folder"] == "INBOX") |
| 505 | assert inbox["uidvalidity"] == "99" |
| 506 | |
| 507 | |
| 508 | # --- the tenant boundary --------------------------------------------------------------------- |
| 509 | |
| 510 | def test_the_same_message_in_two_peoples_mailboxes_is_two_rows(box): |
| 511 | """The predecessor deduped on Message-ID alone and made one row of it.""" |
| 512 | for user in ("me@dove.test", "alice@dove.test"): |
| 513 | box.server.user(user) |
| 514 | box.server.deliver(user, mail("<shared@x>", to="me@dove.test, alice@dove.test")) |
| 515 | account(box, "acct-1", owner="u1", user="me@dove.test") |
| 516 | account(box, "acct-2", owner="u2", user="alice@dove.test") |
| 517 | run(box) |
| 518 | messages = rows(box, "messages") |
| 519 | assert sorted(m["owner_id"] for m in messages) == ["u1", "u2"] |
| 520 | assert len({m["thread_id"] for m in messages}) == 2 |
| 521 | threads = {t["id"]: t["owner_id"] for t in rows(box, "message_threads")} |
| 522 | assert all(threads[m["thread_id"]] == m["owner_id"] for m in messages) |
| 523 | recipients = rows(box, "message_recipients") |
| 524 | owner_of = {m["id"]: m["owner_id"] for m in messages} |
| 525 | assert all(r["owner_id"] == owner_of[r["message_id"]] for r in recipients) |
| 526 | |
| 527 | |
| 528 | def test_a_reply_never_joins_another_persons_thread(box): |
| 529 | """u1 holds the original; u2 receives only a reply to it. u2's reply |
| 530 | must start u2's own thread.""" |
| 531 | box.server.user("me@dove.test") |
| 532 | box.server.user("alice@dove.test") |
| 533 | box.server.deliver("me@dove.test", mail("<orig@x>", subject="Plans")) |
| 534 | account(box, "acct-1", owner="u1", user="me@dove.test") |
| 535 | account(box, "acct-2", owner="u2", user="alice@dove.test") |
| 536 | run(box) |
| 537 | box.server.deliver("alice@dove.test", mail("<reply@x>", subject="Re: Plans", |
| 538 | in_reply_to="<orig@x>", references="<orig@x>")) |
| 539 | retry_now(box, "acct-2") |
| 540 | run(box) |
| 541 | by_owner = {m["owner_id"]: m for m in rows(box, "messages")} |
| 542 | assert by_owner["u1"]["thread_id"] != by_owner["u2"]["thread_id"] |
| 543 | thread = object_records.get_collection_record("message_threads", by_owner["u2"]["thread_id"], |
| 544 | base_dir=box.data) |
| 545 | assert thread["owner_id"] == "u2" |
| 546 | |
| 547 | |
| 548 | # --- threading ---------------------------------------------------------------------------------- |
| 549 | |
| 550 | def test_a_reply_joins_the_sent_message_it_answers_and_a_subject_joins_nothing(box): |
| 551 | box.server.user("me@dove.test") |
| 552 | box.server.deliver("me@dove.test", mail("<out@x>", sender="me@dove.test", to="bob@example.org", |
| 553 | subject="Invoice"), folder="Sent") |
| 554 | box.server.deliver("me@dove.test", mail("<back@x>", sender="bob@example.org", |
| 555 | subject="Re: Invoice", in_reply_to="<out@x>")) |
| 556 | box.server.deliver("me@dove.test", mail("<other@x>", sender="eve@example.org", |
| 557 | subject="Invoice")) |
| 558 | account(box) |
| 559 | run(box) |
| 560 | messages = {m["external_message_id"]: m for m in rows(box, "messages")} |
| 561 | assert messages["<out@x>"]["direction"] == "outbound" |
| 562 | assert messages["<out@x>"]["is_read"] == "true" |
| 563 | assert messages["<back@x>"]["thread_id"] == messages["<out@x>"]["thread_id"] |
| 564 | assert messages["<other@x>"]["thread_id"] != messages["<out@x>"]["thread_id"] |
| 565 | thread = object_records.get_collection_record( |
| 566 | "message_threads", messages["<out@x>"]["thread_id"], base_dir=box.data) |
| 567 | assert thread["message_count"] == "2" |
| 568 | assert thread["is_read"] == "false", "an unread reply makes its thread unread" |
| 569 | |
| 570 | |
| 571 | # --- failures are written down -------------------------------------------------------------- |
| 572 | |
| 573 | def test_a_refused_login_is_recorded_without_the_password_and_backs_off(box): |
| 574 | box.server.user("me@dove.test", password="something else") |
| 575 | account(box) |
| 576 | result = run(box) |
| 577 | assert result["failed"] == 1 |
| 578 | row = acct(box) |
| 579 | assert row["sync_attempts"] == "1" |
| 580 | assert "refused the login" in row["sync_error"] |
| 581 | assert PASSWORD not in json.dumps(result) and PASSWORD not in row["sync_error"] |
| 582 | |
| 583 | connections = box.server.connections |
| 584 | again = run(box) |
| 585 | assert box.server.connections == connections, "a failing account waits before the next try" |
| 586 | assert "waiting" in again["results"][0] |
| 587 | |
| 588 | |
| 589 | def test_three_failures_in_a_row_are_counted_on_the_home_page(box): |
| 590 | box.server.user("me@dove.test", password="something else") |
| 591 | account(box) |
| 592 | assert run(box, "COUNT", "system_mail_sync_attention") == {"count": 0} |
| 593 | for _ in range(3): |
| 594 | retry_now_keep = acct(box)["sync_attempts"] |
| 595 | object_records.update_collection_record( |
| 596 | "mail_accounts", "acct-1", {"sync_finished_at": "2000-01-01T00:00:00Z"}, |
| 597 | base_dir=box.data, actor="test", preserve_read_only=True) |
| 598 | run(box) |
| 599 | assert int(acct(box)["sync_attempts"]) == int(retry_now_keep or 0) + 1 |
| 600 | counted = run(box, "COUNT", "system_mail_sync_attention") |
| 601 | assert counted["count"] == 1 |
| 602 | assert "me@dove.test" not in json.dumps(counted), "the home page count names nobody" |
| 603 | |
| 604 | |
| 605 | def test_a_missing_folder_is_an_error_on_the_account_not_a_silent_skip(box): |
| 606 | box.server.user("me@dove.test") |
| 607 | box.server.deliver("me@dove.test", mail("<m1@x>")) |
| 608 | account(box, folders="INBOX, Archive") |
| 609 | run(box) |
| 610 | row = acct(box) |
| 611 | assert row["sync_attempts"] == "1" and "Archive" in row["sync_error"] |
| 612 | assert len(rows(box, "messages")) == 1, "the folder that works still syncs" |
| 613 | |
| 614 | |
| 615 | def test_a_private_address_is_refused_unless_the_operator_allows_it(box, monkeypatch): |
| 616 | """The host is typed by a member and the server is the one connecting: |
| 617 | without this, anyone could make it knock on its own network.""" |
| 618 | box.server.user("me@dove.test") |
| 619 | box.addresses["host"] = "10.0.0.5" |
| 620 | account(box) |
| 621 | run(box) |
| 622 | assert box.server.connections == 0 |
| 623 | assert "DBBASIC_MAIL_SYNC_ALLOW_PRIVATE_HOSTS" in acct(box)["sync_error"] |
| 624 | |
| 625 | monkeypatch.setenv("DBBASIC_MAIL_SYNC_ALLOW_PRIVATE_HOSTS", "true") |
| 626 | retry_now(box) |
| 627 | run(box) |
| 628 | assert box.server.connections == 1 |
| 629 | assert acct(box)["sync_error"] == "" |
| 630 | |
| 631 | |
| 632 | # --- what it never does ------------------------------------------------------------------------ |
| 633 | |
| 634 | def test_it_never_fetches_a_url_from_a_header_and_never_sends(box, monkeypatch): |
| 635 | """List-Unsubscribe is stored for a person to act on; the server opening |
| 636 | it would let any sender make it issue requests.""" |
| 637 | import http.client |
| 638 | import smtplib |
| 639 | import urllib.request |
| 640 | |
| 641 | def refuse(*args, **kwargs): |
| 642 | raise AssertionError("the mail sync opened a network connection it must not") |
| 643 | |
| 644 | monkeypatch.setattr(urllib.request, "urlopen", refuse) |
| 645 | monkeypatch.setattr(http.client.HTTPConnection, "connect", refuse) |
| 646 | monkeypatch.setattr(smtplib, "SMTP", refuse) |
| 647 | monkeypatch.setattr(smtplib, "SMTP_SSL", refuse) |
| 648 | box.server.user("me@dove.test") |
| 649 | box.server.deliver("me@dove.test", mail( |
| 650 | "<m1@x>", extra="List-Unsubscribe: <https://example.org/u>\r\nList-Unsubscribe-Post: " |
| 651 | "List-Unsubscribe=One-Click")) |
| 652 | account(box) |
| 653 | assert run(box)["imported"] == 1 |
| 654 | source = SOURCE.read_text() + (REPO_ROOT / "object_mail.py").read_text() |
| 655 | for module in ("urllib", "http.client", "smtplib", "requests"): |
| 656 | assert f"import {module}" not in source |
| 657 | |
| 658 | |
| 659 | def test_the_scheduled_entry_point_is_event_with_post_as_its_alias(): |
| 660 | """AGENTS.md: the dispatcher calls EVENT; an object with only POST never |
| 661 | runs and never complains.""" |
| 662 | source = SOURCE.read_text() |
| 663 | assert "def EVENT(request)" in source and "POST = EVENT" in source |
| 664 | |
| 665 | |
| 666 | # --- the page --------------------------------------------------------------------------------- |
| 667 | |
| 668 | def page(box, user_id=None): |
| 669 | payload = {"_identity": {"user_id": user_id}} if user_id else {} |
| 670 | outcome = object_execution.execute_object( |
| 671 | RUNTIME, object_execution.ObjectExecutionRequest("site_mail_accounts", method="GET", |
| 672 | payload=payload), |
| 673 | roots=[box.objects]) |
| 674 | assert outcome.ok, outcome.error.to_dict() if outcome.error else outcome |
| 675 | return outcome.result["body"] |
| 676 | |
| 677 | |
| 678 | def test_the_page_shows_whether_a_password_is_set_and_never_the_password(box): |
| 679 | account(box, "acct-1", owner="u1", label="Work") |
| 680 | account(box, "acct-2", owner="u1", label="Home", password=None) |
| 681 | account(box, "acct-3", owner="u2", label="Someone else's") |
| 682 | body = page(box, "u1") |
| 683 | assert "Work" in body and "Home" in body |
| 684 | assert "Someone else" not in body, "only the signed-in person's accounts" |
| 685 | assert body.count("<strong data-pwstate>set</strong>") == 1 |
| 686 | assert body.count("<strong data-pwstate>not set</strong>") == 1 |
| 687 | assert PASSWORD not in body |
| 688 | assert "/service-keys" in body and '"imap-" + id' in body |
| 689 | |
| 690 | |
| 691 | def test_the_page_says_when_the_operator_has_not_turned_sync_on(box, monkeypatch): |
| 692 | monkeypatch.delenv("DBBASIC_ENABLE_MAIL_SYNC") |
| 693 | account(box) |
| 694 | assert "DBBASIC_ENABLE_MAIL_SYNC" in page(box, "u1") |
| 695 | |
| 696 | |
| 697 | def test_a_visitor_gets_a_sign_in_prompt(box): |
| 698 | account(box) |
| 699 | body = page(box) |
| 700 | assert "Sign in" in body and "Work" not in body |
| 701 | |
| 702 | |
| 703 | # --- a real server, when one is named ------------------------------------------------------------- |
| 704 | |
| 705 | REAL = os.environ.get("DBBASIC_TEST_IMAP", "") |
| 706 | |
| 707 | |
| 708 | @pytest.mark.skipif(not REAL, reason="set DBBASIC_TEST_IMAP=host:port:user:password to run") |
| 709 | def test_against_a_real_imap_server(tmp_path, monkeypatch): |
| 710 | """Plain IMAP on this machine (a GreenMail or Dovecot test server): the |
| 711 | sync copies whatever the window holds and leaves every flag as it was.""" |
| 712 | host, port, user, password = REAL.split(":", 3) |
| 713 | data_dir = tmp_path / "data" |
| 714 | for name in ("message_threads", "messages", "message_recipients"): |
| 715 | stage_collection(data_dir, "app-messaging", name) |
| 716 | for name in ("mail_accounts", "mail_sync_state", "mail_synced_messages"): |
| 717 | stage_collection(data_dir, "app-mail-sync", name) |
| 718 | objects_root = tmp_path / "objects" |
| 719 | shutil.copytree(PACKAGE / "objects", objects_root) |
| 720 | monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir)) |
| 721 | monkeypatch.setenv("DBBASIC_ENABLE_MAIL_SYNC", "true") |
| 722 | monkeypatch.setenv("DBBASIC_MAIL_SYNC_ALLOW_PRIVATE_HOSTS", "true") |
| 723 | |
| 724 | def flags(): |
| 725 | conn = imaplib.IMAP4(host, int(port), timeout=10) |
| 726 | conn.login(user, password) |
| 727 | conn.select("INBOX", readonly=True) |
| 728 | typ, data = conn.fetch("1:*", "(UID FLAGS)") |
| 729 | conn.logout() |
| 730 | return sorted(object_mail._as_text(d) for d in data if d) |
| 731 | |
| 732 | before = flags() |
| 733 | object_records.create_collection_record("mail_accounts", { |
| 734 | "id": "real-1", "email_address": user if "@" in user else f"{user}@localhost", |
| 735 | "host": host, "port": port, "security": "plain", "username": user, |
| 736 | "owner_id": "u1"}, base_dir=data_dir, actor="test") |
| 737 | object_service_keys.set_service_key("u1", "imap-real-1", password, base_dir=data_dir) |
| 738 | result = object_execution.execute_object( |
| 739 | RUNTIME, object_execution.ObjectExecutionRequest("system_mail_sync", method="POST", |
| 740 | payload={}), |
| 741 | roots=[objects_root]).result |
| 742 | assert result["failed"] == 0, result |
| 743 | assert flags() == before, "the sync changed a flag on the server" |
| 744 | assert len(object_records.read_collection_records("messages", base_dir=data_dir)) >= 1 |