Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_mail_sync.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_mail_sync.py · 31.4 KB · 744 lines
1"""system_mail_sync against a fake IMAP server.
2
3The fake is injected where the real one would be -- imaplib.IMAP4_SSL and
4imaplib.IMAP4, plus socket.getaddrinfo for the host check -- the way the
5email outbox tests swap object_email.sender_for. It records every command,
6so the claims that matter can be asserted as facts about the conversation
7rather than inferred from the rows: the folder was EXAMINEd, every body was
8fetched with PEEK, nothing was ever STOREd, APPENDed or EXPUNGEd.
9
10The lessons the predecessor taught are each a test here, named for the
11claim: mail is never marked read; the same Message-ID in two people's
12mailboxes is two rows; a reply never joins another person's thread; a
13failure is written on the account and counted, never swallowed; a link in
14a header is stored and never fetched; and a pass that dies half way loses
15nothing and duplicates nothing.
16
17An OPTIONAL test at the bottom talks to a real IMAP server, and is skipped
18unless DBBASIC_TEST_IMAP=host:port:user:password names one.
19"""
20
21import imaplib
22import json
23import os
24import shutil
25import socket
26from datetime import datetime, timedelta, timezone
27from email.utils import format_datetime
28from pathlib import Path
29
30import pytest
31from conftest import stage_collection
32
33import object_execution
34import object_mail
35import object_records
36import object_service_keys
37import python_object_runtime
38
39REPO_ROOT = Path(__file__).resolve().parents[1]
40PACKAGE = REPO_ROOT / "packages" / "app-mail-sync"
41SOURCE = PACKAGE / "objects" / "system" / "mail_sync.py"
42RUNTIME = python_object_runtime.PythonObjectRuntime()
43
44NOW = datetime.now(timezone.utc)
45PASSWORD = "correct horse battery"
46
47
48# --- the fake server ------------------------------------------------------------
49
50def internaldate(moment):
51 return object_mail.imap_date(moment) + moment.strftime(" %H:%M:%S +0000")
52
53
54class FakeServer:
55 """Mailboxes by (user, folder). Each message: uid, raw, flags, date."""
56
57 def __init__(self):
58 self.users = {}
59 self.boxes = {}
60 self.uidvalidity = {}
61 self.commands = []
62 self.timeouts = []
63 self.connections = 0
64 self.fail_fetch_uid = None
65 self.sent_folder = "Sent"
66 self.send_uidnext = True
67
68 def user(self, name, password=PASSWORD):
69 self.users[name] = password
70 self.boxes.setdefault((name, "INBOX"), [])
71 self.boxes.setdefault((name, self.sent_folder), [])
72 self.uidvalidity.setdefault((name, "INBOX"), 1)
73 self.uidvalidity.setdefault((name, self.sent_folder), 1)
74
75 def deliver(self, user, raw, *, folder="INBOX", flags=(), when=None, size=None):
76 box = self.boxes.setdefault((user, folder), [])
77 self.uidvalidity.setdefault((user, folder), 1)
78 uid = (max((m["uid"] for m in box), default=0)) + 1
79 box.append({"uid": uid, "raw": raw, "flags": list(flags),
80 "when": when or NOW - timedelta(hours=1), "size": size})
81 return uid
82
83 def connect(self, host, port, timeout=None, **_):
84 self.connections += 1
85 self.timeouts.append(timeout)
86 return FakeConnection(self)
87
88
89class FakeConnection:
90 def __init__(self, server):
91 self.server = server
92 self.user = None
93 self.box = None
94 self.readonly = None
95 self.pending = {}
96
97 def _log(self, *parts):
98 self.server.commands.append(" ".join(str(p) for p in parts))
99
100 def starttls(self, ssl_context=None):
101 self._log("STARTTLS")
102
103 def login(self, user, password):
104 self._log("LOGIN", user)
105 if self.server.users.get(user) != password:
106 raise imaplib.IMAP4.error(b"[AUTHENTICATIONFAILED] Invalid credentials")
107 self.user = user
108 return "OK", [b"logged in"]
109
110 def logout(self):
111 self._log("LOGOUT")
112 return "BYE", [b""]
113
114 def list(self, directory='""', pattern="*"):
115 self._log("LIST")
116 lines = []
117 for (user, folder) in self.server.boxes:
118 if user != self.user:
119 continue
120 attrs = "\\HasNoChildren \\Sent" if folder == self.server.sent_folder else "\\HasNoChildren"
121 lines.append(f'({attrs}) "/" "{folder}"'.encode())
122 return "OK", lines
123
124 def select(self, mailbox, readonly=False):
125 name = mailbox[1:-1] if mailbox.startswith('"') else mailbox
126 self._log("EXAMINE" if readonly else "SELECT", name)
127 key = (self.user, name)
128 if key not in self.server.boxes:
129 return "NO", [b"Mailbox does not exist"]
130 self.box = key
131 self.readonly = readonly
132 messages = self.server.boxes[key]
133 self.pending = {"UIDVALIDITY": [str(self.server.uidvalidity[key]).encode()]}
134 if self.server.send_uidnext:
135 self.pending["UIDNEXT"] = [
136 str(max((m["uid"] for m in messages), default=0) + 1).encode()]
137 return "OK", [str(len(messages)).encode()]
138
139 def response(self, code):
140 return code, self.pending.pop(code, [None])
141
142 def _messages(self):
143 return self.server.boxes[self.box]
144
145 def uid(self, command, *args):
146 self._log("UID", command, *args)
147 command = command.upper()
148 if command == "SEARCH":
149 return "OK", [" ".join(str(u) for u in self._search(args)).encode()]
150 if command == "FETCH":
151 return self._fetch(args[0], args[1])
152 raise AssertionError(f"the sync issued UID {command}, which it must never do")
153
154 def _search(self, args):
155 messages = self._messages()
156 if args[0] == "ALL":
157 return [m["uid"] for m in messages]
158 if args[0] == "SINCE":
159 day, month, year = args[1].split("-")
160 since = datetime(int(year), object_mail._MONTHS.index(month) + 1, int(day),
161 tzinfo=timezone.utc)
162 return [m["uid"] for m in messages if m["when"] >= since]
163 if args[0] == "UID":
164 low = int(args[1].split(":")[0])
165 found = [m["uid"] for m in messages if m["uid"] >= low]
166 # Real servers answer n:* with the highest message even when n
167 # is past it. The sync must not import that message again.
168 if not found and messages:
169 found = [messages[-1]["uid"]]
170 return found
171 raise AssertionError(f"unexpected SEARCH {args}")
172
173 def _fetch(self, uid_set, items):
174 wanted = {int(u) for u in uid_set.split(",")}
175 data = []
176 for m in self._messages():
177 if m["uid"] not in wanted:
178 continue
179 if "BODY.PEEK[" in items:
180 if self.server.fail_fetch_uid == m["uid"]:
181 raise OSError("connection reset by peer")
182 raw = m["raw"]
183 if "HEADER" in items:
184 raw = raw.split(b"\r\n\r\n", 1)[0] + b"\r\n\r\n"
185 data.append((f"{m['uid']} (UID {m['uid']} BODY[] {{{len(raw)}}}".encode(), raw))
186 data.append(b")")
187 elif "BODY[" in items or "RFC822" in items.replace("RFC822.SIZE", ""):
188 # A non-PEEK body fetch sets \Seen on a real server.
189 if "\\Seen" not in m["flags"]:
190 m["flags"].append("\\Seen")
191 raise AssertionError("the sync fetched a body without PEEK")
192 else:
193 size = m["size"] if m["size"] is not None else len(m["raw"])
194 flags = " ".join(m["flags"])
195 data.append(
196 f'{m["uid"]} (UID {m["uid"]} FLAGS ({flags}) RFC822.SIZE {size} '
197 f'INTERNALDATE "{internaldate(m["when"])}")'.encode())
198 return "OK", data
199
200
201def mail(message_id=None, *, sender="alice@example.org", to="me@dove.test", subject="Hello",
202 body="Hi there.", in_reply_to=None, references=None, extra=""):
203 lines = [f"From: Alice <{sender}>", f"To: {to}", f"Subject: {subject}",
204 f"Date: {format_datetime(NOW - timedelta(hours=2))}"]
205 if message_id:
206 lines.append(f"Message-ID: {message_id}")
207 if in_reply_to:
208 lines.append(f"In-Reply-To: {in_reply_to}")
209 if references:
210 lines.append(f"References: {references}")
211 if extra:
212 lines.append(extra)
213 lines += ["MIME-Version: 1.0", "Content-Type: text/plain; charset=utf-8", "", body, ""]
214 return "\r\n".join(lines).encode("utf-8")
215
216
217# --- the box -------------------------------------------------------------------------
218
219@pytest.fixture
220def box(tmp_path, monkeypatch):
221 data_dir = tmp_path / "data"
222 for name in ("message_threads", "messages", "message_recipients"):
223 stage_collection(data_dir, "app-messaging", name)
224 for name in ("mail_accounts", "mail_sync_state", "mail_synced_messages"):
225 stage_collection(data_dir, "app-mail-sync", name)
226 objects_root = tmp_path / "objects"
227 shutil.copytree(PACKAGE / "objects", objects_root)
228
229 server = FakeServer()
230
231 def connect(*args, **kwargs):
232 return server.connect(*args, **kwargs)
233
234 # The object names imaplib.IMAP4.error / .abort; the stand-in keeps them.
235 connect.error, connect.abort = imaplib.IMAP4.error, imaplib.IMAP4.abort
236 monkeypatch.setattr(imaplib, "IMAP4_SSL", connect)
237 monkeypatch.setattr(imaplib, "IMAP4", connect)
238 addresses = {"host": "1.1.1.1"}
239 monkeypatch.setattr(socket, "getaddrinfo", lambda host, port, *a, **k: [
240 (socket.AF_INET, socket.SOCK_STREAM, 6, "", (addresses["host"], port))])
241 monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir))
242 monkeypatch.setenv("DBBASIC_ENABLE_MAIL_SYNC", "true")
243 monkeypatch.delenv("DBBASIC_MAIL_SYNC_ALLOW_PRIVATE_HOSTS", raising=False)
244
245 class Box:
246 pass
247
248 b = Box()
249 b.data, b.objects, b.server, b.addresses = data_dir, objects_root, server, addresses
250 return b
251
252
253def account(box, account_id="acct-1", *, owner="u1", user="me@dove.test", password=PASSWORD,
254 **fields):
255 row = {"id": account_id, "label": "Work", "email_address": user,
256 "host": "imap.example.test", "security": "ssl", "username": user,
257 "owner_id": owner}
258 row.update(fields)
259 object_records.create_collection_record("mail_accounts", row, base_dir=box.data, actor="test")
260 if password is not None:
261 object_service_keys.set_service_key(owner, f"imap-{account_id}", password,
262 base_dir=box.data)
263 return row
264
265
266def run(box, method="POST", object_id="system_mail_sync"):
267 outcome = object_execution.execute_object(
268 RUNTIME, object_execution.ObjectExecutionRequest(object_id, method=method, payload={}),
269 roots=[box.objects])
270 assert outcome.ok, outcome.error.to_dict() if outcome.error else outcome
271 return outcome.result
272
273
274def rows(box, collection):
275 return object_records.read_collection_records(collection, base_dir=box.data)
276
277
278def acct(box, account_id="acct-1"):
279 return object_records.get_collection_record("mail_accounts", account_id, base_dir=box.data)
280
281
282def retry_now(box, account_id="acct-1"):
283 """What the owner does after fixing something: zero the failure count."""
284 object_records.update_collection_record("mail_accounts", account_id,
285 {"sync_attempts": "0"}, base_dir=box.data,
286 actor="u1")
287
288
289def change_count(box, collection):
290 path = box.data / "record_changes" / collection / "changes.jsonl"
291 return len(path.read_text().splitlines()) if path.exists() else 0
292
293
294# --- off unless the operator says so -----------------------------------------------------
295
296def test_the_pass_is_off_until_the_operator_opts_in_and_says_which_variable(box, monkeypatch):
297 monkeypatch.delenv("DBBASIC_ENABLE_MAIL_SYNC")
298 account(box)
299 result = run(box)
300 assert "DBBASIC_ENABLE_MAIL_SYNC" in result["skipped"]
301 assert box.server.connections == 0
302
303
304def test_the_kill_switch_stops_it_without_a_redeploy(box):
305 stage_collection(box.data, "app-settings", "feature_flags")
306 object_records.create_collection_record(
307 "feature_flags", {"id": "f1", "flag": "mail_sync_enabled", "value": "off"},
308 base_dir=box.data, actor="test")
309 account(box)
310 result = run(box)
311 assert "mail_sync_enabled" in result["skipped"]
312 assert box.server.connections == 0
313
314
315def test_an_account_without_a_password_is_skipped_not_failed(box):
316 """Not set up is not a failure: nothing counts against the account, so
317 setting the password later just works."""
318 account(box, password=None)
319 result = run(box)
320 assert result["declined"] == 1
321 assert "imap-acct-1" in result["results"][0]["declined"]
322 assert box.server.connections == 0
323 assert acct(box)["sync_attempts"] in ("", "0")
324
325
326# --- copying without touching the mailbox -------------------------------------------------
327
328def test_mail_is_copied_and_never_marked_read(box):
329 """The predecessor's worst habit: syncing marked everything read, and
330 mail read elsewhere was never synced. Here the folder is EXAMINEd, the
331 body is PEEKed, and is_read is whatever the server said."""
332 box.server.user("me@dove.test")
333 box.server.deliver("me@dove.test", mail("<m1@x>", subject="Unread one"))
334 box.server.deliver("me@dove.test", mail("<m2@x>", subject="Read elsewhere"), flags=["\\Seen"])
335 account(box)
336
337 result = run(box)
338 assert result["imported"] == 2, result
339
340 messages = {m["subject"]: m for m in rows(box, "messages")}
341 assert messages["Unread one"]["is_read"] == "false"
342 assert messages["Read elsewhere"]["is_read"] == "true"
343 assert all(m["owner_id"] == "u1" and m["direction"] == "inbound" for m in messages.values())
344 flags = [m["flags"] for m in box.server.boxes[("me@dove.test", "INBOX")]]
345 assert flags == [[], ["\\Seen"]]
346 assert not any(c.startswith("SELECT") for c in box.server.commands)
347 fetches = [c for c in box.server.commands if c.startswith("UID FETCH") and "BODY" in c]
348 assert fetches and all("BODY.PEEK[" in c for c in fetches)
349 for verb in ("STORE", "APPEND", "EXPUNGE", "COPY", "MOVE"):
350 assert not any(verb in c.split()[:2] for c in box.server.commands), verb
351
352
353def test_recipients_threads_and_transport_facts_land_in_their_own_places(box):
354 box.server.user("me@dove.test")
355 box.server.deliver("me@dove.test", mail(
356 "<m1@x>", to="me@dove.test, Bob <bob@example.org>",
357 extra="Cc: carol@example.org\r\nList-Id: <news.example.org>\r\n"
358 "List-Unsubscribe: <https://example.org/unsub?u=1>"))
359 account(box)
360 run(box)
361
362 message = rows(box, "messages")[0]
363 recipients = sorted((r["kind"], r["address"]) for r in rows(box, "message_recipients"))
364 assert recipients == [("cc", "carol@example.org"), ("to", "bob@example.org"),
365 ("to", "me@dove.test")]
366 thread = rows(box, "message_threads")[0]
367 assert thread["id"] == message["thread_id"] and thread["message_count"] == "1"
368 assert thread["is_read"] == "false" and thread["owner_id"] == "u1"
369 synced = rows(box, "mail_synced_messages")[0]
370 assert synced["message_id"] == message["id"] and synced["folder"] == "INBOX"
371 assert synced["list_unsubscribe"] == "<https://example.org/unsub?u=1>"
372 assert synced["list_id"] == "<news.example.org>"
373
374
375def test_the_first_sync_takes_a_recent_window_not_the_whole_history(box):
376 box.server.user("me@dove.test")
377 box.server.deliver("me@dove.test", mail("<old@x>", subject="Old"), when=NOW - timedelta(days=90))
378 box.server.deliver("me@dove.test", mail("<new@x>", subject="New"))
379 account(box)
380 run(box)
381 assert [m["subject"] for m in rows(box, "messages")] == ["New"]
382
383
384def test_folders_are_found_on_the_first_sync_and_written_back(box):
385 box.server.user("me@dove.test")
386 account(box)
387 run(box)
388 assert acct(box)["folders"] == "INBOX, Sent"
389 roles = {s["folder"]: s["role"] for s in rows(box, "mail_sync_state")}
390 assert roles == {"INBOX": "inbox", "Sent": "sent"}
391
392
393def test_a_pass_copies_at_most_fifty_messages_and_the_next_one_continues(box):
394 """The pass runs inside the daemon's loop: a backlog drains over
395 several passes instead of one pass holding everything else up."""
396 box.server.user("me@dove.test")
397 for n in range(60):
398 box.server.deliver("me@dove.test", mail(f"<m{n}@x>", subject=f"n{n}"))
399 account(box)
400 assert run(box)["imported"] == 50
401 assert run(box)["imported"] == 10
402 assert len(rows(box, "messages")) == 60
403
404
405def test_a_huge_message_arrives_as_headers_and_says_so(box):
406 box.server.user("me@dove.test")
407 box.server.deliver("me@dove.test", mail("<big@x>", subject="Big", body="body text"),
408 size=40 * 1024 * 1024)
409 account(box)
410 run(box)
411 message = rows(box, "messages")[0]
412 assert message["subject"] == "Big"
413 assert "only its headers were copied" in message["body_text"]
414 assert any("BODY.PEEK[HEADER]" in c for c in box.server.commands)
415
416
417def test_every_connection_has_a_timeout(box):
418 box.server.user("me@dove.test")
419 account(box)
420 run(box)
421 assert box.server.timeouts and all(t and t > 0 for t in box.server.timeouts)
422
423
424# --- dedupe and at-least-once -------------------------------------------------------------
425
426def test_running_twice_copies_nothing_twice_and_an_idle_pass_writes_nothing(box):
427 box.server.user("me@dove.test")
428 box.server.deliver("me@dove.test", mail("<m1@x>"))
429 account(box)
430 run(box)
431 before = {c: change_count(box, c) for c in ("mail_accounts", "messages", "mail_sync_state")}
432 second = run(box)
433 assert second["imported"] == 0 and second["results"][0].get("up_to_date") is True
434 after = {c: change_count(box, c) for c in ("mail_accounts", "messages", "mail_sync_state")}
435 assert after == before, "an account with nothing new must not rewrite its rows"
436 assert len(rows(box, "messages")) == 1
437
438
439def test_a_server_without_uidnext_does_not_hand_back_its_newest_message_forever(box):
440 """UID SEARCH UID n:* answers with the highest message even when n is
441 past it. Without the filter every pass would download that message
442 again (dedupe would drop it, but the download is the cost)."""
443 box.server.send_uidnext = False
444 box.server.user("me@dove.test")
445 box.server.deliver("me@dove.test", mail("<m1@x>"))
446 account(box)
447 run(box)
448 box.server.commands.clear()
449 run(box)
450 assert not [c for c in box.server.commands if "BODY.PEEK" in c]
451 assert len(rows(box, "messages")) == 1
452
453
454def test_a_pass_that_dies_half_way_loses_nothing_and_duplicates_nothing(box):
455 """The cursor moves only past rows that are written; the retry
456 re-meets nothing it already wrote because every id is derived from the
457 message."""
458 box.server.user("me@dove.test")
459 for n in range(3):
460 box.server.deliver("me@dove.test", mail(f"<m{n}@x>", subject=f"n{n}"))
461 account(box)
462 box.server.fail_fetch_uid = 2
463 first = run(box)
464 assert first["failed"] == 1
465 assert [m["subject"] for m in rows(box, "messages")] == ["n0"]
466 inbox = next(s for s in rows(box, "mail_sync_state") if s["folder"] == "INBOX")
467 assert inbox["last_uid"] == "1"
468 assert acct(box)["sync_attempts"] == "1"
469
470 box.server.fail_fetch_uid = None
471 retry_now(box)
472 run(box)
473 assert sorted(m["subject"] for m in rows(box, "messages")) == ["n0", "n1", "n2"]
474 assert acct(box)["sync_attempts"] == "0" and acct(box)["sync_error"] == ""
475
476
477def test_replaying_a_batch_whose_cursor_never_moved_writes_nothing_new(box):
478 """The worst crash: every row written, the cursor not. Same ids, so the
479 replay is refused row by row and nothing is doubled."""
480 box.server.user("me@dove.test")
481 box.server.deliver("me@dove.test", mail("<m1@x>"))
482 box.server.deliver("me@dove.test", mail("<m2@x>", in_reply_to="<m1@x>"))
483 account(box)
484 run(box)
485 inbox = next(s for s in rows(box, "mail_sync_state") if s["folder"] == "INBOX")
486 object_records.update_collection_record("mail_sync_state", inbox["id"], {"last_uid": "0"},
487 base_dir=box.data, actor="test")
488 counts = {c: len(rows(box, c)) for c in ("messages", "message_threads",
489 "message_recipients", "mail_synced_messages")}
490 run(box)
491 assert {c: len(rows(box, c)) for c in counts} == counts
492 assert rows(box, "message_threads")[0]["message_count"] == "2"
493
494
495def test_a_uidvalidity_change_restarts_the_folder_without_duplicates(box):
496 box.server.user("me@dove.test")
497 box.server.deliver("me@dove.test", mail("<m1@x>"))
498 account(box)
499 run(box)
500 box.server.uidvalidity[("me@dove.test", "INBOX")] = 99
501 box.server.deliver("me@dove.test", mail("<m2@x>"))
502 run(box)
503 assert sorted(m["external_message_id"] for m in rows(box, "messages")) == ["<m1@x>", "<m2@x>"]
504 inbox = next(s for s in rows(box, "mail_sync_state") if s["folder"] == "INBOX")
505 assert inbox["uidvalidity"] == "99"
506
507
508# --- the tenant boundary ---------------------------------------------------------------------
509
510def test_the_same_message_in_two_peoples_mailboxes_is_two_rows(box):
511 """The predecessor deduped on Message-ID alone and made one row of it."""
512 for user in ("me@dove.test", "alice@dove.test"):
513 box.server.user(user)
514 box.server.deliver(user, mail("<shared@x>", to="me@dove.test, alice@dove.test"))
515 account(box, "acct-1", owner="u1", user="me@dove.test")
516 account(box, "acct-2", owner="u2", user="alice@dove.test")
517 run(box)
518 messages = rows(box, "messages")
519 assert sorted(m["owner_id"] for m in messages) == ["u1", "u2"]
520 assert len({m["thread_id"] for m in messages}) == 2
521 threads = {t["id"]: t["owner_id"] for t in rows(box, "message_threads")}
522 assert all(threads[m["thread_id"]] == m["owner_id"] for m in messages)
523 recipients = rows(box, "message_recipients")
524 owner_of = {m["id"]: m["owner_id"] for m in messages}
525 assert all(r["owner_id"] == owner_of[r["message_id"]] for r in recipients)
526
527
528def test_a_reply_never_joins_another_persons_thread(box):
529 """u1 holds the original; u2 receives only a reply to it. u2's reply
530 must start u2's own thread."""
531 box.server.user("me@dove.test")
532 box.server.user("alice@dove.test")
533 box.server.deliver("me@dove.test", mail("<orig@x>", subject="Plans"))
534 account(box, "acct-1", owner="u1", user="me@dove.test")
535 account(box, "acct-2", owner="u2", user="alice@dove.test")
536 run(box)
537 box.server.deliver("alice@dove.test", mail("<reply@x>", subject="Re: Plans",
538 in_reply_to="<orig@x>", references="<orig@x>"))
539 retry_now(box, "acct-2")
540 run(box)
541 by_owner = {m["owner_id"]: m for m in rows(box, "messages")}
542 assert by_owner["u1"]["thread_id"] != by_owner["u2"]["thread_id"]
543 thread = object_records.get_collection_record("message_threads", by_owner["u2"]["thread_id"],
544 base_dir=box.data)
545 assert thread["owner_id"] == "u2"
546
547
548# --- threading ----------------------------------------------------------------------------------
549
550def test_a_reply_joins_the_sent_message_it_answers_and_a_subject_joins_nothing(box):
551 box.server.user("me@dove.test")
552 box.server.deliver("me@dove.test", mail("<out@x>", sender="me@dove.test", to="bob@example.org",
553 subject="Invoice"), folder="Sent")
554 box.server.deliver("me@dove.test", mail("<back@x>", sender="bob@example.org",
555 subject="Re: Invoice", in_reply_to="<out@x>"))
556 box.server.deliver("me@dove.test", mail("<other@x>", sender="eve@example.org",
557 subject="Invoice"))
558 account(box)
559 run(box)
560 messages = {m["external_message_id"]: m for m in rows(box, "messages")}
561 assert messages["<out@x>"]["direction"] == "outbound"
562 assert messages["<out@x>"]["is_read"] == "true"
563 assert messages["<back@x>"]["thread_id"] == messages["<out@x>"]["thread_id"]
564 assert messages["<other@x>"]["thread_id"] != messages["<out@x>"]["thread_id"]
565 thread = object_records.get_collection_record(
566 "message_threads", messages["<out@x>"]["thread_id"], base_dir=box.data)
567 assert thread["message_count"] == "2"
568 assert thread["is_read"] == "false", "an unread reply makes its thread unread"
569
570
571# --- failures are written down --------------------------------------------------------------
572
573def test_a_refused_login_is_recorded_without_the_password_and_backs_off(box):
574 box.server.user("me@dove.test", password="something else")
575 account(box)
576 result = run(box)
577 assert result["failed"] == 1
578 row = acct(box)
579 assert row["sync_attempts"] == "1"
580 assert "refused the login" in row["sync_error"]
581 assert PASSWORD not in json.dumps(result) and PASSWORD not in row["sync_error"]
582
583 connections = box.server.connections
584 again = run(box)
585 assert box.server.connections == connections, "a failing account waits before the next try"
586 assert "waiting" in again["results"][0]
587
588
589def test_three_failures_in_a_row_are_counted_on_the_home_page(box):
590 box.server.user("me@dove.test", password="something else")
591 account(box)
592 assert run(box, "COUNT", "system_mail_sync_attention") == {"count": 0}
593 for _ in range(3):
594 retry_now_keep = acct(box)["sync_attempts"]
595 object_records.update_collection_record(
596 "mail_accounts", "acct-1", {"sync_finished_at": "2000-01-01T00:00:00Z"},
597 base_dir=box.data, actor="test", preserve_read_only=True)
598 run(box)
599 assert int(acct(box)["sync_attempts"]) == int(retry_now_keep or 0) + 1
600 counted = run(box, "COUNT", "system_mail_sync_attention")
601 assert counted["count"] == 1
602 assert "me@dove.test" not in json.dumps(counted), "the home page count names nobody"
603
604
605def test_a_missing_folder_is_an_error_on_the_account_not_a_silent_skip(box):
606 box.server.user("me@dove.test")
607 box.server.deliver("me@dove.test", mail("<m1@x>"))
608 account(box, folders="INBOX, Archive")
609 run(box)
610 row = acct(box)
611 assert row["sync_attempts"] == "1" and "Archive" in row["sync_error"]
612 assert len(rows(box, "messages")) == 1, "the folder that works still syncs"
613
614
615def test_a_private_address_is_refused_unless_the_operator_allows_it(box, monkeypatch):
616 """The host is typed by a member and the server is the one connecting:
617 without this, anyone could make it knock on its own network."""
618 box.server.user("me@dove.test")
619 box.addresses["host"] = "10.0.0.5"
620 account(box)
621 run(box)
622 assert box.server.connections == 0
623 assert "DBBASIC_MAIL_SYNC_ALLOW_PRIVATE_HOSTS" in acct(box)["sync_error"]
624
625 monkeypatch.setenv("DBBASIC_MAIL_SYNC_ALLOW_PRIVATE_HOSTS", "true")
626 retry_now(box)
627 run(box)
628 assert box.server.connections == 1
629 assert acct(box)["sync_error"] == ""
630
631
632# --- what it never does ------------------------------------------------------------------------
633
634def test_it_never_fetches_a_url_from_a_header_and_never_sends(box, monkeypatch):
635 """List-Unsubscribe is stored for a person to act on; the server opening
636 it would let any sender make it issue requests."""
637 import http.client
638 import smtplib
639 import urllib.request
640
641 def refuse(*args, **kwargs):
642 raise AssertionError("the mail sync opened a network connection it must not")
643
644 monkeypatch.setattr(urllib.request, "urlopen", refuse)
645 monkeypatch.setattr(http.client.HTTPConnection, "connect", refuse)
646 monkeypatch.setattr(smtplib, "SMTP", refuse)
647 monkeypatch.setattr(smtplib, "SMTP_SSL", refuse)
648 box.server.user("me@dove.test")
649 box.server.deliver("me@dove.test", mail(
650 "<m1@x>", extra="List-Unsubscribe: <https://example.org/u>\r\nList-Unsubscribe-Post: "
651 "List-Unsubscribe=One-Click"))
652 account(box)
653 assert run(box)["imported"] == 1
654 source = SOURCE.read_text() + (REPO_ROOT / "object_mail.py").read_text()
655 for module in ("urllib", "http.client", "smtplib", "requests"):
656 assert f"import {module}" not in source
657
658
659def test_the_scheduled_entry_point_is_event_with_post_as_its_alias():
660 """AGENTS.md: the dispatcher calls EVENT; an object with only POST never
661 runs and never complains."""
662 source = SOURCE.read_text()
663 assert "def EVENT(request)" in source and "POST = EVENT" in source
664
665
666# --- the page ---------------------------------------------------------------------------------
667
668def page(box, user_id=None):
669 payload = {"_identity": {"user_id": user_id}} if user_id else {}
670 outcome = object_execution.execute_object(
671 RUNTIME, object_execution.ObjectExecutionRequest("site_mail_accounts", method="GET",
672 payload=payload),
673 roots=[box.objects])
674 assert outcome.ok, outcome.error.to_dict() if outcome.error else outcome
675 return outcome.result["body"]
676
677
678def test_the_page_shows_whether_a_password_is_set_and_never_the_password(box):
679 account(box, "acct-1", owner="u1", label="Work")
680 account(box, "acct-2", owner="u1", label="Home", password=None)
681 account(box, "acct-3", owner="u2", label="Someone else's")
682 body = page(box, "u1")
683 assert "Work" in body and "Home" in body
684 assert "Someone else" not in body, "only the signed-in person's accounts"
685 assert body.count("<strong data-pwstate>set</strong>") == 1
686 assert body.count("<strong data-pwstate>not set</strong>") == 1
687 assert PASSWORD not in body
688 assert "/service-keys" in body and '"imap-" + id' in body
689
690
691def test_the_page_says_when_the_operator_has_not_turned_sync_on(box, monkeypatch):
692 monkeypatch.delenv("DBBASIC_ENABLE_MAIL_SYNC")
693 account(box)
694 assert "DBBASIC_ENABLE_MAIL_SYNC" in page(box, "u1")
695
696
697def test_a_visitor_gets_a_sign_in_prompt(box):
698 account(box)
699 body = page(box)
700 assert "Sign in" in body and "Work" not in body
701
702
703# --- a real server, when one is named -------------------------------------------------------------
704
705REAL = os.environ.get("DBBASIC_TEST_IMAP", "")
706
707
708@pytest.mark.skipif(not REAL, reason="set DBBASIC_TEST_IMAP=host:port:user:password to run")
709def test_against_a_real_imap_server(tmp_path, monkeypatch):
710 """Plain IMAP on this machine (a GreenMail or Dovecot test server): the
711 sync copies whatever the window holds and leaves every flag as it was."""
712 host, port, user, password = REAL.split(":", 3)
713 data_dir = tmp_path / "data"
714 for name in ("message_threads", "messages", "message_recipients"):
715 stage_collection(data_dir, "app-messaging", name)
716 for name in ("mail_accounts", "mail_sync_state", "mail_synced_messages"):
717 stage_collection(data_dir, "app-mail-sync", name)
718 objects_root = tmp_path / "objects"
719 shutil.copytree(PACKAGE / "objects", objects_root)
720 monkeypatch.setenv("DBBASIC_DATA_DIR", str(data_dir))
721 monkeypatch.setenv("DBBASIC_ENABLE_MAIL_SYNC", "true")
722 monkeypatch.setenv("DBBASIC_MAIL_SYNC_ALLOW_PRIVATE_HOSTS", "true")
723
724 def flags():
725 conn = imaplib.IMAP4(host, int(port), timeout=10)
726 conn.login(user, password)
727 conn.select("INBOX", readonly=True)
728 typ, data = conn.fetch("1:*", "(UID FLAGS)")
729 conn.logout()
730 return sorted(object_mail._as_text(d) for d in data if d)
731
732 before = flags()
733 object_records.create_collection_record("mail_accounts", {
734 "id": "real-1", "email_address": user if "@" in user else f"{user}@localhost",
735 "host": host, "port": port, "security": "plain", "username": user,
736 "owner_id": "u1"}, base_dir=data_dir, actor="test")
737 object_service_keys.set_service_key("u1", "imap-real-1", password, base_dir=data_dir)
738 result = object_execution.execute_object(
739 RUNTIME, object_execution.ObjectExecutionRequest("system_mail_sync", method="POST",
740 payload={}),
741 roots=[objects_root]).result
742 assert result["failed"] == 0, result
743 assert flags() == before, "the sync changed a flag on the server"
744 assert len(object_records.read_collection_records("messages", base_dir=data_dir)) >= 1