Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.

0 downloads

dbbasic-object-server/tests/test_object_mail.py
Download zip
Clone
Clone with git
git clone https://gitstable.com/git/askrobots/dbbasic-object-server.git
HTTPS
https://gitstable.com/git/askrobots/dbbasic-object-server.git
SSH
ssh://git@gitstable.com:2222/askrobots/dbbasic-object-server.git

Cloning needs no sign-in. Pushing over HTTPS asks for your username and an access token. SSH needs your key under SSH keys.

danq9 signup: the person chooses the password and name on the confirm page 83c7aa6 2 weeks, 5 days ago History
test_object_mail.py · 16.5 KB · 325 lines
1"""object_mail: the pure half of inbound mail -- parsing, dedupe, threading,
2cursor planning -- tested with no server, no clock and no data directory.
3
4The object-level claims (never marks read, never crosses owners, never
5loses a message to a crash) are in tests/test_mail_sync.py; these are the
6decisions underneath them, one at a time.
7"""
8
9from datetime import datetime, timedelta, timezone
10
11import object_mail
12
13NOW = datetime(2026, 9, 26, 12, 0, tzinfo=timezone.utc)
14
15
16def raw(*headers, body="Hello.", content_type="text/plain; charset=utf-8"):
17 head = "\r\n".join(headers + ("MIME-Version: 1.0", f"Content-Type: {content_type}"))
18 return (head + "\r\n\r\n" + body + "\r\n").encode("utf-8")
19
20
21# --- parsing -----------------------------------------------------------------------
22
23def test_encoded_headers_are_decoded_and_dates_use_the_server_spelling():
24 parsed = object_mail.parse_message(raw(
25 "From: =?utf-8?q?J=C3=B6rg?= <jorg@example.org>",
26 'To: me@dove.test, "Doe, Jane" <jane@example.org>',
27 "Subject: =?utf-8?q?Gr=C3=BC=C3=9Fe?=",
28 "Date: Fri, 25 Sep 2026 10:00:00 +0200",
29 "Message-ID: <abc@example.org>"), internaldate="25-Sep-2026 08:00:01 +0000")
30 assert parsed["subject"] == "Grüße"
31 assert parsed["from_name"] == "Jörg" and parsed["from_address"] == "jorg@example.org"
32 assert parsed["to"] == ["me@dove.test", "jane@example.org"]
33 assert parsed["date"] == "2026-09-25T08:00:00Z"
34 assert parsed["internaldate"] == "2026-09-25T08:00:01Z"
35 assert parsed["message_id"] == "<abc@example.org>"
36
37
38def test_plain_text_is_preferred_and_html_is_turned_into_text_when_alone():
39 alternative = (
40 "--b\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nPlain words\r\n"
41 "--b\r\nContent-Type: text/html; charset=utf-8\r\n\r\n<p>Html words</p>\r\n--b--")
42 parsed = object_mail.parse_message(raw(
43 "From: a@example.org", body=alternative,
44 content_type="multipart/alternative; boundary=b"))
45 assert parsed["body_text"] == "Plain words"
46
47 only_html = object_mail.parse_message(raw(
48 "From: a@example.org",
49 body="<html><head><style>p{color:red}</style></head><body><p>One&nbsp;<b>two</b></p>"
50 "<script>alert(1)</script><ul><li>three</li></ul></body></html>",
51 content_type="text/html; charset=utf-8"))
52 assert only_html["body_text"] == "One two\n\n- three"
53
54
55def test_attachments_are_named_not_copied():
56 body = ("--b\r\nContent-Type: text/plain\r\n\r\nSee attached.\r\n"
57 "--b\r\nContent-Type: application/pdf\r\nContent-Disposition: attachment; "
58 "filename=invoice.pdf\r\nContent-Transfer-Encoding: base64\r\n\r\nJVBERi0=\r\n--b--")
59 parsed = object_mail.parse_message(raw("From: a@example.org", body=body,
60 content_type="multipart/mixed; boundary=b"))
61 assert parsed["body_text"] == "See attached.\n\n[Attachments not copied: invoice.pdf]"
62
63
64def test_broken_bytes_become_storable_text_instead_of_a_failed_write():
65 """A raw 8-bit subject, an unknown charset and a NUL must not stop the
66 message from being stored; the substrate refuses NUL and lone surrogates."""
67 message = (b"From: a@example.org\r\nSubject: caf\xe9 \x00menu\r\n"
68 b"Content-Type: text/plain; charset=x-unknown\r\n\r\nbody \xff\x00 text\r\n")
69 parsed = object_mail.parse_message(message)
70 for value in (parsed["subject"], parsed["body_text"]):
71 value.encode("utf-8")
72 assert "\x00" not in value
73 assert parsed["subject"].startswith("caf")
74 assert "text" in parsed["body_text"]
75
76
77def test_a_broken_date_is_left_blank_not_guessed():
78 parsed = object_mail.parse_message(raw("From: a@example.org", "Date: yesterday-ish"))
79 assert parsed["date"] == ""
80 assert object_mail.parse_internaldate("not a date") == ""
81 assert object_mail.parse_internaldate(" 1-Jan-2026 00:00:00 -0500") == "2026-01-01T05:00:00Z"
82
83
84def test_a_long_body_is_cut_and_says_so():
85 parsed = object_mail.parse_message(raw("From: a@example.org",
86 body="x" * (object_mail.MAX_BODY_CHARS + 50)))
87 assert len(parsed["body_text"]) < object_mail.MAX_BODY_CHARS + 200
88 assert "the whole message is on the mail server" in parsed["body_text"]
89
90
91def test_a_header_only_copy_says_how_big_the_message_was():
92 parsed = object_mail.parse_message(raw("From: a@example.org"),
93 header_only_size=12 * 1024 * 1024)
94 assert "12.0 MB" in parsed["body_text"] and "only its headers" in parsed["body_text"]
95
96
97def test_is_read_is_the_servers_seen_flag():
98 assert object_mail.parse_message(raw("From: a@x.org"), flags="\\Seen \\Flagged")["is_read"]
99 assert not object_mail.parse_message(raw("From: a@x.org"), flags=["\\Flagged"])["is_read"]
100
101
102def test_list_headers_are_kept_as_text():
103 parsed = object_mail.parse_message(raw(
104 "From: a@example.org", "List-Unsubscribe: <https://example.org/u>, <mailto:u@example.org>",
105 "List-Unsubscribe-Post: List-Unsubscribe=One-Click", "List-Id: News <news.example.org>",
106 "Precedence: bulk"))
107 assert parsed["list_unsubscribe"] == "<https://example.org/u>, <mailto:u@example.org>"
108 assert parsed["list_unsubscribe_post"] == "List-Unsubscribe=One-Click"
109 assert parsed["list_id"] == "News <news.example.org>"
110 assert parsed["precedence"] == "bulk"
111
112
113# --- dedupe and ids -------------------------------------------------------------------
114
115def test_the_dedupe_key_is_the_message_id_or_a_hash_without_one():
116 with_id = object_mail.parse_message(raw("From: a@x.org", "Message-ID: <k@x>"))
117 assert object_mail.dedupe_key(with_id) == "mid:<k@x>"
118 without = object_mail.parse_message(raw("From: a@x.org", "Subject: s", "Date: Fri, 25 Sep 2026 10:00:00 +0000"))
119 key = object_mail.dedupe_key(without)
120 assert key.startswith("sha256:") and key == object_mail.dedupe_key(dict(without))
121 other = object_mail.parse_message(raw("From: b@x.org", "Subject: s", "Date: Fri, 25 Sep 2026 10:00:00 +0000"))
122 assert object_mail.dedupe_key(other) != key
123
124
125def test_ids_are_the_same_for_the_same_facts_and_differ_by_owner():
126 """The replay safety and the tenant boundary are the same property."""
127 assert object_mail.stable_id("message", "u1", "a1", "mid:<k@x>") == \
128 object_mail.stable_id("message", "u1", "a1", "mid:<k@x>")
129 assert object_mail.stable_id("message", "u1", "a1", "mid:<k@x>") != \
130 object_mail.stable_id("message", "u2", "a1", "mid:<k@x>")
131
132
133# --- rows --------------------------------------------------------------------------------
134
135def test_message_rows_fit_the_app_messaging_schema():
136 parsed = object_mail.parse_message(raw(
137 "From: a@example.org", "To: me@dove.test, ME@dove.test", "Cc: c@example.org",
138 "Subject: " + "s" * 400, "Message-ID: <k@x>"), internaldate="25-Sep-2026 08:00:00 +0000")
139 row, recipients = object_mail.message_rows(parsed, owner_id="u1", message_id="m1",
140 thread_id="t1", direction="inbound")
141 assert len(row["subject"]) == object_mail.MAX_SUBJECT
142 assert row["is_read"] == "false" and row["received_at"] == "2026-09-25T08:00:00Z"
143 assert "sent_at" not in row
144 assert [(r["kind"], r["address"]) for r in recipients] == [("to", "me@dove.test"),
145 ("cc", "c@example.org")]
146 assert {r["owner_id"] for r in recipients} == {"u1"}
147
148 outbound, _ = object_mail.message_rows(parsed, owner_id="u1", message_id="m1",
149 thread_id="t1", direction="outbound")
150 assert outbound["is_read"] == "true" and "sent_at" in outbound
151
152
153def test_a_message_with_no_from_says_so():
154 parsed = object_mail.parse_message(raw("Subject: who"))
155 row, _ = object_mail.message_rows(parsed, owner_id="u1", message_id="m", thread_id="t",
156 direction="inbound")
157 assert row["from_address"] == object_mail.NO_SENDER
158
159
160def test_direction_is_the_sent_folder_or_the_accounts_own_address():
161 parsed = {"from_address": "Me@Dove.test"}
162 assert object_mail.direction_for(parsed, account_address="me@dove.test", folder_role="inbox") == "outbound"
163 assert object_mail.direction_for({"from_address": "x@y"}, account_address="me@dove.test",
164 folder_role="sent") == "outbound"
165 assert object_mail.direction_for({"from_address": "x@y"}, account_address="me@dove.test",
166 folder_role="inbox") == "inbound"
167
168
169def test_a_late_old_message_does_not_make_a_thread_look_fresh():
170 thread = {"message_count": "3", "last_message_at": "2026-09-25T10:00:00Z",
171 "participant_summary": "a@example.org"}
172 older = {"internaldate": "2026-09-01T10:00:00Z", "is_read": False,
173 "from_address": "b@example.org"}
174 changes = object_mail.thread_changes(thread, older, direction="inbound",
175 account_address="me@dove.test")
176 assert changes["message_count"] == "4"
177 assert "last_message_at" not in changes
178 assert changes["is_read"] == "false"
179 assert changes["participant_summary"] == "a@example.org, b@example.org"
180 newer = dict(older, internaldate="2026-09-25T10:00:01+00:00", is_read=True)
181 assert object_mail.thread_changes(thread, newer, direction="inbound",
182 account_address="me@dove.test")["last_message_at"]
183
184
185# --- threading ------------------------------------------------------------------------------
186
187def _index(owner="u1"):
188 messages = [
189 {"id": "m1", "owner_id": "u1", "thread_id": "t1", "external_message_id": "<root@x>"},
190 {"id": "m2", "owner_id": "u2", "thread_id": "t9", "external_message_id": "<theirs@x>"},
191 {"id": "m3", "owner_id": "u1", "thread_id": "t3", "external_message_id": "<kid@x>",
192 "in_reply_to": "<late-parent@x>"},
193 ]
194 synced = [{"owner_id": "u1", "message_id": "m1", "references": "<ancient@x>"}]
195 return object_mail.ThreadIndex.from_rows(owner, messages, synced)
196
197
198def test_threading_follows_in_reply_to_then_references_then_a_reply_that_came_first():
199 index = _index()
200 assert object_mail.match_thread({"in_reply_to": "<root@x>"}, index) == "t1"
201 assert object_mail.match_thread({"references": ["<nope@x>", "<root@x>"]}, index) == "t1"
202 assert object_mail.match_thread({"message_id": "<late-parent@x>"}, index) == "t3"
203 assert object_mail.match_thread({"message_id": "<ancient@x>"}, index) == "t1"
204
205
206def test_threading_never_crosses_owners_even_when_handed_their_rows():
207 assert object_mail.match_thread({"in_reply_to": "<theirs@x>"}, _index("u1")) is None
208 assert object_mail.match_thread({"in_reply_to": "<root@x>"}, _index("u2")) is None
209
210
211def test_the_subject_alone_joins_nothing():
212 index = _index()
213 index.by_message_id["<x@x>"] = "t1"
214 assert object_mail.match_thread({"subject": "Re: Invoice", "message_id": "<new@x>"}, index) is None
215
216
217# --- cursor planning ------------------------------------------------------------------------
218
219def test_plan_folder_resets_when_uidvalidity_changes():
220 assert object_mail.plan_folder(None, uidvalidity="5") == "fresh"
221 assert object_mail.plan_folder({"uidvalidity": "5"}, uidvalidity="5") == "continue"
222 assert object_mail.plan_folder({"uidvalidity": "5"}, uidvalidity="6") == "reset"
223
224
225def test_the_first_window_is_the_newest_n_and_an_empty_window_starts_at_the_top():
226 assert object_mail.window_baseline(range(1, 501), first_window=200, highest_uid=500) == 300
227 assert object_mail.window_baseline([], first_window=200, highest_uid=42) == 42
228 assert object_mail.window_baseline([7], first_window=200, highest_uid=42) == 6
229
230
231def test_next_uids_drops_the_highest_message_n_star_hands_back():
232 assert object_mail.next_uids([10], last_uid=10, limit=50) == []
233 assert object_mail.next_uids([13, 11, 12, 11], last_uid=10, limit=2) == [11, 12]
234
235
236def test_downloads_are_bounded_and_a_big_message_never_stalls_a_folder():
237 metas = [{"uid": 1, "size": 30}, {"uid": 2, "size": 500}, {"uid": 3, "size": 30},
238 {"uid": 4, "size": 30}]
239 assert object_mail.plan_downloads(metas, max_message_bytes=100, byte_budget=70) == [
240 (1, "full"), (2, "header"), (3, "full")]
241 assert object_mail.plan_downloads([{"uid": 9, "size": 90}], max_message_bytes=100,
242 byte_budget=10) == [(9, "full")]
243
244
245def test_imap_dates_do_not_depend_on_the_locale():
246 assert object_mail.imap_date(datetime(2026, 9, 6)) == "6-Sep-2026"
247
248
249# --- when an account is due -------------------------------------------------------------------
250
251def _due(account, now=NOW):
252 return object_mail.account_due(account, now, claim_ttl_seconds=900,
253 retry_base_seconds=120, retry_max_seconds=1800)
254
255
256def test_account_due_respects_disable_claims_and_backoff():
257 assert _due({"enabled": "false"}) == (False, "disabled")
258 assert _due({})[0] is True
259 held = {"sync_started_at": "2026-09-26T11:55:00Z", "sync_finished_at": "2026-09-26T11:00:00Z"}
260 assert _due(held)[0] is False
261 stale = dict(held, sync_started_at="2026-09-26T11:00:30Z", sync_finished_at="2026-09-26T11:00:00Z")
262 assert _due(stale)[0] is True, "a claim nobody finished expires"
263 failing = {"sync_attempts": "3", "sync_finished_at": "2026-09-26T11:55:00Z"}
264 assert _due(failing)[0] is False # waits 8 minutes after 3 failures
265 assert _due(failing, NOW + timedelta(minutes=4))[0] is True
266 hopeless = {"sync_attempts": "40", "sync_finished_at": "2026-09-26T11:00:00Z"}
267 assert _due(hopeless)[0] is True, "the wait stops growing at the cap"
268
269
270# --- where the server may connect -------------------------------------------------------------
271
272def test_private_and_plaintext_hosts_are_refused_by_default():
273 assert object_mail.host_refusal("imap.x", ["1.1.1.1"], security="ssl", allow_private=False) is None
274 refused = object_mail.host_refusal("db", ["10.1.2.3"], security="ssl", allow_private=False)
275 assert object_mail.ALLOW_PRIVATE_HOSTS_ENV in refused
276 assert object_mail.host_refusal("db", ["10.1.2.3"], security="ssl", allow_private=True) is None
277 assert object_mail.host_refusal("x", ["169.254.169.254"], security="ssl", allow_private=False)
278 assert object_mail.host_refusal("x", ["1.1.1.1"], security="plain", allow_private=True)
279 assert object_mail.host_refusal("localhost", ["127.0.0.1", "::1"], security="plain",
280 allow_private=True) is None
281 assert object_mail.host_refusal("x", [], security="ssl", allow_private=True)
282
283
284# --- reading IMAP answers ----------------------------------------------------------------------
285
286def test_fetch_answers_are_read_without_a_server():
287 metas = object_mail.parse_fetch_meta([
288 b'1 (UID 7 FLAGS (\\Seen \\Answered) RFC822.SIZE 1234 INTERNALDATE "25-Sep-2026 08:00:00 +0000")',
289 b'2 (UID 8 FLAGS () RFC822.SIZE 99 INTERNALDATE " 5-Sep-2026 08:00:00 +0000")'])
290 assert metas[0] == {"uid": 7, "size": 1234, "flags": ["\\Seen", "\\Answered"],
291 "internaldate": "25-Sep-2026 08:00:00 +0000"}
292 assert metas[1]["flags"] == []
293 assert object_mail.fetch_literal([(b"1 (UID 7 BODY[] {3}", b"abc"), b")"]) == b"abc"
294 assert object_mail.fetch_literal([b"1 (UID 7)"]) is None
295 assert object_mail.parse_search([b"3 4 5"]) == [3, 4, 5]
296 assert object_mail.parse_search([None]) == []
297
298
299def test_folders_are_found_by_the_sent_attribute_then_by_name():
300 entries = object_mail.parse_list([
301 b'(\\HasNoChildren) "/" "INBOX"',
302 b'(\\HasNoChildren \\Sent) "/" "Sent Items"',
303 b'(\\HasNoChildren) "/" Archive',
304 (b'(\\HasNoChildren) "/" {10}', b'Quoted "x"'),
305 ])
306 assert ([], "Archive") not in entries
307 assert (["\\HasNoChildren"], "Archive") in entries
308 assert (["\\HasNoChildren"], 'Quoted "x"') in entries
309 assert object_mail.choose_folders(entries) == [("INBOX", "inbox"), ("Sent Items", "sent")]
310 by_name = object_mail.choose_folders([([], "INBOX"), ([], "Sent")])
311 assert by_name == [("INBOX", "inbox"), ("Sent", "sent")]
312 assert object_mail.choose_folders([([], "INBOX")]) == [("INBOX", "inbox")]
313 assert object_mail.folder_role("Sent Items", entries) == "sent"
314 assert object_mail.folder_role("Archive", entries) == "other"
315 assert object_mail.quote_mailbox('a "b"') == '"a \\"b\\""'
316 assert object_mail.split_folders("INBOX, Sent, ,INBOX") == ["INBOX", "Sent"]
317
318
319def test_a_line_break_in_a_folder_name_is_refused_not_sent():
320 """Inside a quoted string a CRLF ends the command and starts another:
321 'INBOX\\r\\nA1 DELETE INBOX' would delete the person's inbox."""
322 import pytest
323
324 with pytest.raises(ValueError):
325 object_mail.quote_mailbox("INBOX\r\nA1 DELETE INBOX")