askrobots / dbbasic-object-server
Public
Runs live, versioned Python application objects on one VM, with a suite of apps: projects, notes, tasks, files and more.
danq9
signup: the person chooses the password and name on the confirm page
83c7aa6
2 weeks, 5 days ago
History
test_object_mail.py · 16.5 KB · 325 lines
| 1 | """object_mail: the pure half of inbound mail -- parsing, dedupe, threading, |
| 2 | cursor planning -- tested with no server, no clock and no data directory. |
| 3 | |
| 4 | The object-level claims (never marks read, never crosses owners, never |
| 5 | loses a message to a crash) are in tests/test_mail_sync.py; these are the |
| 6 | decisions underneath them, one at a time. |
| 7 | """ |
| 8 | |
| 9 | from datetime import datetime, timedelta, timezone |
| 10 | |
| 11 | import object_mail |
| 12 | |
| 13 | NOW = datetime(2026, 9, 26, 12, 0, tzinfo=timezone.utc) |
| 14 | |
| 15 | |
| 16 | def raw(*headers, body="Hello.", content_type="text/plain; charset=utf-8"): |
| 17 | head = "\r\n".join(headers + ("MIME-Version: 1.0", f"Content-Type: {content_type}")) |
| 18 | return (head + "\r\n\r\n" + body + "\r\n").encode("utf-8") |
| 19 | |
| 20 | |
| 21 | # --- parsing ----------------------------------------------------------------------- |
| 22 | |
| 23 | def test_encoded_headers_are_decoded_and_dates_use_the_server_spelling(): |
| 24 | parsed = object_mail.parse_message(raw( |
| 25 | "From: =?utf-8?q?J=C3=B6rg?= <jorg@example.org>", |
| 26 | 'To: me@dove.test, "Doe, Jane" <jane@example.org>', |
| 27 | "Subject: =?utf-8?q?Gr=C3=BC=C3=9Fe?=", |
| 28 | "Date: Fri, 25 Sep 2026 10:00:00 +0200", |
| 29 | "Message-ID: <abc@example.org>"), internaldate="25-Sep-2026 08:00:01 +0000") |
| 30 | assert parsed["subject"] == "Grüße" |
| 31 | assert parsed["from_name"] == "Jörg" and parsed["from_address"] == "jorg@example.org" |
| 32 | assert parsed["to"] == ["me@dove.test", "jane@example.org"] |
| 33 | assert parsed["date"] == "2026-09-25T08:00:00Z" |
| 34 | assert parsed["internaldate"] == "2026-09-25T08:00:01Z" |
| 35 | assert parsed["message_id"] == "<abc@example.org>" |
| 36 | |
| 37 | |
| 38 | def test_plain_text_is_preferred_and_html_is_turned_into_text_when_alone(): |
| 39 | alternative = ( |
| 40 | "--b\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nPlain words\r\n" |
| 41 | "--b\r\nContent-Type: text/html; charset=utf-8\r\n\r\n<p>Html words</p>\r\n--b--") |
| 42 | parsed = object_mail.parse_message(raw( |
| 43 | "From: a@example.org", body=alternative, |
| 44 | content_type="multipart/alternative; boundary=b")) |
| 45 | assert parsed["body_text"] == "Plain words" |
| 46 | |
| 47 | only_html = object_mail.parse_message(raw( |
| 48 | "From: a@example.org", |
| 49 | body="<html><head><style>p{color:red}</style></head><body><p>One <b>two</b></p>" |
| 50 | "<script>alert(1)</script><ul><li>three</li></ul></body></html>", |
| 51 | content_type="text/html; charset=utf-8")) |
| 52 | assert only_html["body_text"] == "One two\n\n- three" |
| 53 | |
| 54 | |
| 55 | def test_attachments_are_named_not_copied(): |
| 56 | body = ("--b\r\nContent-Type: text/plain\r\n\r\nSee attached.\r\n" |
| 57 | "--b\r\nContent-Type: application/pdf\r\nContent-Disposition: attachment; " |
| 58 | "filename=invoice.pdf\r\nContent-Transfer-Encoding: base64\r\n\r\nJVBERi0=\r\n--b--") |
| 59 | parsed = object_mail.parse_message(raw("From: a@example.org", body=body, |
| 60 | content_type="multipart/mixed; boundary=b")) |
| 61 | assert parsed["body_text"] == "See attached.\n\n[Attachments not copied: invoice.pdf]" |
| 62 | |
| 63 | |
| 64 | def test_broken_bytes_become_storable_text_instead_of_a_failed_write(): |
| 65 | """A raw 8-bit subject, an unknown charset and a NUL must not stop the |
| 66 | message from being stored; the substrate refuses NUL and lone surrogates.""" |
| 67 | message = (b"From: a@example.org\r\nSubject: caf\xe9 \x00menu\r\n" |
| 68 | b"Content-Type: text/plain; charset=x-unknown\r\n\r\nbody \xff\x00 text\r\n") |
| 69 | parsed = object_mail.parse_message(message) |
| 70 | for value in (parsed["subject"], parsed["body_text"]): |
| 71 | value.encode("utf-8") |
| 72 | assert "\x00" not in value |
| 73 | assert parsed["subject"].startswith("caf") |
| 74 | assert "text" in parsed["body_text"] |
| 75 | |
| 76 | |
| 77 | def test_a_broken_date_is_left_blank_not_guessed(): |
| 78 | parsed = object_mail.parse_message(raw("From: a@example.org", "Date: yesterday-ish")) |
| 79 | assert parsed["date"] == "" |
| 80 | assert object_mail.parse_internaldate("not a date") == "" |
| 81 | assert object_mail.parse_internaldate(" 1-Jan-2026 00:00:00 -0500") == "2026-01-01T05:00:00Z" |
| 82 | |
| 83 | |
| 84 | def test_a_long_body_is_cut_and_says_so(): |
| 85 | parsed = object_mail.parse_message(raw("From: a@example.org", |
| 86 | body="x" * (object_mail.MAX_BODY_CHARS + 50))) |
| 87 | assert len(parsed["body_text"]) < object_mail.MAX_BODY_CHARS + 200 |
| 88 | assert "the whole message is on the mail server" in parsed["body_text"] |
| 89 | |
| 90 | |
| 91 | def test_a_header_only_copy_says_how_big_the_message_was(): |
| 92 | parsed = object_mail.parse_message(raw("From: a@example.org"), |
| 93 | header_only_size=12 * 1024 * 1024) |
| 94 | assert "12.0 MB" in parsed["body_text"] and "only its headers" in parsed["body_text"] |
| 95 | |
| 96 | |
| 97 | def test_is_read_is_the_servers_seen_flag(): |
| 98 | assert object_mail.parse_message(raw("From: a@x.org"), flags="\\Seen \\Flagged")["is_read"] |
| 99 | assert not object_mail.parse_message(raw("From: a@x.org"), flags=["\\Flagged"])["is_read"] |
| 100 | |
| 101 | |
| 102 | def test_list_headers_are_kept_as_text(): |
| 103 | parsed = object_mail.parse_message(raw( |
| 104 | "From: a@example.org", "List-Unsubscribe: <https://example.org/u>, <mailto:u@example.org>", |
| 105 | "List-Unsubscribe-Post: List-Unsubscribe=One-Click", "List-Id: News <news.example.org>", |
| 106 | "Precedence: bulk")) |
| 107 | assert parsed["list_unsubscribe"] == "<https://example.org/u>, <mailto:u@example.org>" |
| 108 | assert parsed["list_unsubscribe_post"] == "List-Unsubscribe=One-Click" |
| 109 | assert parsed["list_id"] == "News <news.example.org>" |
| 110 | assert parsed["precedence"] == "bulk" |
| 111 | |
| 112 | |
| 113 | # --- dedupe and ids ------------------------------------------------------------------- |
| 114 | |
| 115 | def test_the_dedupe_key_is_the_message_id_or_a_hash_without_one(): |
| 116 | with_id = object_mail.parse_message(raw("From: a@x.org", "Message-ID: <k@x>")) |
| 117 | assert object_mail.dedupe_key(with_id) == "mid:<k@x>" |
| 118 | without = object_mail.parse_message(raw("From: a@x.org", "Subject: s", "Date: Fri, 25 Sep 2026 10:00:00 +0000")) |
| 119 | key = object_mail.dedupe_key(without) |
| 120 | assert key.startswith("sha256:") and key == object_mail.dedupe_key(dict(without)) |
| 121 | other = object_mail.parse_message(raw("From: b@x.org", "Subject: s", "Date: Fri, 25 Sep 2026 10:00:00 +0000")) |
| 122 | assert object_mail.dedupe_key(other) != key |
| 123 | |
| 124 | |
| 125 | def test_ids_are_the_same_for_the_same_facts_and_differ_by_owner(): |
| 126 | """The replay safety and the tenant boundary are the same property.""" |
| 127 | assert object_mail.stable_id("message", "u1", "a1", "mid:<k@x>") == \ |
| 128 | object_mail.stable_id("message", "u1", "a1", "mid:<k@x>") |
| 129 | assert object_mail.stable_id("message", "u1", "a1", "mid:<k@x>") != \ |
| 130 | object_mail.stable_id("message", "u2", "a1", "mid:<k@x>") |
| 131 | |
| 132 | |
| 133 | # --- rows -------------------------------------------------------------------------------- |
| 134 | |
| 135 | def test_message_rows_fit_the_app_messaging_schema(): |
| 136 | parsed = object_mail.parse_message(raw( |
| 137 | "From: a@example.org", "To: me@dove.test, ME@dove.test", "Cc: c@example.org", |
| 138 | "Subject: " + "s" * 400, "Message-ID: <k@x>"), internaldate="25-Sep-2026 08:00:00 +0000") |
| 139 | row, recipients = object_mail.message_rows(parsed, owner_id="u1", message_id="m1", |
| 140 | thread_id="t1", direction="inbound") |
| 141 | assert len(row["subject"]) == object_mail.MAX_SUBJECT |
| 142 | assert row["is_read"] == "false" and row["received_at"] == "2026-09-25T08:00:00Z" |
| 143 | assert "sent_at" not in row |
| 144 | assert [(r["kind"], r["address"]) for r in recipients] == [("to", "me@dove.test"), |
| 145 | ("cc", "c@example.org")] |
| 146 | assert {r["owner_id"] for r in recipients} == {"u1"} |
| 147 | |
| 148 | outbound, _ = object_mail.message_rows(parsed, owner_id="u1", message_id="m1", |
| 149 | thread_id="t1", direction="outbound") |
| 150 | assert outbound["is_read"] == "true" and "sent_at" in outbound |
| 151 | |
| 152 | |
| 153 | def test_a_message_with_no_from_says_so(): |
| 154 | parsed = object_mail.parse_message(raw("Subject: who")) |
| 155 | row, _ = object_mail.message_rows(parsed, owner_id="u1", message_id="m", thread_id="t", |
| 156 | direction="inbound") |
| 157 | assert row["from_address"] == object_mail.NO_SENDER |
| 158 | |
| 159 | |
| 160 | def test_direction_is_the_sent_folder_or_the_accounts_own_address(): |
| 161 | parsed = {"from_address": "Me@Dove.test"} |
| 162 | assert object_mail.direction_for(parsed, account_address="me@dove.test", folder_role="inbox") == "outbound" |
| 163 | assert object_mail.direction_for({"from_address": "x@y"}, account_address="me@dove.test", |
| 164 | folder_role="sent") == "outbound" |
| 165 | assert object_mail.direction_for({"from_address": "x@y"}, account_address="me@dove.test", |
| 166 | folder_role="inbox") == "inbound" |
| 167 | |
| 168 | |
| 169 | def test_a_late_old_message_does_not_make_a_thread_look_fresh(): |
| 170 | thread = {"message_count": "3", "last_message_at": "2026-09-25T10:00:00Z", |
| 171 | "participant_summary": "a@example.org"} |
| 172 | older = {"internaldate": "2026-09-01T10:00:00Z", "is_read": False, |
| 173 | "from_address": "b@example.org"} |
| 174 | changes = object_mail.thread_changes(thread, older, direction="inbound", |
| 175 | account_address="me@dove.test") |
| 176 | assert changes["message_count"] == "4" |
| 177 | assert "last_message_at" not in changes |
| 178 | assert changes["is_read"] == "false" |
| 179 | assert changes["participant_summary"] == "a@example.org, b@example.org" |
| 180 | newer = dict(older, internaldate="2026-09-25T10:00:01+00:00", is_read=True) |
| 181 | assert object_mail.thread_changes(thread, newer, direction="inbound", |
| 182 | account_address="me@dove.test")["last_message_at"] |
| 183 | |
| 184 | |
| 185 | # --- threading ------------------------------------------------------------------------------ |
| 186 | |
| 187 | def _index(owner="u1"): |
| 188 | messages = [ |
| 189 | {"id": "m1", "owner_id": "u1", "thread_id": "t1", "external_message_id": "<root@x>"}, |
| 190 | {"id": "m2", "owner_id": "u2", "thread_id": "t9", "external_message_id": "<theirs@x>"}, |
| 191 | {"id": "m3", "owner_id": "u1", "thread_id": "t3", "external_message_id": "<kid@x>", |
| 192 | "in_reply_to": "<late-parent@x>"}, |
| 193 | ] |
| 194 | synced = [{"owner_id": "u1", "message_id": "m1", "references": "<ancient@x>"}] |
| 195 | return object_mail.ThreadIndex.from_rows(owner, messages, synced) |
| 196 | |
| 197 | |
| 198 | def test_threading_follows_in_reply_to_then_references_then_a_reply_that_came_first(): |
| 199 | index = _index() |
| 200 | assert object_mail.match_thread({"in_reply_to": "<root@x>"}, index) == "t1" |
| 201 | assert object_mail.match_thread({"references": ["<nope@x>", "<root@x>"]}, index) == "t1" |
| 202 | assert object_mail.match_thread({"message_id": "<late-parent@x>"}, index) == "t3" |
| 203 | assert object_mail.match_thread({"message_id": "<ancient@x>"}, index) == "t1" |
| 204 | |
| 205 | |
| 206 | def test_threading_never_crosses_owners_even_when_handed_their_rows(): |
| 207 | assert object_mail.match_thread({"in_reply_to": "<theirs@x>"}, _index("u1")) is None |
| 208 | assert object_mail.match_thread({"in_reply_to": "<root@x>"}, _index("u2")) is None |
| 209 | |
| 210 | |
| 211 | def test_the_subject_alone_joins_nothing(): |
| 212 | index = _index() |
| 213 | index.by_message_id["<x@x>"] = "t1" |
| 214 | assert object_mail.match_thread({"subject": "Re: Invoice", "message_id": "<new@x>"}, index) is None |
| 215 | |
| 216 | |
| 217 | # --- cursor planning ------------------------------------------------------------------------ |
| 218 | |
| 219 | def test_plan_folder_resets_when_uidvalidity_changes(): |
| 220 | assert object_mail.plan_folder(None, uidvalidity="5") == "fresh" |
| 221 | assert object_mail.plan_folder({"uidvalidity": "5"}, uidvalidity="5") == "continue" |
| 222 | assert object_mail.plan_folder({"uidvalidity": "5"}, uidvalidity="6") == "reset" |
| 223 | |
| 224 | |
| 225 | def test_the_first_window_is_the_newest_n_and_an_empty_window_starts_at_the_top(): |
| 226 | assert object_mail.window_baseline(range(1, 501), first_window=200, highest_uid=500) == 300 |
| 227 | assert object_mail.window_baseline([], first_window=200, highest_uid=42) == 42 |
| 228 | assert object_mail.window_baseline([7], first_window=200, highest_uid=42) == 6 |
| 229 | |
| 230 | |
| 231 | def test_next_uids_drops_the_highest_message_n_star_hands_back(): |
| 232 | assert object_mail.next_uids([10], last_uid=10, limit=50) == [] |
| 233 | assert object_mail.next_uids([13, 11, 12, 11], last_uid=10, limit=2) == [11, 12] |
| 234 | |
| 235 | |
| 236 | def test_downloads_are_bounded_and_a_big_message_never_stalls_a_folder(): |
| 237 | metas = [{"uid": 1, "size": 30}, {"uid": 2, "size": 500}, {"uid": 3, "size": 30}, |
| 238 | {"uid": 4, "size": 30}] |
| 239 | assert object_mail.plan_downloads(metas, max_message_bytes=100, byte_budget=70) == [ |
| 240 | (1, "full"), (2, "header"), (3, "full")] |
| 241 | assert object_mail.plan_downloads([{"uid": 9, "size": 90}], max_message_bytes=100, |
| 242 | byte_budget=10) == [(9, "full")] |
| 243 | |
| 244 | |
| 245 | def test_imap_dates_do_not_depend_on_the_locale(): |
| 246 | assert object_mail.imap_date(datetime(2026, 9, 6)) == "6-Sep-2026" |
| 247 | |
| 248 | |
| 249 | # --- when an account is due ------------------------------------------------------------------- |
| 250 | |
| 251 | def _due(account, now=NOW): |
| 252 | return object_mail.account_due(account, now, claim_ttl_seconds=900, |
| 253 | retry_base_seconds=120, retry_max_seconds=1800) |
| 254 | |
| 255 | |
| 256 | def test_account_due_respects_disable_claims_and_backoff(): |
| 257 | assert _due({"enabled": "false"}) == (False, "disabled") |
| 258 | assert _due({})[0] is True |
| 259 | held = {"sync_started_at": "2026-09-26T11:55:00Z", "sync_finished_at": "2026-09-26T11:00:00Z"} |
| 260 | assert _due(held)[0] is False |
| 261 | stale = dict(held, sync_started_at="2026-09-26T11:00:30Z", sync_finished_at="2026-09-26T11:00:00Z") |
| 262 | assert _due(stale)[0] is True, "a claim nobody finished expires" |
| 263 | failing = {"sync_attempts": "3", "sync_finished_at": "2026-09-26T11:55:00Z"} |
| 264 | assert _due(failing)[0] is False # waits 8 minutes after 3 failures |
| 265 | assert _due(failing, NOW + timedelta(minutes=4))[0] is True |
| 266 | hopeless = {"sync_attempts": "40", "sync_finished_at": "2026-09-26T11:00:00Z"} |
| 267 | assert _due(hopeless)[0] is True, "the wait stops growing at the cap" |
| 268 | |
| 269 | |
| 270 | # --- where the server may connect ------------------------------------------------------------- |
| 271 | |
| 272 | def test_private_and_plaintext_hosts_are_refused_by_default(): |
| 273 | assert object_mail.host_refusal("imap.x", ["1.1.1.1"], security="ssl", allow_private=False) is None |
| 274 | refused = object_mail.host_refusal("db", ["10.1.2.3"], security="ssl", allow_private=False) |
| 275 | assert object_mail.ALLOW_PRIVATE_HOSTS_ENV in refused |
| 276 | assert object_mail.host_refusal("db", ["10.1.2.3"], security="ssl", allow_private=True) is None |
| 277 | assert object_mail.host_refusal("x", ["169.254.169.254"], security="ssl", allow_private=False) |
| 278 | assert object_mail.host_refusal("x", ["1.1.1.1"], security="plain", allow_private=True) |
| 279 | assert object_mail.host_refusal("localhost", ["127.0.0.1", "::1"], security="plain", |
| 280 | allow_private=True) is None |
| 281 | assert object_mail.host_refusal("x", [], security="ssl", allow_private=True) |
| 282 | |
| 283 | |
| 284 | # --- reading IMAP answers ---------------------------------------------------------------------- |
| 285 | |
| 286 | def test_fetch_answers_are_read_without_a_server(): |
| 287 | metas = object_mail.parse_fetch_meta([ |
| 288 | b'1 (UID 7 FLAGS (\\Seen \\Answered) RFC822.SIZE 1234 INTERNALDATE "25-Sep-2026 08:00:00 +0000")', |
| 289 | b'2 (UID 8 FLAGS () RFC822.SIZE 99 INTERNALDATE " 5-Sep-2026 08:00:00 +0000")']) |
| 290 | assert metas[0] == {"uid": 7, "size": 1234, "flags": ["\\Seen", "\\Answered"], |
| 291 | "internaldate": "25-Sep-2026 08:00:00 +0000"} |
| 292 | assert metas[1]["flags"] == [] |
| 293 | assert object_mail.fetch_literal([(b"1 (UID 7 BODY[] {3}", b"abc"), b")"]) == b"abc" |
| 294 | assert object_mail.fetch_literal([b"1 (UID 7)"]) is None |
| 295 | assert object_mail.parse_search([b"3 4 5"]) == [3, 4, 5] |
| 296 | assert object_mail.parse_search([None]) == [] |
| 297 | |
| 298 | |
| 299 | def test_folders_are_found_by_the_sent_attribute_then_by_name(): |
| 300 | entries = object_mail.parse_list([ |
| 301 | b'(\\HasNoChildren) "/" "INBOX"', |
| 302 | b'(\\HasNoChildren \\Sent) "/" "Sent Items"', |
| 303 | b'(\\HasNoChildren) "/" Archive', |
| 304 | (b'(\\HasNoChildren) "/" {10}', b'Quoted "x"'), |
| 305 | ]) |
| 306 | assert ([], "Archive") not in entries |
| 307 | assert (["\\HasNoChildren"], "Archive") in entries |
| 308 | assert (["\\HasNoChildren"], 'Quoted "x"') in entries |
| 309 | assert object_mail.choose_folders(entries) == [("INBOX", "inbox"), ("Sent Items", "sent")] |
| 310 | by_name = object_mail.choose_folders([([], "INBOX"), ([], "Sent")]) |
| 311 | assert by_name == [("INBOX", "inbox"), ("Sent", "sent")] |
| 312 | assert object_mail.choose_folders([([], "INBOX")]) == [("INBOX", "inbox")] |
| 313 | assert object_mail.folder_role("Sent Items", entries) == "sent" |
| 314 | assert object_mail.folder_role("Archive", entries) == "other" |
| 315 | assert object_mail.quote_mailbox('a "b"') == '"a \\"b\\""' |
| 316 | assert object_mail.split_folders("INBOX, Sent, ,INBOX") == ["INBOX", "Sent"] |
| 317 | |
| 318 | |
| 319 | def test_a_line_break_in_a_folder_name_is_refused_not_sent(): |
| 320 | """Inside a quoted string a CRLF ends the command and starts another: |
| 321 | 'INBOX\\r\\nA1 DELETE INBOX' would delete the person's inbox.""" |
| 322 | import pytest |
| 323 | |
| 324 | with pytest.raises(ValueError): |
| 325 | object_mail.quote_mailbox("INBOX\r\nA1 DELETE INBOX") |