freemyid 0.2.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. freemyid/__init__.py +2 -0
  2. freemyid/_data/brokers/LICENSE-DATA.md +28 -0
  3. freemyid/_data/brokers/registry.sample.yaml +674 -0
  4. freemyid/_data/brokers/state_laws.yaml +448 -0
  5. freemyid/_data/schema/registry.schema.json +668 -0
  6. freemyid/_data/schema/setup-config.schema.json +538 -0
  7. freemyid/_data/schema/state_laws.schema.json +222 -0
  8. freemyid/cli.py +725 -0
  9. freemyid/cli_json.py +198 -0
  10. freemyid/config.py +43 -0
  11. freemyid/consent.py +82 -0
  12. freemyid/crypto.py +197 -0
  13. freemyid/doctor.py +374 -0
  14. freemyid/engine/__init__.py +0 -0
  15. freemyid/engine/_dev/__init__.py +101 -0
  16. freemyid/engine/_dev/devrun.py +101 -0
  17. freemyid/engine/_dev/inbox/devauto.eml +7 -0
  18. freemyid/engine/_dev/registry.yaml +106 -0
  19. freemyid/engine/_dev/site/devauto.example/confirm.html +1 -0
  20. freemyid/engine/_dev/site/devauto.example/name/jane-testperson/il.html +17 -0
  21. freemyid/engine/_dev/site/devauto.example/name/jane-testperson/nc.html +4 -0
  22. freemyid/engine/_dev/site/devauto.example/name/jane-testperson/oh.html +4 -0
  23. freemyid/engine/_dev/site/devauto.example/optout/done.html +1 -0
  24. freemyid/engine/_dev/site/devauto.example/optout.html +6 -0
  25. freemyid/engine/_dev/site/devblocked.example/name/jane-testperson/il.html +3 -0
  26. freemyid/engine/_dev/site/devblocked.example/optout.html +3 -0
  27. freemyid/engine/_dev/site/devfill.example/ajax.html +20 -0
  28. freemyid/engine/_dev/site/devfill.example/aria.html +16 -0
  29. freemyid/engine/_dev/site/devfill.example/consent.html +20 -0
  30. freemyid/engine/_dev/site/devfill.example/done.html +3 -0
  31. freemyid/engine/_dev/site/devfill.example/empty.html +1 -0
  32. freemyid/engine/_dev/site/devfill.example/fullname.html +12 -0
  33. freemyid/engine/_dev/site/devfill.example/labels.html +32 -0
  34. freemyid/engine/_dev/site/devfill.example/late.html +22 -0
  35. freemyid/engine/_dev/site/devfind.example/hostile.html +12 -0
  36. freemyid/engine/_dev/site/devfind.example/p/jane-testperson-shelbyville-il.html +6 -0
  37. freemyid/engine/_dev/site/devfind.example/p/jane-testperson-springfield-il.html +6 -0
  38. freemyid/engine/_dev/site/devfind.example/results.html +11 -0
  39. freemyid/engine/_dev/site/devfind.example/search.html +16 -0
  40. freemyid/engine/_dev/site/devweb.example/remove.html +1 -0
  41. freemyid/engine/browser.py +430 -0
  42. freemyid/engine/fill_for_me.py +630 -0
  43. freemyid/engine/filler.py +523 -0
  44. freemyid/engine/handoff.py +237 -0
  45. freemyid/engine/letters.py +125 -0
  46. freemyid/engine/listing_finder.py +538 -0
  47. freemyid/engine/mail.py +406 -0
  48. freemyid/engine/mail_tls.py +198 -0
  49. freemyid/engine/matcher.py +208 -0
  50. freemyid/engine/outbox.py +421 -0
  51. freemyid/engine/plugins.py +68 -0
  52. freemyid/engine/runner.py +791 -0
  53. freemyid/engine/tiers.py +107 -0
  54. freemyid/interfaces.py +164 -0
  55. freemyid/logging_setup.py +152 -0
  56. freemyid/models.py +700 -0
  57. freemyid/netpolicy.py +145 -0
  58. freemyid/notify.py +58 -0
  59. freemyid/paths.py +83 -0
  60. freemyid/registry.py +275 -0
  61. freemyid/scheduler/__init__.py +203 -0
  62. freemyid/scheduler/render.py +91 -0
  63. freemyid/secret_backends.py +562 -0
  64. freemyid/secrets.py +25 -0
  65. freemyid/setup_config.py +311 -0
  66. freemyid/state.py +151 -0
  67. freemyid/storage/__init__.py +3 -0
  68. freemyid/storage/db.py +394 -0
  69. freemyid/submitters/__init__.py +0 -0
  70. freemyid/submitters/assisted.py +171 -0
  71. freemyid/submitters/base.py +119 -0
  72. freemyid/submitters/email.py +40 -0
  73. freemyid/submitters/web_form.py +276 -0
  74. freemyid/templates/_identity.txt.j2 +19 -0
  75. freemyid/templates/_signature.txt.j2 +7 -0
  76. freemyid/templates/ccpa.txt.j2 +16 -0
  77. freemyid/templates/follow_up.txt.j2 +15 -0
  78. freemyid/templates/generic_removal.txt.j2 +12 -0
  79. freemyid/templates/legal_demand.txt.j2 +20 -0
  80. freemyid/templates/phone_script.txt.j2 +25 -0
  81. freemyid/templates/postal_letter.txt.j2 +20 -0
  82. freemyid/templates/state_privacy.txt.j2 +17 -0
  83. freemyid/wizard/__init__.py +0 -0
  84. freemyid/wizard/dashboard.py +401 -0
  85. freemyid/wizard/desktop.py +188 -0
  86. freemyid/wizard/draft.py +627 -0
  87. freemyid/wizard/firstrun.py +42 -0
  88. freemyid/wizard/flows.py +283 -0
  89. freemyid/wizard/report.py +86 -0
  90. freemyid/wizard/server.py +834 -0
  91. freemyid/wizard/static/__init__.py +0 -0
  92. freemyid/wizard/static/app.js +98 -0
  93. freemyid/wizard/static/freemyid.png +0 -0
  94. freemyid/wizard/static/style.css +85 -0
  95. freemyid/wizard/ui.py +343 -0
  96. freemyid-0.2.0.dev1.dist-info/METADATA +396 -0
  97. freemyid-0.2.0.dev1.dist-info/RECORD +100 -0
  98. freemyid-0.2.0.dev1.dist-info/WHEEL +4 -0
  99. freemyid-0.2.0.dev1.dist-info/entry_points.txt +2 -0
  100. freemyid-0.2.0.dev1.dist-info/licenses/LICENSE +661 -0
@@ -0,0 +1,406 @@
1
+ """SMTP send + IMAP confirmation polling for a profile's dedicated removal mailbox.
2
+
3
+ * Password comes ONLY from the OS keychain (`secrets.get_mailbox_password`).
4
+ * Every host is checked with NetPolicy before connecting.
5
+ * `wait_for_confirmation` returns only https links whose host is inside the broker's
6
+ `confirmation_domains` (or `domains` if that is empty). Mail text is data, never
7
+ instructions: we extract URLs and nothing else.
8
+ * `FakeMailer` records sends in memory: used for dry runs, dev runs and tests.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import asyncio
13
+ import contextlib
14
+ import email
15
+ import email.policy
16
+ import imaplib
17
+ import logging
18
+ import re
19
+ import smtplib
20
+ from dataclasses import dataclass, field
21
+ from datetime import UTC, datetime, timedelta
22
+ from email.message import EmailMessage
23
+ from email.utils import formatdate, make_msgid, parsedate_to_datetime
24
+ from html import unescape
25
+ from typing import TYPE_CHECKING
26
+
27
+ from ..models import Broker, MailboxConfig, Profile, utcnow
28
+ from ..netpolicy import host_of
29
+ from .mail_tls import TlsPinError, check_peer, client_context
30
+
31
+ if TYPE_CHECKING:
32
+ from ..netpolicy import NetPolicy
33
+
34
+ log = logging.getLogger(__name__)
35
+
36
+ _URL = re.compile(r"https://[^\s<>\"')\]]+", re.IGNORECASE)
37
+ _HREF = re.compile(r"""href\s*=\s*["']([^"']+)["']""", re.IGNORECASE)
38
+ _ANCHOR = re.compile(r"""<a\b[^>]*?href\s*=\s*["']([^"']+)["'][^>]*>(.*?)</a\s*>""",
39
+ re.IGNORECASE | re.DOTALL)
40
+ # SEC-18: links that would cancel/undo/unsubscribe (or are just boilerplate) are never clicked
41
+ _AVOID = re.compile(r"cancel|unsubscribe|opt.?in|not (you|me)|did ?n.?t (request|ask)|"
42
+ r"did not (request|ask)|report|preferences|privacy|terms|login|sign.?in",
43
+ re.IGNORECASE)
44
+ _CONFIRM = re.compile(r"confirm|verif|validate|activate|complete", re.IGNORECASE)
45
+ # A broker reply that asks for an identity document. Such mail is never auto-handled: no link
46
+ # in it is clicked and the runner opens an ID_UPLOAD to-do (Free My ID never sends ID).
47
+ _ID_REQUEST = re.compile(
48
+ r"(government|state)[- ]issued|photo (id\b|identification)|driver'?s?[- ]licen[cs]e|"
49
+ r"passport|(copy|scan|photo|picture|image) of (your |a )?(valid )?"
50
+ r"(photo )?(id\b|identification|identity|license|licence)|social security (number|card)|"
51
+ r"\bssn\b|last (4|four) digits|notari[sz]ed|utility bill|"
52
+ r"proof of (identity|address|residency)|identity document|id document",
53
+ re.IGNORECASE)
54
+ DATE_SLACK = timedelta(hours=1) # broker clock skew vs. our last_submitted_at
55
+ MAX_MESSAGES = 50 # per poll; we only need recent broker mail
56
+ MAX_MESSAGE_BYTES = 2_000_000
57
+
58
+
59
+ class MailError(RuntimeError):
60
+ """PII-free mail failure (message holds a short reason only)."""
61
+
62
+
63
+ def _host_in(host: str | None, domains: list[str]) -> bool:
64
+ if not host:
65
+ return False
66
+ return any(host == d or host.endswith("." + d) for d in (x.lower() for x in domains))
67
+
68
+
69
+ def confirmation_hosts(broker: Broker) -> list[str]:
70
+ return list(broker.confirmation_domains or broker.domains)
71
+
72
+
73
+ def build_message(mb: MailboxConfig, to: str, subject: str, body: str,
74
+ request_id: str) -> EmailMessage:
75
+ """Plain-text message from the removal mailbox. Message-ID carries the request id so
76
+ replies can be tied to a Request without any PII."""
77
+ msg = EmailMessage()
78
+ msg["From"] = mb.address
79
+ msg["To"] = to
80
+ msg["Subject"] = subject
81
+ msg["Date"] = formatdate(localtime=False)
82
+ domain = mb.address.rpartition("@")[2] or "freemyid.invalid"
83
+ msg["Message-ID"] = make_msgid(idstring=f"freemyid.{request_id}", domain=domain)
84
+ msg.set_content(body)
85
+ return msg
86
+
87
+
88
+ def extract_links(raw: bytes, allowed_domains: list[str]) -> list[str]:
89
+ """Pure: https links in a raw RFC822 message whose host is inside `allowed_domains`."""
90
+ if len(raw) > MAX_MESSAGE_BYTES:
91
+ return []
92
+ msg = email.message_from_bytes(raw, policy=email.policy.default)
93
+ texts: list[str] = []
94
+ for part in msg.walk():
95
+ if part.get_content_maintype() != "text":
96
+ continue
97
+ try:
98
+ texts.append(part.get_content())
99
+ except (LookupError, ValueError):
100
+ continue
101
+ found: list[str] = []
102
+ text_of: dict[str, str] = {}
103
+ for t in texts:
104
+ for href, inner in _ANCHOR.findall(t):
105
+ u = unescape(href).strip()
106
+ label = " ".join(unescape(re.sub(r"<[^>]+>", " ", inner)).split())
107
+ text_of[u] = f"{text_of.get(u, '')} {label}".strip()
108
+ cands = [unescape(h) for h in _HREF.findall(t)] + _URL.findall(unescape(t))
109
+ for u in cands:
110
+ u = u.strip().rstrip(".,;")
111
+ if not u.lower().startswith("https://"):
112
+ continue
113
+ h = host_of(u)
114
+ if h and _host_in(h, allowed_domains) and u not in found:
115
+ found.append(u)
116
+ keep = [u for u in found if not _avoid(u, text_of.get(u, ""))]
117
+ return rank_links(keep, text_of)
118
+
119
+
120
+ def _url_tail(u: str) -> str:
121
+ """Path, query and fragment only (a host like privacy.broker.example is not a signal)."""
122
+ rest = u.split("://", 1)[-1]
123
+ i = min((rest.find(c) for c in "/?#" if c in rest), default=len(rest))
124
+ return rest[i:]
125
+
126
+
127
+ def _avoid(u: str, text: str) -> bool:
128
+ return bool(_AVOID.search(_url_tail(u)) or _AVOID.search(text))
129
+
130
+
131
+ def rank_links(links: list[str], text_of: dict[str, str] | None = None) -> list[str]:
132
+ """Confirm/verify-looking links first (stable)."""
133
+ t = text_of or {}
134
+ return sorted(links, key=lambda u: not (_CONFIRM.search(_url_tail(u))
135
+ or _CONFIRM.search(t.get(u, ""))))
136
+
137
+
138
+ def sender_domain(raw: bytes) -> str | None:
139
+ msg = email.message_from_bytes(raw[:65536], policy=email.policy.default)
140
+ frm = str(msg.get("From", ""))
141
+ m = re.search(r"@([A-Za-z0-9.-]+)", frm)
142
+ return m.group(1).lower().rstrip(".") if m else None
143
+
144
+
145
+ @dataclass
146
+ class ReplyScan:
147
+ """What the broker's replies say, PII-free (links are confirmation URLs only)."""
148
+ links: list[str] = field(default_factory=list) # newest message first, ranked inside
149
+ id_requested: bool = False # some reply asks for an ID document -> human to-do
150
+ messages: int = 0 # distinct broker messages considered
151
+ duplicates: int = 0 # same Message-ID (or same bytes) seen again
152
+ stale: int = 0 # older than the submission this reply would answer
153
+
154
+
155
+ def message_date(raw: bytes) -> datetime | None:
156
+ msg = email.message_from_bytes(raw[:65536], policy=email.policy.default)
157
+ try:
158
+ d = parsedate_to_datetime(str(msg.get("Date", "")))
159
+ except (TypeError, ValueError, IndexError):
160
+ return None
161
+ if d is None:
162
+ return None
163
+ return d if d.tzinfo else d.replace(tzinfo=UTC)
164
+
165
+
166
+ def _message_key(raw: bytes) -> str:
167
+ msg = email.message_from_bytes(raw[:65536], policy=email.policy.default)
168
+ mid = str(msg.get("Message-ID", "")).strip()
169
+ if mid:
170
+ return mid
171
+ import hashlib
172
+ return hashlib.sha256(raw).hexdigest()
173
+
174
+
175
+ def _plain_text(raw: bytes) -> str:
176
+ msg = email.message_from_bytes(raw, policy=email.policy.default)
177
+ parts: list[str] = [str(msg.get("Subject", ""))]
178
+ for part in msg.walk():
179
+ if part.get_content_maintype() != "text":
180
+ continue
181
+ try:
182
+ parts.append(unescape(re.sub(r"<[^>]+>", " ", part.get_content())))
183
+ except (LookupError, ValueError):
184
+ continue
185
+ return " ".join(" ".join(parts).split())
186
+
187
+
188
+ def asks_for_id(raw: bytes) -> bool:
189
+ """Pure: does this message ask for an identity document (ID, licence, SSN, ...)?"""
190
+ if len(raw) > MAX_MESSAGE_BYTES:
191
+ return False
192
+ return bool(_ID_REQUEST.search(_plain_text(raw)))
193
+
194
+
195
+ def scan_replies(raws: list[bytes], broker: Broker, since: datetime | None = None, *,
196
+ check_sender: bool = True) -> ReplyScan:
197
+ """Pure reply matcher.
198
+
199
+ * only mail From a broker/confirmation domain (unless check_sender=False);
200
+ * the same message twice (Message-ID, else bytes) counts once;
201
+ * mail dated before `since` (minus DATE_SLACK) answers an earlier request: ignored, so
202
+ an old confirmation link is never clicked for a new submission;
203
+ * a message that asks for ID sets id_requested and contributes no links;
204
+ * links come newest message first (a broker that answers twice: its latest link wins).
205
+ """
206
+ allowed = confirmation_hosts(broker)
207
+ out = ReplyScan()
208
+ seen: set[str] = set()
209
+ msgs: list[tuple[datetime | None, int, bytes]] = []
210
+ for i, raw in enumerate(raws):
211
+ if check_sender and not _host_in(sender_domain(raw), [*broker.domains, *allowed]):
212
+ continue
213
+ key = _message_key(raw)
214
+ if key in seen:
215
+ out.duplicates += 1
216
+ continue
217
+ seen.add(key)
218
+ d = message_date(raw)
219
+ if since is not None and d is not None:
220
+ ref = since if since.tzinfo else since.replace(tzinfo=UTC)
221
+ if d < ref - DATE_SLACK:
222
+ out.stale += 1
223
+ continue
224
+ msgs.append((d, i, raw))
225
+ out.messages = len(msgs)
226
+ floor = datetime.min.replace(tzinfo=UTC)
227
+ msgs.sort(key=lambda m: (m[0] or floor, m[1]), reverse=True)
228
+ for _d, _i, raw in msgs:
229
+ if asks_for_id(raw):
230
+ out.id_requested = True
231
+ continue
232
+ for u in extract_links(raw, allowed):
233
+ if u not in out.links:
234
+ out.links.append(u)
235
+ return out
236
+
237
+
238
+ class SmtpImapMailer:
239
+ """Real mailer. Never used in dev/tests (they use FakeMailer)."""
240
+
241
+ def __init__(self, netpolicy: NetPolicy, password_getter=None, timeout_s: float = 30.0,
242
+ poll_interval_s: float = 30.0) -> None:
243
+ from .. import secrets
244
+ self.netpolicy = netpolicy
245
+ self._pw = password_getter or secrets.get_mailbox_password
246
+ self.timeout_s = timeout_s
247
+ self.poll_interval_s = poll_interval_s
248
+ self.last_raw: bytes | None = None # last sent message, for evidence
249
+
250
+ def _password(self, profile: Profile) -> str:
251
+ mb = profile.mailbox
252
+ if mb is None:
253
+ raise MailError("profile has no removal mailbox")
254
+ pw = self._pw(profile.id, mb)
255
+ if not pw:
256
+ raise MailError("mailbox password not in keychain")
257
+ return pw
258
+
259
+ def _check_host(self, host: str) -> None:
260
+ if not self.netpolicy.is_allowed(host):
261
+ raise MailError("mail host not on allowlist")
262
+
263
+ def send(self, profile: Profile, to: str, subject: str, body: str, request_id: str) -> str:
264
+ mb = profile.mailbox
265
+ if mb is None:
266
+ raise MailError("profile has no removal mailbox")
267
+ self._check_host(mb.smtp_host)
268
+ msg = build_message(mb, to, subject, body, request_id)
269
+ pw = self._password(profile)
270
+ pin = mb.tls_cert_sha256
271
+ try:
272
+ ctx = client_context(mb.smtp_host, pin)
273
+ if mb.smtp_starttls:
274
+ with smtplib.SMTP(mb.smtp_host, mb.smtp_port, timeout=self.timeout_s) as s:
275
+ s.starttls(context=ctx)
276
+ check_peer(s.sock, pin) # before any credential is sent
277
+ s.login(mb.username, pw)
278
+ s.send_message(msg)
279
+ else:
280
+ with smtplib.SMTP_SSL(mb.smtp_host, mb.smtp_port, context=ctx,
281
+ timeout=self.timeout_s) as s:
282
+ check_peer(s.sock, pin)
283
+ s.login(mb.username, pw)
284
+ s.send_message(msg)
285
+ except TlsPinError as e:
286
+ raise MailError(f"smtp refused: {e}") from None
287
+ except (smtplib.SMTPException, OSError) as e:
288
+ raise MailError(f"smtp failed: {type(e).__name__}") from None
289
+ self.last_raw = msg.as_bytes()
290
+ return str(msg["Message-ID"])
291
+
292
+ # ------------------------------------------------------------------ IMAP
293
+ def _imap(self, mb: MailboxConfig) -> imaplib.IMAP4:
294
+ """Connected (not logged-in) IMAP. With a pin, the peer certificate is checked here,
295
+ so `login` is only ever reached on a matching certificate.
296
+
297
+ Plaintext IMAP (no SSL and no STARTTLS) is refused on EVERY host, loopback
298
+ included, before any socket is opened (SEC-53). The MailboxConfig validator
299
+ refuses it too, but it can be bypassed (`model_copy`), so this check stands alone."""
300
+ if not mb.imap_ssl and not mb.imap_starttls:
301
+ raise MailError("mail connection is not encrypted; refusing to sign in")
302
+ self._check_host(mb.imap_host)
303
+ pin = mb.tls_cert_sha256
304
+ ctx = client_context(mb.imap_host, pin)
305
+ if mb.imap_ssl and not mb.imap_starttls:
306
+ conn: imaplib.IMAP4 = imaplib.IMAP4_SSL(mb.imap_host, mb.imap_port,
307
+ ssl_context=ctx, timeout=self.timeout_s)
308
+ else:
309
+ conn = imaplib.IMAP4(mb.imap_host, mb.imap_port, timeout=self.timeout_s)
310
+ if mb.imap_starttls:
311
+ conn.starttls(ssl_context=ctx)
312
+ try:
313
+ check_peer(conn.sock, pin)
314
+ except TlsPinError:
315
+ with contextlib.suppress(Exception):
316
+ conn.shutdown()
317
+ raise
318
+ return conn
319
+
320
+ def fetch_since(self, profile: Profile, since: datetime) -> list[bytes]:
321
+ mb = profile.mailbox
322
+ if mb is None:
323
+ raise MailError("profile has no removal mailbox")
324
+ pw = self._password(profile)
325
+ out: list[bytes] = []
326
+ try:
327
+ conn = self._imap(mb)
328
+ try:
329
+ conn.login(mb.username, pw)
330
+ conn.select("INBOX", readonly=True)
331
+ typ, data = conn.search(None, "SINCE", since.strftime("%d-%b-%Y"))
332
+ if typ != "OK" or not data or not data[0]:
333
+ return []
334
+ for num in data[0].split()[-MAX_MESSAGES:]:
335
+ typ, msgdata = conn.fetch(num, "(BODY.PEEK[])")
336
+ if typ == "OK" and msgdata and isinstance(msgdata[0], tuple):
337
+ out.append(msgdata[0][1])
338
+ finally:
339
+ with contextlib.suppress(Exception):
340
+ conn.logout()
341
+ except TlsPinError as e:
342
+ raise MailError(f"imap refused: {e}") from None
343
+ except (imaplib.IMAP4.error, OSError) as e:
344
+ raise MailError(f"imap failed: {type(e).__name__}") from None
345
+ return out
346
+
347
+ def links_from(self, raws: list[bytes], broker: Broker) -> list[str]:
348
+ return scan_replies(raws, broker).links # filtered + ranked (SEC-18)
349
+
350
+ async def scan_replies(self, profile: Profile, broker: Broker, since: datetime,
351
+ timeout_s: int = 0) -> ReplyScan:
352
+ loop = asyncio.get_running_loop()
353
+ deadline = loop.time() + timeout_s
354
+ while True:
355
+ raws = await asyncio.to_thread(self.fetch_since, profile, since)
356
+ scan = scan_replies(raws, broker, since)
357
+ if scan.links or scan.id_requested or loop.time() >= deadline:
358
+ return scan
359
+ await asyncio.sleep(min(self.poll_interval_s, max(0.0, deadline - loop.time())))
360
+
361
+ async def wait_for_confirmation(self, profile: Profile, broker: Broker, since: datetime,
362
+ timeout_s: int = 600) -> list[str]:
363
+ scan = await self.scan_replies(profile, broker, since, timeout_s)
364
+ return [] if scan.id_requested else scan.links
365
+
366
+
367
+ @dataclass
368
+ class SentMail:
369
+ profile_id: str
370
+ to: str
371
+ subject: str
372
+ body: str
373
+ request_id: str
374
+ message_id: str
375
+
376
+
377
+ @dataclass
378
+ class FakeMailer:
379
+ """In-memory mailer for dry runs, the dev fixture run and tests. Sends nothing."""
380
+ inbox: dict[str, list[bytes]] = field(default_factory=dict) # broker_id -> raw mails
381
+ sent: list[SentMail] = field(default_factory=list)
382
+ last_raw: bytes | None = None
383
+
384
+ def send(self, profile: Profile, to: str, subject: str, body: str, request_id: str) -> str:
385
+ mb = profile.mailbox or MailboxConfig(address="removal@freemyid.invalid",
386
+ imap_host="imap.freemyid.invalid",
387
+ smtp_host="smtp.freemyid.invalid",
388
+ username="fake")
389
+ msg = build_message(mb, to, subject, body, request_id)
390
+ self.last_raw = msg.as_bytes()
391
+ mid = str(msg["Message-ID"])
392
+ self.sent.append(SentMail(profile.id, to, subject, body, request_id, mid))
393
+ return mid
394
+
395
+ async def scan_replies(self, profile: Profile, broker: Broker, since: datetime,
396
+ timeout_s: int = 0) -> ReplyScan:
397
+ return scan_replies(self.inbox.get(broker.id, []), broker, since)
398
+
399
+ async def wait_for_confirmation(self, profile: Profile, broker: Broker, since: datetime,
400
+ timeout_s: int = 600) -> list[str]:
401
+ scan = await self.scan_replies(profile, broker, since, timeout_s)
402
+ return [] if scan.id_requested else scan.links
403
+
404
+
405
+ def now() -> datetime:
406
+ return utcnow()
@@ -0,0 +1,198 @@
1
+ """TLS for the removal mailbox, with certificate PINNING for local bridges.
2
+
3
+ Proton Mail Bridge runs on the user's own computer (127.0.0.1) and serves a self-signed
4
+ certificate, so normal CA verification always fails. We never switch verification off.
5
+ Instead the user confirms the certificate once (trust on first use) and its SHA-256
6
+ fingerprint is saved in `MailboxConfig.tls_cert_sha256`. On every connection:
7
+
8
+ * no pin -> `ssl.create_default_context()` (full CA + hostname checks, unchanged);
9
+ * pin + loopback -> handshake, then the peer's DER leaf certificate is hashed and compared
10
+ to the pin in constant time BEFORE any login/credential is sent;
11
+ a mismatch aborts the connection;
12
+ * pin + any other host -> refused with a clear error (pins are only for local bridges;
13
+ a real mail provider must pass normal verification).
14
+ """
15
+ from __future__ import annotations
16
+
17
+ import contextlib
18
+ import hashlib
19
+ import hmac
20
+ import imaplib
21
+ import smtplib
22
+ import ssl
23
+ from dataclasses import dataclass
24
+ from typing import TYPE_CHECKING, Literal
25
+
26
+ if TYPE_CHECKING:
27
+ from ..models import MailboxConfig
28
+ from ..netpolicy import NetPolicy
29
+
30
+ LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"})
31
+ # The connected peer must be one of these EXACT addresses (SEC-50). The name `localhost` is
32
+ # accepted in settings, but the resolver decides where it goes, so it is never trusted on
33
+ # its own: what counts is the address the socket actually reached.
34
+ LOOPBACK_PEER_IPS = frozenset({"127.0.0.1", "::1"})
35
+
36
+
37
+ class TlsPinError(RuntimeError):
38
+ """Pin refused or mismatched. Message is short, plain-language and PII-free."""
39
+
40
+
41
+ def is_loopback(host: str) -> bool:
42
+ return host.strip().lower().rstrip(".").strip("[]") in LOOPBACK_HOSTS
43
+
44
+
45
+ def fingerprint(der: bytes) -> str:
46
+ """Lowercase hex SHA-256 of a DER certificate (the format stored as the pin)."""
47
+ return hashlib.sha256(der).hexdigest()
48
+
49
+
50
+ def format_fingerprint(fp: str) -> str:
51
+ """'ab12cd..' -> 'AB:12:CD:..' (how certificate viewers and Bridge show it)."""
52
+ fp = fp.upper()
53
+ return ":".join(fp[i:i + 2] for i in range(0, len(fp), 2))
54
+
55
+
56
+ def _unverified_context() -> ssl.SSLContext:
57
+ """Handshake-only context. ONLY used where the peer certificate is checked against a
58
+ pin before anything is sent (`client_context` with a pin) or where nothing but the
59
+ certificate is read (`fetch_loopback_fingerprint`). Never returned for an unpinned
60
+ connection that carries credentials."""
61
+ ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
62
+ ctx.minimum_version = ssl.TLSVersion.TLSv1_2
63
+ ctx.check_hostname = False
64
+ ctx.verify_mode = ssl.CERT_NONE # authenticity comes from the pin check instead
65
+ return ctx
66
+
67
+
68
+ def client_context(host: str, pin: str | None) -> ssl.SSLContext:
69
+ """TLS context for connecting to `host`. Callers MUST call `check_peer(sock, pin)`
70
+ after the handshake when `pin` is set, before sending credentials."""
71
+ if pin is None:
72
+ return ssl.create_default_context()
73
+ if not is_loopback(host):
74
+ raise TlsPinError("a certificate pin is only allowed for a mail app on this "
75
+ "computer (127.0.0.1, ::1 or localhost); remove the pin or use "
76
+ "the provider's normal settings")
77
+ if len(pin) != 64:
78
+ raise TlsPinError("saved certificate fingerprint is malformed")
79
+ return _unverified_context()
80
+
81
+
82
+ def check_loopback_peer(sock: object) -> None:
83
+ """Raise TlsPinError unless the socket is connected to exactly 127.0.0.1 or ::1.
84
+ Fails closed: no `getpeername`, an error, or an unexpected shape all refuse."""
85
+ try:
86
+ peer = sock.getpeername()[0] # type: ignore[attr-defined]
87
+ except (OSError, AttributeError, TypeError, IndexError, KeyError, ValueError):
88
+ peer = None
89
+ if not isinstance(peer, str) or peer not in LOOPBACK_PEER_IPS:
90
+ raise TlsPinError("the mail app is not on this computer; not trusting it")
91
+
92
+
93
+ def check_peer(sock: object, pin: str | None) -> None:
94
+ """Raise TlsPinError unless the TLS peer certificate's SHA-256 equals `pin` and the
95
+ peer is an exact loopback address (SEC-50). No-op without a pin (the default context
96
+ already verified the peer)."""
97
+ if pin is None:
98
+ return
99
+ get = getattr(sock, "getpeercert", None)
100
+ der = get(binary_form=True) if get else None
101
+ if not der:
102
+ raise TlsPinError("mail connection is not encrypted; refusing to sign in")
103
+ check_loopback_peer(sock)
104
+ if not hmac.compare_digest(fingerprint(der).encode(), pin.lower().encode()):
105
+ raise TlsPinError("the mail app's security certificate changed and does not match "
106
+ "the one you approved; not signing in")
107
+
108
+
109
+ # ------------------------------------------------------------------ trust on first use
110
+ def _loopback_policy() -> NetPolicy:
111
+ """Fallback for callers that do not pass a policy yet: the global config allowlist
112
+ plus loopback. Combined with `is_loopback` it still only ever admits this computer."""
113
+ from ..netpolicy import NetPolicy
114
+ return NetPolicy(None, allow_loopback=True)
115
+
116
+
117
+ def fetch_loopback_fingerprint(host: str, port: int, protocol: Literal["imap", "smtp"],
118
+ *, starttls: bool, timeout_s: float = 10.0,
119
+ netpolicy: NetPolicy | None = None) -> str:
120
+ """Connect to a LOCAL mail app (e.g. Proton Bridge), read its certificate and return the
121
+ SHA-256 fingerprint for the user to confirm. Sends no username or password.
122
+ The host must pass `netpolicy.is_allowed` before any socket opens (SEC-51)."""
123
+ if not is_loopback(host):
124
+ raise TlsPinError("only a mail app on this computer can be trusted this way")
125
+ pol = netpolicy if netpolicy is not None else _loopback_policy()
126
+ if not pol.is_allowed(host):
127
+ raise TlsPinError("mail host not on allowlist")
128
+ ctx = _unverified_context()
129
+ sock: object
130
+ try:
131
+ if protocol == "imap":
132
+ if starttls:
133
+ conn = imaplib.IMAP4(host, port, timeout=timeout_s)
134
+ conn.starttls(ssl_context=ctx)
135
+ else:
136
+ conn = imaplib.IMAP4_SSL(host, port, ssl_context=ctx, timeout=timeout_s)
137
+ sock = conn.sock
138
+ try:
139
+ check_loopback_peer(sock)
140
+ der = sock.getpeercert(binary_form=True) # type: ignore[attr-defined]
141
+ finally:
142
+ with contextlib.suppress(Exception):
143
+ conn.logout()
144
+ else:
145
+ if starttls:
146
+ s = smtplib.SMTP(host, port, timeout=timeout_s)
147
+ s.starttls(context=ctx)
148
+ else:
149
+ s = smtplib.SMTP_SSL(host, port, context=ctx, timeout=timeout_s)
150
+ try:
151
+ check_loopback_peer(s.sock)
152
+ der = s.sock.getpeercert(binary_form=True) # type: ignore[union-attr]
153
+ finally:
154
+ with contextlib.suppress(Exception):
155
+ s.quit()
156
+ except (OSError, imaplib.IMAP4.error, smtplib.SMTPException) as e:
157
+ raise TlsPinError(f"could not reach the mail app ({type(e).__name__}); "
158
+ "is it running?") from None
159
+ if not der:
160
+ raise TlsPinError("the mail app did not offer a certificate")
161
+ return fingerprint(der)
162
+
163
+
164
+ @dataclass
165
+ class TrustResult:
166
+ fingerprint: str # lowercase hex, ready for MailboxConfig.tls_cert_sha256
167
+ display: str # 'AB:12:..' for showing to the user
168
+ message: str # plain-language prompt for the confirm screen
169
+
170
+
171
+ def trust_on_first_use(mb: MailboxConfig, timeout_s: float = 10.0, *,
172
+ netpolicy: NetPolicy | None = None) -> TrustResult:
173
+ """Read the certificate of a local mail app from both the IMAP and SMTP ports.
174
+ The caller shows `message` and saves `fingerprint` only after the user says yes.
175
+ Both ports must present the same certificate (Proton Bridge does).
176
+ Pass the caller's `netpolicy` (SEC-51); both hosts are checked before connecting."""
177
+ pol = netpolicy if netpolicy is not None else _loopback_policy()
178
+ for h in (mb.imap_host, mb.smtp_host):
179
+ if not is_loopback(h):
180
+ raise TlsPinError("only a mail app on this computer can be trusted this way")
181
+ if not pol.is_allowed(h):
182
+ raise TlsPinError("mail host not on allowlist")
183
+ imap_fp = fetch_loopback_fingerprint(
184
+ mb.imap_host, mb.imap_port, "imap",
185
+ starttls=mb.imap_starttls or not mb.imap_ssl, timeout_s=timeout_s, netpolicy=pol)
186
+ smtp_fp = fetch_loopback_fingerprint(
187
+ mb.smtp_host, mb.smtp_port, "smtp", starttls=mb.smtp_starttls, timeout_s=timeout_s,
188
+ netpolicy=pol)
189
+ if not hmac.compare_digest(imap_fp, smtp_fp):
190
+ raise TlsPinError("the mail app uses two different certificates; "
191
+ "this setup is not supported")
192
+ shown = format_fingerprint(imap_fp)
193
+ msg = ("Free My ID found the mail app on this computer (for example Proton Mail Bridge). "
194
+ "Its security code is:\n\n" + shown + "\n\n"
195
+ "If you just installed and started Proton Mail Bridge yourself, click Yes to "
196
+ "trust it. Free My ID will remember this code and refuse to sign in if it "
197
+ "ever changes.")
198
+ return TrustResult(imap_fp, shown, msg)