freemyid 0.2.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- freemyid/__init__.py +2 -0
- freemyid/_data/brokers/LICENSE-DATA.md +28 -0
- freemyid/_data/brokers/registry.sample.yaml +674 -0
- freemyid/_data/brokers/state_laws.yaml +448 -0
- freemyid/_data/schema/registry.schema.json +668 -0
- freemyid/_data/schema/setup-config.schema.json +538 -0
- freemyid/_data/schema/state_laws.schema.json +222 -0
- freemyid/cli.py +725 -0
- freemyid/cli_json.py +198 -0
- freemyid/config.py +43 -0
- freemyid/consent.py +82 -0
- freemyid/crypto.py +197 -0
- freemyid/doctor.py +374 -0
- freemyid/engine/__init__.py +0 -0
- freemyid/engine/_dev/__init__.py +101 -0
- freemyid/engine/_dev/devrun.py +101 -0
- freemyid/engine/_dev/inbox/devauto.eml +7 -0
- freemyid/engine/_dev/registry.yaml +106 -0
- freemyid/engine/_dev/site/devauto.example/confirm.html +1 -0
- freemyid/engine/_dev/site/devauto.example/name/jane-testperson/il.html +17 -0
- freemyid/engine/_dev/site/devauto.example/name/jane-testperson/nc.html +4 -0
- freemyid/engine/_dev/site/devauto.example/name/jane-testperson/oh.html +4 -0
- freemyid/engine/_dev/site/devauto.example/optout/done.html +1 -0
- freemyid/engine/_dev/site/devauto.example/optout.html +6 -0
- freemyid/engine/_dev/site/devblocked.example/name/jane-testperson/il.html +3 -0
- freemyid/engine/_dev/site/devblocked.example/optout.html +3 -0
- freemyid/engine/_dev/site/devfill.example/ajax.html +20 -0
- freemyid/engine/_dev/site/devfill.example/aria.html +16 -0
- freemyid/engine/_dev/site/devfill.example/consent.html +20 -0
- freemyid/engine/_dev/site/devfill.example/done.html +3 -0
- freemyid/engine/_dev/site/devfill.example/empty.html +1 -0
- freemyid/engine/_dev/site/devfill.example/fullname.html +12 -0
- freemyid/engine/_dev/site/devfill.example/labels.html +32 -0
- freemyid/engine/_dev/site/devfill.example/late.html +22 -0
- freemyid/engine/_dev/site/devfind.example/hostile.html +12 -0
- freemyid/engine/_dev/site/devfind.example/p/jane-testperson-shelbyville-il.html +6 -0
- freemyid/engine/_dev/site/devfind.example/p/jane-testperson-springfield-il.html +6 -0
- freemyid/engine/_dev/site/devfind.example/results.html +11 -0
- freemyid/engine/_dev/site/devfind.example/search.html +16 -0
- freemyid/engine/_dev/site/devweb.example/remove.html +1 -0
- freemyid/engine/browser.py +430 -0
- freemyid/engine/fill_for_me.py +630 -0
- freemyid/engine/filler.py +523 -0
- freemyid/engine/handoff.py +237 -0
- freemyid/engine/letters.py +125 -0
- freemyid/engine/listing_finder.py +538 -0
- freemyid/engine/mail.py +406 -0
- freemyid/engine/mail_tls.py +198 -0
- freemyid/engine/matcher.py +208 -0
- freemyid/engine/outbox.py +421 -0
- freemyid/engine/plugins.py +68 -0
- freemyid/engine/runner.py +791 -0
- freemyid/engine/tiers.py +107 -0
- freemyid/interfaces.py +164 -0
- freemyid/logging_setup.py +152 -0
- freemyid/models.py +700 -0
- freemyid/netpolicy.py +145 -0
- freemyid/notify.py +58 -0
- freemyid/paths.py +83 -0
- freemyid/registry.py +275 -0
- freemyid/scheduler/__init__.py +203 -0
- freemyid/scheduler/render.py +91 -0
- freemyid/secret_backends.py +562 -0
- freemyid/secrets.py +25 -0
- freemyid/setup_config.py +311 -0
- freemyid/state.py +151 -0
- freemyid/storage/__init__.py +3 -0
- freemyid/storage/db.py +394 -0
- freemyid/submitters/__init__.py +0 -0
- freemyid/submitters/assisted.py +171 -0
- freemyid/submitters/base.py +119 -0
- freemyid/submitters/email.py +40 -0
- freemyid/submitters/web_form.py +276 -0
- freemyid/templates/_identity.txt.j2 +19 -0
- freemyid/templates/_signature.txt.j2 +7 -0
- freemyid/templates/ccpa.txt.j2 +16 -0
- freemyid/templates/follow_up.txt.j2 +15 -0
- freemyid/templates/generic_removal.txt.j2 +12 -0
- freemyid/templates/legal_demand.txt.j2 +20 -0
- freemyid/templates/phone_script.txt.j2 +25 -0
- freemyid/templates/postal_letter.txt.j2 +20 -0
- freemyid/templates/state_privacy.txt.j2 +17 -0
- freemyid/wizard/__init__.py +0 -0
- freemyid/wizard/dashboard.py +401 -0
- freemyid/wizard/desktop.py +188 -0
- freemyid/wizard/draft.py +627 -0
- freemyid/wizard/firstrun.py +42 -0
- freemyid/wizard/flows.py +283 -0
- freemyid/wizard/report.py +86 -0
- freemyid/wizard/server.py +834 -0
- freemyid/wizard/static/__init__.py +0 -0
- freemyid/wizard/static/app.js +98 -0
- freemyid/wizard/static/freemyid.png +0 -0
- freemyid/wizard/static/style.css +85 -0
- freemyid/wizard/ui.py +343 -0
- freemyid-0.2.0.dev1.dist-info/METADATA +396 -0
- freemyid-0.2.0.dev1.dist-info/RECORD +100 -0
- freemyid-0.2.0.dev1.dist-info/WHEEL +4 -0
- freemyid-0.2.0.dev1.dist-info/entry_points.txt +2 -0
- freemyid-0.2.0.dev1.dist-info/licenses/LICENSE +661 -0
freemyid/engine/mail.py
ADDED
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
"""SMTP send + IMAP confirmation polling for a profile's dedicated removal mailbox.
|
|
2
|
+
|
|
3
|
+
* Password comes ONLY from the OS keychain (`secrets.get_mailbox_password`).
|
|
4
|
+
* Every host is checked with NetPolicy before connecting.
|
|
5
|
+
* `wait_for_confirmation` returns only https links whose host is inside the broker's
|
|
6
|
+
`confirmation_domains` (or `domains` if that is empty). Mail text is data, never
|
|
7
|
+
instructions: we extract URLs and nothing else.
|
|
8
|
+
* `FakeMailer` records sends in memory: used for dry runs, dev runs and tests.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import asyncio
|
|
13
|
+
import contextlib
|
|
14
|
+
import email
|
|
15
|
+
import email.policy
|
|
16
|
+
import imaplib
|
|
17
|
+
import logging
|
|
18
|
+
import re
|
|
19
|
+
import smtplib
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
from datetime import UTC, datetime, timedelta
|
|
22
|
+
from email.message import EmailMessage
|
|
23
|
+
from email.utils import formatdate, make_msgid, parsedate_to_datetime
|
|
24
|
+
from html import unescape
|
|
25
|
+
from typing import TYPE_CHECKING
|
|
26
|
+
|
|
27
|
+
from ..models import Broker, MailboxConfig, Profile, utcnow
|
|
28
|
+
from ..netpolicy import host_of
|
|
29
|
+
from .mail_tls import TlsPinError, check_peer, client_context
|
|
30
|
+
|
|
31
|
+
if TYPE_CHECKING:
|
|
32
|
+
from ..netpolicy import NetPolicy
|
|
33
|
+
|
|
34
|
+
log = logging.getLogger(__name__)
|
|
35
|
+
|
|
36
|
+
_URL = re.compile(r"https://[^\s<>\"')\]]+", re.IGNORECASE)
|
|
37
|
+
_HREF = re.compile(r"""href\s*=\s*["']([^"']+)["']""", re.IGNORECASE)
|
|
38
|
+
_ANCHOR = re.compile(r"""<a\b[^>]*?href\s*=\s*["']([^"']+)["'][^>]*>(.*?)</a\s*>""",
|
|
39
|
+
re.IGNORECASE | re.DOTALL)
|
|
40
|
+
# SEC-18: links that would cancel/undo/unsubscribe (or are just boilerplate) are never clicked
|
|
41
|
+
_AVOID = re.compile(r"cancel|unsubscribe|opt.?in|not (you|me)|did ?n.?t (request|ask)|"
|
|
42
|
+
r"did not (request|ask)|report|preferences|privacy|terms|login|sign.?in",
|
|
43
|
+
re.IGNORECASE)
|
|
44
|
+
_CONFIRM = re.compile(r"confirm|verif|validate|activate|complete", re.IGNORECASE)
|
|
45
|
+
# A broker reply that asks for an identity document. Such mail is never auto-handled: no link
|
|
46
|
+
# in it is clicked and the runner opens an ID_UPLOAD to-do (Free My ID never sends ID).
|
|
47
|
+
_ID_REQUEST = re.compile(
|
|
48
|
+
r"(government|state)[- ]issued|photo (id\b|identification)|driver'?s?[- ]licen[cs]e|"
|
|
49
|
+
r"passport|(copy|scan|photo|picture|image) of (your |a )?(valid )?"
|
|
50
|
+
r"(photo )?(id\b|identification|identity|license|licence)|social security (number|card)|"
|
|
51
|
+
r"\bssn\b|last (4|four) digits|notari[sz]ed|utility bill|"
|
|
52
|
+
r"proof of (identity|address|residency)|identity document|id document",
|
|
53
|
+
re.IGNORECASE)
|
|
54
|
+
DATE_SLACK = timedelta(hours=1) # broker clock skew vs. our last_submitted_at
|
|
55
|
+
MAX_MESSAGES = 50 # per poll; we only need recent broker mail
|
|
56
|
+
MAX_MESSAGE_BYTES = 2_000_000
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class MailError(RuntimeError):
|
|
60
|
+
"""PII-free mail failure (message holds a short reason only)."""
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def _host_in(host: str | None, domains: list[str]) -> bool:
|
|
64
|
+
if not host:
|
|
65
|
+
return False
|
|
66
|
+
return any(host == d or host.endswith("." + d) for d in (x.lower() for x in domains))
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def confirmation_hosts(broker: Broker) -> list[str]:
|
|
70
|
+
return list(broker.confirmation_domains or broker.domains)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def build_message(mb: MailboxConfig, to: str, subject: str, body: str,
|
|
74
|
+
request_id: str) -> EmailMessage:
|
|
75
|
+
"""Plain-text message from the removal mailbox. Message-ID carries the request id so
|
|
76
|
+
replies can be tied to a Request without any PII."""
|
|
77
|
+
msg = EmailMessage()
|
|
78
|
+
msg["From"] = mb.address
|
|
79
|
+
msg["To"] = to
|
|
80
|
+
msg["Subject"] = subject
|
|
81
|
+
msg["Date"] = formatdate(localtime=False)
|
|
82
|
+
domain = mb.address.rpartition("@")[2] or "freemyid.invalid"
|
|
83
|
+
msg["Message-ID"] = make_msgid(idstring=f"freemyid.{request_id}", domain=domain)
|
|
84
|
+
msg.set_content(body)
|
|
85
|
+
return msg
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def extract_links(raw: bytes, allowed_domains: list[str]) -> list[str]:
|
|
89
|
+
"""Pure: https links in a raw RFC822 message whose host is inside `allowed_domains`."""
|
|
90
|
+
if len(raw) > MAX_MESSAGE_BYTES:
|
|
91
|
+
return []
|
|
92
|
+
msg = email.message_from_bytes(raw, policy=email.policy.default)
|
|
93
|
+
texts: list[str] = []
|
|
94
|
+
for part in msg.walk():
|
|
95
|
+
if part.get_content_maintype() != "text":
|
|
96
|
+
continue
|
|
97
|
+
try:
|
|
98
|
+
texts.append(part.get_content())
|
|
99
|
+
except (LookupError, ValueError):
|
|
100
|
+
continue
|
|
101
|
+
found: list[str] = []
|
|
102
|
+
text_of: dict[str, str] = {}
|
|
103
|
+
for t in texts:
|
|
104
|
+
for href, inner in _ANCHOR.findall(t):
|
|
105
|
+
u = unescape(href).strip()
|
|
106
|
+
label = " ".join(unescape(re.sub(r"<[^>]+>", " ", inner)).split())
|
|
107
|
+
text_of[u] = f"{text_of.get(u, '')} {label}".strip()
|
|
108
|
+
cands = [unescape(h) for h in _HREF.findall(t)] + _URL.findall(unescape(t))
|
|
109
|
+
for u in cands:
|
|
110
|
+
u = u.strip().rstrip(".,;")
|
|
111
|
+
if not u.lower().startswith("https://"):
|
|
112
|
+
continue
|
|
113
|
+
h = host_of(u)
|
|
114
|
+
if h and _host_in(h, allowed_domains) and u not in found:
|
|
115
|
+
found.append(u)
|
|
116
|
+
keep = [u for u in found if not _avoid(u, text_of.get(u, ""))]
|
|
117
|
+
return rank_links(keep, text_of)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _url_tail(u: str) -> str:
|
|
121
|
+
"""Path, query and fragment only (a host like privacy.broker.example is not a signal)."""
|
|
122
|
+
rest = u.split("://", 1)[-1]
|
|
123
|
+
i = min((rest.find(c) for c in "/?#" if c in rest), default=len(rest))
|
|
124
|
+
return rest[i:]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _avoid(u: str, text: str) -> bool:
|
|
128
|
+
return bool(_AVOID.search(_url_tail(u)) or _AVOID.search(text))
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def rank_links(links: list[str], text_of: dict[str, str] | None = None) -> list[str]:
|
|
132
|
+
"""Confirm/verify-looking links first (stable)."""
|
|
133
|
+
t = text_of or {}
|
|
134
|
+
return sorted(links, key=lambda u: not (_CONFIRM.search(_url_tail(u))
|
|
135
|
+
or _CONFIRM.search(t.get(u, ""))))
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def sender_domain(raw: bytes) -> str | None:
|
|
139
|
+
msg = email.message_from_bytes(raw[:65536], policy=email.policy.default)
|
|
140
|
+
frm = str(msg.get("From", ""))
|
|
141
|
+
m = re.search(r"@([A-Za-z0-9.-]+)", frm)
|
|
142
|
+
return m.group(1).lower().rstrip(".") if m else None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@dataclass
|
|
146
|
+
class ReplyScan:
|
|
147
|
+
"""What the broker's replies say, PII-free (links are confirmation URLs only)."""
|
|
148
|
+
links: list[str] = field(default_factory=list) # newest message first, ranked inside
|
|
149
|
+
id_requested: bool = False # some reply asks for an ID document -> human to-do
|
|
150
|
+
messages: int = 0 # distinct broker messages considered
|
|
151
|
+
duplicates: int = 0 # same Message-ID (or same bytes) seen again
|
|
152
|
+
stale: int = 0 # older than the submission this reply would answer
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def message_date(raw: bytes) -> datetime | None:
|
|
156
|
+
msg = email.message_from_bytes(raw[:65536], policy=email.policy.default)
|
|
157
|
+
try:
|
|
158
|
+
d = parsedate_to_datetime(str(msg.get("Date", "")))
|
|
159
|
+
except (TypeError, ValueError, IndexError):
|
|
160
|
+
return None
|
|
161
|
+
if d is None:
|
|
162
|
+
return None
|
|
163
|
+
return d if d.tzinfo else d.replace(tzinfo=UTC)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _message_key(raw: bytes) -> str:
|
|
167
|
+
msg = email.message_from_bytes(raw[:65536], policy=email.policy.default)
|
|
168
|
+
mid = str(msg.get("Message-ID", "")).strip()
|
|
169
|
+
if mid:
|
|
170
|
+
return mid
|
|
171
|
+
import hashlib
|
|
172
|
+
return hashlib.sha256(raw).hexdigest()
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _plain_text(raw: bytes) -> str:
|
|
176
|
+
msg = email.message_from_bytes(raw, policy=email.policy.default)
|
|
177
|
+
parts: list[str] = [str(msg.get("Subject", ""))]
|
|
178
|
+
for part in msg.walk():
|
|
179
|
+
if part.get_content_maintype() != "text":
|
|
180
|
+
continue
|
|
181
|
+
try:
|
|
182
|
+
parts.append(unescape(re.sub(r"<[^>]+>", " ", part.get_content())))
|
|
183
|
+
except (LookupError, ValueError):
|
|
184
|
+
continue
|
|
185
|
+
return " ".join(" ".join(parts).split())
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def asks_for_id(raw: bytes) -> bool:
|
|
189
|
+
"""Pure: does this message ask for an identity document (ID, licence, SSN, ...)?"""
|
|
190
|
+
if len(raw) > MAX_MESSAGE_BYTES:
|
|
191
|
+
return False
|
|
192
|
+
return bool(_ID_REQUEST.search(_plain_text(raw)))
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def scan_replies(raws: list[bytes], broker: Broker, since: datetime | None = None, *,
|
|
196
|
+
check_sender: bool = True) -> ReplyScan:
|
|
197
|
+
"""Pure reply matcher.
|
|
198
|
+
|
|
199
|
+
* only mail From a broker/confirmation domain (unless check_sender=False);
|
|
200
|
+
* the same message twice (Message-ID, else bytes) counts once;
|
|
201
|
+
* mail dated before `since` (minus DATE_SLACK) answers an earlier request: ignored, so
|
|
202
|
+
an old confirmation link is never clicked for a new submission;
|
|
203
|
+
* a message that asks for ID sets id_requested and contributes no links;
|
|
204
|
+
* links come newest message first (a broker that answers twice: its latest link wins).
|
|
205
|
+
"""
|
|
206
|
+
allowed = confirmation_hosts(broker)
|
|
207
|
+
out = ReplyScan()
|
|
208
|
+
seen: set[str] = set()
|
|
209
|
+
msgs: list[tuple[datetime | None, int, bytes]] = []
|
|
210
|
+
for i, raw in enumerate(raws):
|
|
211
|
+
if check_sender and not _host_in(sender_domain(raw), [*broker.domains, *allowed]):
|
|
212
|
+
continue
|
|
213
|
+
key = _message_key(raw)
|
|
214
|
+
if key in seen:
|
|
215
|
+
out.duplicates += 1
|
|
216
|
+
continue
|
|
217
|
+
seen.add(key)
|
|
218
|
+
d = message_date(raw)
|
|
219
|
+
if since is not None and d is not None:
|
|
220
|
+
ref = since if since.tzinfo else since.replace(tzinfo=UTC)
|
|
221
|
+
if d < ref - DATE_SLACK:
|
|
222
|
+
out.stale += 1
|
|
223
|
+
continue
|
|
224
|
+
msgs.append((d, i, raw))
|
|
225
|
+
out.messages = len(msgs)
|
|
226
|
+
floor = datetime.min.replace(tzinfo=UTC)
|
|
227
|
+
msgs.sort(key=lambda m: (m[0] or floor, m[1]), reverse=True)
|
|
228
|
+
for _d, _i, raw in msgs:
|
|
229
|
+
if asks_for_id(raw):
|
|
230
|
+
out.id_requested = True
|
|
231
|
+
continue
|
|
232
|
+
for u in extract_links(raw, allowed):
|
|
233
|
+
if u not in out.links:
|
|
234
|
+
out.links.append(u)
|
|
235
|
+
return out
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
class SmtpImapMailer:
|
|
239
|
+
"""Real mailer. Never used in dev/tests (they use FakeMailer)."""
|
|
240
|
+
|
|
241
|
+
def __init__(self, netpolicy: NetPolicy, password_getter=None, timeout_s: float = 30.0,
|
|
242
|
+
poll_interval_s: float = 30.0) -> None:
|
|
243
|
+
from .. import secrets
|
|
244
|
+
self.netpolicy = netpolicy
|
|
245
|
+
self._pw = password_getter or secrets.get_mailbox_password
|
|
246
|
+
self.timeout_s = timeout_s
|
|
247
|
+
self.poll_interval_s = poll_interval_s
|
|
248
|
+
self.last_raw: bytes | None = None # last sent message, for evidence
|
|
249
|
+
|
|
250
|
+
def _password(self, profile: Profile) -> str:
|
|
251
|
+
mb = profile.mailbox
|
|
252
|
+
if mb is None:
|
|
253
|
+
raise MailError("profile has no removal mailbox")
|
|
254
|
+
pw = self._pw(profile.id, mb)
|
|
255
|
+
if not pw:
|
|
256
|
+
raise MailError("mailbox password not in keychain")
|
|
257
|
+
return pw
|
|
258
|
+
|
|
259
|
+
def _check_host(self, host: str) -> None:
|
|
260
|
+
if not self.netpolicy.is_allowed(host):
|
|
261
|
+
raise MailError("mail host not on allowlist")
|
|
262
|
+
|
|
263
|
+
def send(self, profile: Profile, to: str, subject: str, body: str, request_id: str) -> str:
|
|
264
|
+
mb = profile.mailbox
|
|
265
|
+
if mb is None:
|
|
266
|
+
raise MailError("profile has no removal mailbox")
|
|
267
|
+
self._check_host(mb.smtp_host)
|
|
268
|
+
msg = build_message(mb, to, subject, body, request_id)
|
|
269
|
+
pw = self._password(profile)
|
|
270
|
+
pin = mb.tls_cert_sha256
|
|
271
|
+
try:
|
|
272
|
+
ctx = client_context(mb.smtp_host, pin)
|
|
273
|
+
if mb.smtp_starttls:
|
|
274
|
+
with smtplib.SMTP(mb.smtp_host, mb.smtp_port, timeout=self.timeout_s) as s:
|
|
275
|
+
s.starttls(context=ctx)
|
|
276
|
+
check_peer(s.sock, pin) # before any credential is sent
|
|
277
|
+
s.login(mb.username, pw)
|
|
278
|
+
s.send_message(msg)
|
|
279
|
+
else:
|
|
280
|
+
with smtplib.SMTP_SSL(mb.smtp_host, mb.smtp_port, context=ctx,
|
|
281
|
+
timeout=self.timeout_s) as s:
|
|
282
|
+
check_peer(s.sock, pin)
|
|
283
|
+
s.login(mb.username, pw)
|
|
284
|
+
s.send_message(msg)
|
|
285
|
+
except TlsPinError as e:
|
|
286
|
+
raise MailError(f"smtp refused: {e}") from None
|
|
287
|
+
except (smtplib.SMTPException, OSError) as e:
|
|
288
|
+
raise MailError(f"smtp failed: {type(e).__name__}") from None
|
|
289
|
+
self.last_raw = msg.as_bytes()
|
|
290
|
+
return str(msg["Message-ID"])
|
|
291
|
+
|
|
292
|
+
# ------------------------------------------------------------------ IMAP
|
|
293
|
+
def _imap(self, mb: MailboxConfig) -> imaplib.IMAP4:
|
|
294
|
+
"""Connected (not logged-in) IMAP. With a pin, the peer certificate is checked here,
|
|
295
|
+
so `login` is only ever reached on a matching certificate.
|
|
296
|
+
|
|
297
|
+
Plaintext IMAP (no SSL and no STARTTLS) is refused on EVERY host, loopback
|
|
298
|
+
included, before any socket is opened (SEC-53). The MailboxConfig validator
|
|
299
|
+
refuses it too, but it can be bypassed (`model_copy`), so this check stands alone."""
|
|
300
|
+
if not mb.imap_ssl and not mb.imap_starttls:
|
|
301
|
+
raise MailError("mail connection is not encrypted; refusing to sign in")
|
|
302
|
+
self._check_host(mb.imap_host)
|
|
303
|
+
pin = mb.tls_cert_sha256
|
|
304
|
+
ctx = client_context(mb.imap_host, pin)
|
|
305
|
+
if mb.imap_ssl and not mb.imap_starttls:
|
|
306
|
+
conn: imaplib.IMAP4 = imaplib.IMAP4_SSL(mb.imap_host, mb.imap_port,
|
|
307
|
+
ssl_context=ctx, timeout=self.timeout_s)
|
|
308
|
+
else:
|
|
309
|
+
conn = imaplib.IMAP4(mb.imap_host, mb.imap_port, timeout=self.timeout_s)
|
|
310
|
+
if mb.imap_starttls:
|
|
311
|
+
conn.starttls(ssl_context=ctx)
|
|
312
|
+
try:
|
|
313
|
+
check_peer(conn.sock, pin)
|
|
314
|
+
except TlsPinError:
|
|
315
|
+
with contextlib.suppress(Exception):
|
|
316
|
+
conn.shutdown()
|
|
317
|
+
raise
|
|
318
|
+
return conn
|
|
319
|
+
|
|
320
|
+
def fetch_since(self, profile: Profile, since: datetime) -> list[bytes]:
|
|
321
|
+
mb = profile.mailbox
|
|
322
|
+
if mb is None:
|
|
323
|
+
raise MailError("profile has no removal mailbox")
|
|
324
|
+
pw = self._password(profile)
|
|
325
|
+
out: list[bytes] = []
|
|
326
|
+
try:
|
|
327
|
+
conn = self._imap(mb)
|
|
328
|
+
try:
|
|
329
|
+
conn.login(mb.username, pw)
|
|
330
|
+
conn.select("INBOX", readonly=True)
|
|
331
|
+
typ, data = conn.search(None, "SINCE", since.strftime("%d-%b-%Y"))
|
|
332
|
+
if typ != "OK" or not data or not data[0]:
|
|
333
|
+
return []
|
|
334
|
+
for num in data[0].split()[-MAX_MESSAGES:]:
|
|
335
|
+
typ, msgdata = conn.fetch(num, "(BODY.PEEK[])")
|
|
336
|
+
if typ == "OK" and msgdata and isinstance(msgdata[0], tuple):
|
|
337
|
+
out.append(msgdata[0][1])
|
|
338
|
+
finally:
|
|
339
|
+
with contextlib.suppress(Exception):
|
|
340
|
+
conn.logout()
|
|
341
|
+
except TlsPinError as e:
|
|
342
|
+
raise MailError(f"imap refused: {e}") from None
|
|
343
|
+
except (imaplib.IMAP4.error, OSError) as e:
|
|
344
|
+
raise MailError(f"imap failed: {type(e).__name__}") from None
|
|
345
|
+
return out
|
|
346
|
+
|
|
347
|
+
def links_from(self, raws: list[bytes], broker: Broker) -> list[str]:
|
|
348
|
+
return scan_replies(raws, broker).links # filtered + ranked (SEC-18)
|
|
349
|
+
|
|
350
|
+
async def scan_replies(self, profile: Profile, broker: Broker, since: datetime,
|
|
351
|
+
timeout_s: int = 0) -> ReplyScan:
|
|
352
|
+
loop = asyncio.get_running_loop()
|
|
353
|
+
deadline = loop.time() + timeout_s
|
|
354
|
+
while True:
|
|
355
|
+
raws = await asyncio.to_thread(self.fetch_since, profile, since)
|
|
356
|
+
scan = scan_replies(raws, broker, since)
|
|
357
|
+
if scan.links or scan.id_requested or loop.time() >= deadline:
|
|
358
|
+
return scan
|
|
359
|
+
await asyncio.sleep(min(self.poll_interval_s, max(0.0, deadline - loop.time())))
|
|
360
|
+
|
|
361
|
+
async def wait_for_confirmation(self, profile: Profile, broker: Broker, since: datetime,
|
|
362
|
+
timeout_s: int = 600) -> list[str]:
|
|
363
|
+
scan = await self.scan_replies(profile, broker, since, timeout_s)
|
|
364
|
+
return [] if scan.id_requested else scan.links
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
@dataclass
|
|
368
|
+
class SentMail:
|
|
369
|
+
profile_id: str
|
|
370
|
+
to: str
|
|
371
|
+
subject: str
|
|
372
|
+
body: str
|
|
373
|
+
request_id: str
|
|
374
|
+
message_id: str
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
@dataclass
|
|
378
|
+
class FakeMailer:
|
|
379
|
+
"""In-memory mailer for dry runs, the dev fixture run and tests. Sends nothing."""
|
|
380
|
+
inbox: dict[str, list[bytes]] = field(default_factory=dict) # broker_id -> raw mails
|
|
381
|
+
sent: list[SentMail] = field(default_factory=list)
|
|
382
|
+
last_raw: bytes | None = None
|
|
383
|
+
|
|
384
|
+
def send(self, profile: Profile, to: str, subject: str, body: str, request_id: str) -> str:
|
|
385
|
+
mb = profile.mailbox or MailboxConfig(address="removal@freemyid.invalid",
|
|
386
|
+
imap_host="imap.freemyid.invalid",
|
|
387
|
+
smtp_host="smtp.freemyid.invalid",
|
|
388
|
+
username="fake")
|
|
389
|
+
msg = build_message(mb, to, subject, body, request_id)
|
|
390
|
+
self.last_raw = msg.as_bytes()
|
|
391
|
+
mid = str(msg["Message-ID"])
|
|
392
|
+
self.sent.append(SentMail(profile.id, to, subject, body, request_id, mid))
|
|
393
|
+
return mid
|
|
394
|
+
|
|
395
|
+
async def scan_replies(self, profile: Profile, broker: Broker, since: datetime,
|
|
396
|
+
timeout_s: int = 0) -> ReplyScan:
|
|
397
|
+
return scan_replies(self.inbox.get(broker.id, []), broker, since)
|
|
398
|
+
|
|
399
|
+
async def wait_for_confirmation(self, profile: Profile, broker: Broker, since: datetime,
|
|
400
|
+
timeout_s: int = 600) -> list[str]:
|
|
401
|
+
scan = await self.scan_replies(profile, broker, since, timeout_s)
|
|
402
|
+
return [] if scan.id_requested else scan.links
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def now() -> datetime:
|
|
406
|
+
return utcnow()
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
"""TLS for the removal mailbox, with certificate PINNING for local bridges.
|
|
2
|
+
|
|
3
|
+
Proton Mail Bridge runs on the user's own computer (127.0.0.1) and serves a self-signed
|
|
4
|
+
certificate, so normal CA verification always fails. We never switch verification off.
|
|
5
|
+
Instead the user confirms the certificate once (trust on first use) and its SHA-256
|
|
6
|
+
fingerprint is saved in `MailboxConfig.tls_cert_sha256`. On every connection:
|
|
7
|
+
|
|
8
|
+
* no pin -> `ssl.create_default_context()` (full CA + hostname checks, unchanged);
|
|
9
|
+
* pin + loopback -> handshake, then the peer's DER leaf certificate is hashed and compared
|
|
10
|
+
to the pin in constant time BEFORE any login/credential is sent;
|
|
11
|
+
a mismatch aborts the connection;
|
|
12
|
+
* pin + any other host -> refused with a clear error (pins are only for local bridges;
|
|
13
|
+
a real mail provider must pass normal verification).
|
|
14
|
+
"""
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import contextlib
|
|
18
|
+
import hashlib
|
|
19
|
+
import hmac
|
|
20
|
+
import imaplib
|
|
21
|
+
import smtplib
|
|
22
|
+
import ssl
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
from typing import TYPE_CHECKING, Literal
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from ..models import MailboxConfig
|
|
28
|
+
from ..netpolicy import NetPolicy
|
|
29
|
+
|
|
30
|
+
LOOPBACK_HOSTS = frozenset({"127.0.0.1", "::1", "localhost"})
|
|
31
|
+
# The connected peer must be one of these EXACT addresses (SEC-50). The name `localhost` is
|
|
32
|
+
# accepted in settings, but the resolver decides where it goes, so it is never trusted on
|
|
33
|
+
# its own: what counts is the address the socket actually reached.
|
|
34
|
+
LOOPBACK_PEER_IPS = frozenset({"127.0.0.1", "::1"})
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class TlsPinError(RuntimeError):
|
|
38
|
+
"""Pin refused or mismatched. Message is short, plain-language and PII-free."""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def is_loopback(host: str) -> bool:
|
|
42
|
+
return host.strip().lower().rstrip(".").strip("[]") in LOOPBACK_HOSTS
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def fingerprint(der: bytes) -> str:
|
|
46
|
+
"""Lowercase hex SHA-256 of a DER certificate (the format stored as the pin)."""
|
|
47
|
+
return hashlib.sha256(der).hexdigest()
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def format_fingerprint(fp: str) -> str:
|
|
51
|
+
"""'ab12cd..' -> 'AB:12:CD:..' (how certificate viewers and Bridge show it)."""
|
|
52
|
+
fp = fp.upper()
|
|
53
|
+
return ":".join(fp[i:i + 2] for i in range(0, len(fp), 2))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _unverified_context() -> ssl.SSLContext:
|
|
57
|
+
"""Handshake-only context. ONLY used where the peer certificate is checked against a
|
|
58
|
+
pin before anything is sent (`client_context` with a pin) or where nothing but the
|
|
59
|
+
certificate is read (`fetch_loopback_fingerprint`). Never returned for an unpinned
|
|
60
|
+
connection that carries credentials."""
|
|
61
|
+
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
|
|
62
|
+
ctx.minimum_version = ssl.TLSVersion.TLSv1_2
|
|
63
|
+
ctx.check_hostname = False
|
|
64
|
+
ctx.verify_mode = ssl.CERT_NONE # authenticity comes from the pin check instead
|
|
65
|
+
return ctx
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def client_context(host: str, pin: str | None) -> ssl.SSLContext:
|
|
69
|
+
"""TLS context for connecting to `host`. Callers MUST call `check_peer(sock, pin)`
|
|
70
|
+
after the handshake when `pin` is set, before sending credentials."""
|
|
71
|
+
if pin is None:
|
|
72
|
+
return ssl.create_default_context()
|
|
73
|
+
if not is_loopback(host):
|
|
74
|
+
raise TlsPinError("a certificate pin is only allowed for a mail app on this "
|
|
75
|
+
"computer (127.0.0.1, ::1 or localhost); remove the pin or use "
|
|
76
|
+
"the provider's normal settings")
|
|
77
|
+
if len(pin) != 64:
|
|
78
|
+
raise TlsPinError("saved certificate fingerprint is malformed")
|
|
79
|
+
return _unverified_context()
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def check_loopback_peer(sock: object) -> None:
|
|
83
|
+
"""Raise TlsPinError unless the socket is connected to exactly 127.0.0.1 or ::1.
|
|
84
|
+
Fails closed: no `getpeername`, an error, or an unexpected shape all refuse."""
|
|
85
|
+
try:
|
|
86
|
+
peer = sock.getpeername()[0] # type: ignore[attr-defined]
|
|
87
|
+
except (OSError, AttributeError, TypeError, IndexError, KeyError, ValueError):
|
|
88
|
+
peer = None
|
|
89
|
+
if not isinstance(peer, str) or peer not in LOOPBACK_PEER_IPS:
|
|
90
|
+
raise TlsPinError("the mail app is not on this computer; not trusting it")
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def check_peer(sock: object, pin: str | None) -> None:
|
|
94
|
+
"""Raise TlsPinError unless the TLS peer certificate's SHA-256 equals `pin` and the
|
|
95
|
+
peer is an exact loopback address (SEC-50). No-op without a pin (the default context
|
|
96
|
+
already verified the peer)."""
|
|
97
|
+
if pin is None:
|
|
98
|
+
return
|
|
99
|
+
get = getattr(sock, "getpeercert", None)
|
|
100
|
+
der = get(binary_form=True) if get else None
|
|
101
|
+
if not der:
|
|
102
|
+
raise TlsPinError("mail connection is not encrypted; refusing to sign in")
|
|
103
|
+
check_loopback_peer(sock)
|
|
104
|
+
if not hmac.compare_digest(fingerprint(der).encode(), pin.lower().encode()):
|
|
105
|
+
raise TlsPinError("the mail app's security certificate changed and does not match "
|
|
106
|
+
"the one you approved; not signing in")
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# ------------------------------------------------------------------ trust on first use
|
|
110
|
+
def _loopback_policy() -> NetPolicy:
|
|
111
|
+
"""Fallback for callers that do not pass a policy yet: the global config allowlist
|
|
112
|
+
plus loopback. Combined with `is_loopback` it still only ever admits this computer."""
|
|
113
|
+
from ..netpolicy import NetPolicy
|
|
114
|
+
return NetPolicy(None, allow_loopback=True)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def fetch_loopback_fingerprint(host: str, port: int, protocol: Literal["imap", "smtp"],
|
|
118
|
+
*, starttls: bool, timeout_s: float = 10.0,
|
|
119
|
+
netpolicy: NetPolicy | None = None) -> str:
|
|
120
|
+
"""Connect to a LOCAL mail app (e.g. Proton Bridge), read its certificate and return the
|
|
121
|
+
SHA-256 fingerprint for the user to confirm. Sends no username or password.
|
|
122
|
+
The host must pass `netpolicy.is_allowed` before any socket opens (SEC-51)."""
|
|
123
|
+
if not is_loopback(host):
|
|
124
|
+
raise TlsPinError("only a mail app on this computer can be trusted this way")
|
|
125
|
+
pol = netpolicy if netpolicy is not None else _loopback_policy()
|
|
126
|
+
if not pol.is_allowed(host):
|
|
127
|
+
raise TlsPinError("mail host not on allowlist")
|
|
128
|
+
ctx = _unverified_context()
|
|
129
|
+
sock: object
|
|
130
|
+
try:
|
|
131
|
+
if protocol == "imap":
|
|
132
|
+
if starttls:
|
|
133
|
+
conn = imaplib.IMAP4(host, port, timeout=timeout_s)
|
|
134
|
+
conn.starttls(ssl_context=ctx)
|
|
135
|
+
else:
|
|
136
|
+
conn = imaplib.IMAP4_SSL(host, port, ssl_context=ctx, timeout=timeout_s)
|
|
137
|
+
sock = conn.sock
|
|
138
|
+
try:
|
|
139
|
+
check_loopback_peer(sock)
|
|
140
|
+
der = sock.getpeercert(binary_form=True) # type: ignore[attr-defined]
|
|
141
|
+
finally:
|
|
142
|
+
with contextlib.suppress(Exception):
|
|
143
|
+
conn.logout()
|
|
144
|
+
else:
|
|
145
|
+
if starttls:
|
|
146
|
+
s = smtplib.SMTP(host, port, timeout=timeout_s)
|
|
147
|
+
s.starttls(context=ctx)
|
|
148
|
+
else:
|
|
149
|
+
s = smtplib.SMTP_SSL(host, port, context=ctx, timeout=timeout_s)
|
|
150
|
+
try:
|
|
151
|
+
check_loopback_peer(s.sock)
|
|
152
|
+
der = s.sock.getpeercert(binary_form=True) # type: ignore[union-attr]
|
|
153
|
+
finally:
|
|
154
|
+
with contextlib.suppress(Exception):
|
|
155
|
+
s.quit()
|
|
156
|
+
except (OSError, imaplib.IMAP4.error, smtplib.SMTPException) as e:
|
|
157
|
+
raise TlsPinError(f"could not reach the mail app ({type(e).__name__}); "
|
|
158
|
+
"is it running?") from None
|
|
159
|
+
if not der:
|
|
160
|
+
raise TlsPinError("the mail app did not offer a certificate")
|
|
161
|
+
return fingerprint(der)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@dataclass
|
|
165
|
+
class TrustResult:
|
|
166
|
+
fingerprint: str # lowercase hex, ready for MailboxConfig.tls_cert_sha256
|
|
167
|
+
display: str # 'AB:12:..' for showing to the user
|
|
168
|
+
message: str # plain-language prompt for the confirm screen
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def trust_on_first_use(mb: MailboxConfig, timeout_s: float = 10.0, *,
|
|
172
|
+
netpolicy: NetPolicy | None = None) -> TrustResult:
|
|
173
|
+
"""Read the certificate of a local mail app from both the IMAP and SMTP ports.
|
|
174
|
+
The caller shows `message` and saves `fingerprint` only after the user says yes.
|
|
175
|
+
Both ports must present the same certificate (Proton Bridge does).
|
|
176
|
+
Pass the caller's `netpolicy` (SEC-51); both hosts are checked before connecting."""
|
|
177
|
+
pol = netpolicy if netpolicy is not None else _loopback_policy()
|
|
178
|
+
for h in (mb.imap_host, mb.smtp_host):
|
|
179
|
+
if not is_loopback(h):
|
|
180
|
+
raise TlsPinError("only a mail app on this computer can be trusted this way")
|
|
181
|
+
if not pol.is_allowed(h):
|
|
182
|
+
raise TlsPinError("mail host not on allowlist")
|
|
183
|
+
imap_fp = fetch_loopback_fingerprint(
|
|
184
|
+
mb.imap_host, mb.imap_port, "imap",
|
|
185
|
+
starttls=mb.imap_starttls or not mb.imap_ssl, timeout_s=timeout_s, netpolicy=pol)
|
|
186
|
+
smtp_fp = fetch_loopback_fingerprint(
|
|
187
|
+
mb.smtp_host, mb.smtp_port, "smtp", starttls=mb.smtp_starttls, timeout_s=timeout_s,
|
|
188
|
+
netpolicy=pol)
|
|
189
|
+
if not hmac.compare_digest(imap_fp, smtp_fp):
|
|
190
|
+
raise TlsPinError("the mail app uses two different certificates; "
|
|
191
|
+
"this setup is not supported")
|
|
192
|
+
shown = format_fingerprint(imap_fp)
|
|
193
|
+
msg = ("Free My ID found the mail app on this computer (for example Proton Mail Bridge). "
|
|
194
|
+
"Its security code is:\n\n" + shown + "\n\n"
|
|
195
|
+
"If you just installed and started Proton Mail Bridge yourself, click Yes to "
|
|
196
|
+
"trust it. Free My ID will remember this code and refuse to sign in if it "
|
|
197
|
+
"ever changes.")
|
|
198
|
+
return TrustResult(imap_fp, shown, msg)
|