freemyid 0.2.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- freemyid/__init__.py +2 -0
- freemyid/_data/brokers/LICENSE-DATA.md +28 -0
- freemyid/_data/brokers/registry.sample.yaml +674 -0
- freemyid/_data/brokers/state_laws.yaml +448 -0
- freemyid/_data/schema/registry.schema.json +668 -0
- freemyid/_data/schema/setup-config.schema.json +538 -0
- freemyid/_data/schema/state_laws.schema.json +222 -0
- freemyid/cli.py +725 -0
- freemyid/cli_json.py +198 -0
- freemyid/config.py +43 -0
- freemyid/consent.py +82 -0
- freemyid/crypto.py +197 -0
- freemyid/doctor.py +374 -0
- freemyid/engine/__init__.py +0 -0
- freemyid/engine/_dev/__init__.py +101 -0
- freemyid/engine/_dev/devrun.py +101 -0
- freemyid/engine/_dev/inbox/devauto.eml +7 -0
- freemyid/engine/_dev/registry.yaml +106 -0
- freemyid/engine/_dev/site/devauto.example/confirm.html +1 -0
- freemyid/engine/_dev/site/devauto.example/name/jane-testperson/il.html +17 -0
- freemyid/engine/_dev/site/devauto.example/name/jane-testperson/nc.html +4 -0
- freemyid/engine/_dev/site/devauto.example/name/jane-testperson/oh.html +4 -0
- freemyid/engine/_dev/site/devauto.example/optout/done.html +1 -0
- freemyid/engine/_dev/site/devauto.example/optout.html +6 -0
- freemyid/engine/_dev/site/devblocked.example/name/jane-testperson/il.html +3 -0
- freemyid/engine/_dev/site/devblocked.example/optout.html +3 -0
- freemyid/engine/_dev/site/devfill.example/ajax.html +20 -0
- freemyid/engine/_dev/site/devfill.example/aria.html +16 -0
- freemyid/engine/_dev/site/devfill.example/consent.html +20 -0
- freemyid/engine/_dev/site/devfill.example/done.html +3 -0
- freemyid/engine/_dev/site/devfill.example/empty.html +1 -0
- freemyid/engine/_dev/site/devfill.example/fullname.html +12 -0
- freemyid/engine/_dev/site/devfill.example/labels.html +32 -0
- freemyid/engine/_dev/site/devfill.example/late.html +22 -0
- freemyid/engine/_dev/site/devfind.example/hostile.html +12 -0
- freemyid/engine/_dev/site/devfind.example/p/jane-testperson-shelbyville-il.html +6 -0
- freemyid/engine/_dev/site/devfind.example/p/jane-testperson-springfield-il.html +6 -0
- freemyid/engine/_dev/site/devfind.example/results.html +11 -0
- freemyid/engine/_dev/site/devfind.example/search.html +16 -0
- freemyid/engine/_dev/site/devweb.example/remove.html +1 -0
- freemyid/engine/browser.py +430 -0
- freemyid/engine/fill_for_me.py +630 -0
- freemyid/engine/filler.py +523 -0
- freemyid/engine/handoff.py +237 -0
- freemyid/engine/letters.py +125 -0
- freemyid/engine/listing_finder.py +538 -0
- freemyid/engine/mail.py +406 -0
- freemyid/engine/mail_tls.py +198 -0
- freemyid/engine/matcher.py +208 -0
- freemyid/engine/outbox.py +421 -0
- freemyid/engine/plugins.py +68 -0
- freemyid/engine/runner.py +791 -0
- freemyid/engine/tiers.py +107 -0
- freemyid/interfaces.py +164 -0
- freemyid/logging_setup.py +152 -0
- freemyid/models.py +700 -0
- freemyid/netpolicy.py +145 -0
- freemyid/notify.py +58 -0
- freemyid/paths.py +83 -0
- freemyid/registry.py +275 -0
- freemyid/scheduler/__init__.py +203 -0
- freemyid/scheduler/render.py +91 -0
- freemyid/secret_backends.py +562 -0
- freemyid/secrets.py +25 -0
- freemyid/setup_config.py +311 -0
- freemyid/state.py +151 -0
- freemyid/storage/__init__.py +3 -0
- freemyid/storage/db.py +394 -0
- freemyid/submitters/__init__.py +0 -0
- freemyid/submitters/assisted.py +171 -0
- freemyid/submitters/base.py +119 -0
- freemyid/submitters/email.py +40 -0
- freemyid/submitters/web_form.py +276 -0
- freemyid/templates/_identity.txt.j2 +19 -0
- freemyid/templates/_signature.txt.j2 +7 -0
- freemyid/templates/ccpa.txt.j2 +16 -0
- freemyid/templates/follow_up.txt.j2 +15 -0
- freemyid/templates/generic_removal.txt.j2 +12 -0
- freemyid/templates/legal_demand.txt.j2 +20 -0
- freemyid/templates/phone_script.txt.j2 +25 -0
- freemyid/templates/postal_letter.txt.j2 +20 -0
- freemyid/templates/state_privacy.txt.j2 +17 -0
- freemyid/wizard/__init__.py +0 -0
- freemyid/wizard/dashboard.py +401 -0
- freemyid/wizard/desktop.py +188 -0
- freemyid/wizard/draft.py +627 -0
- freemyid/wizard/firstrun.py +42 -0
- freemyid/wizard/flows.py +283 -0
- freemyid/wizard/report.py +86 -0
- freemyid/wizard/server.py +834 -0
- freemyid/wizard/static/__init__.py +0 -0
- freemyid/wizard/static/app.js +98 -0
- freemyid/wizard/static/freemyid.png +0 -0
- freemyid/wizard/static/style.css +85 -0
- freemyid/wizard/ui.py +343 -0
- freemyid-0.2.0.dev1.dist-info/METADATA +396 -0
- freemyid-0.2.0.dev1.dist-info/RECORD +100 -0
- freemyid-0.2.0.dev1.dist-info/WHEEL +4 -0
- freemyid-0.2.0.dev1.dist-info/entry_points.txt +2 -0
- freemyid-0.2.0.dev1.dist-info/licenses/LICENSE +661 -0
freemyid/doctor.py
ADDED
|
@@ -0,0 +1,374 @@
|
|
|
1
|
+
"""`freemyid doctor`: is this computer ready to run Free My ID? OWNER: automation-engineer.
|
|
2
|
+
|
|
3
|
+
Every check returns ok / warn / fail plus one plain fix line, and never raises. Checks are
|
|
4
|
+
read-only: nothing is installed, no login is attempted, and the only network traffic is a
|
|
5
|
+
TCP/TLS hello to the user's own mail servers (allowed by NetPolicy), skipped with
|
|
6
|
+
``network=False``.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import importlib
|
|
11
|
+
import os
|
|
12
|
+
import shutil
|
|
13
|
+
import socket
|
|
14
|
+
import ssl
|
|
15
|
+
import sys
|
|
16
|
+
from collections.abc import Callable
|
|
17
|
+
from dataclasses import asdict, dataclass
|
|
18
|
+
from datetime import date
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import TYPE_CHECKING, Any, Literal
|
|
21
|
+
|
|
22
|
+
from . import paths
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
25
|
+
from .models import MailboxConfig, Profile
|
|
26
|
+
|
|
27
|
+
Status = Literal["ok", "warn", "fail"]
|
|
28
|
+
REGISTRY_WARN_DAYS = 120
|
|
29
|
+
NET_TIMEOUT_S = 5.0
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class Check:
|
|
34
|
+
id: str
|
|
35
|
+
title: str
|
|
36
|
+
status: Status
|
|
37
|
+
detail: str
|
|
38
|
+
fix: str = ""
|
|
39
|
+
|
|
40
|
+
def as_dict(self) -> dict[str, str]:
|
|
41
|
+
return asdict(self)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _guard(cid: str, title: str, fn: Callable[[], Check]) -> Check:
|
|
45
|
+
try:
|
|
46
|
+
return fn()
|
|
47
|
+
except Exception as e: # noqa: BLE001 - a broken check must not break the others
|
|
48
|
+
return Check(cid, title, "warn", f"could not check ({type(e).__name__})",
|
|
49
|
+
"Run `freemyid doctor -v` and send the log to whoever helps you.")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
# --------------------------------------------------------------------------- python
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def check_python() -> Check:
|
|
56
|
+
v = sys.version_info
|
|
57
|
+
frozen = bool(getattr(sys, "frozen", False))
|
|
58
|
+
how = "installed app (frozen build)" if frozen else f"Python at {sys.executable}"
|
|
59
|
+
detail = f"Python {v.major}.{v.minor}.{v.micro}, {how}"
|
|
60
|
+
if (v.major, v.minor) < (3, 12):
|
|
61
|
+
return Check("python", "Python", "fail", detail,
|
|
62
|
+
"Free My ID needs Python 3.12 or newer: `pipx install --python python3.12 "
|
|
63
|
+
"freemyid`.")
|
|
64
|
+
return Check("python", "Python", "ok", detail)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# --------------------------------------------------------------------------- home folder
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def check_home() -> Check:
|
|
71
|
+
h = paths.home()
|
|
72
|
+
if not h.is_dir():
|
|
73
|
+
return Check("home", "Data folder", "fail", f"{h} is missing",
|
|
74
|
+
f"Create it with `mkdir -p {h}` or set FREEMYID_HOME.")
|
|
75
|
+
if not os.access(h, os.W_OK):
|
|
76
|
+
return Check("home", "Data folder", "fail", f"{h} is not writable",
|
|
77
|
+
f"Make it yours again: `chown -R $USER {h}`.")
|
|
78
|
+
if sys.platform != "win32":
|
|
79
|
+
mode = h.stat().st_mode & 0o777
|
|
80
|
+
if mode & 0o077:
|
|
81
|
+
return Check("home", "Data folder", "warn",
|
|
82
|
+
f"{h} can be read by other users (mode {mode:o})",
|
|
83
|
+
f"Make it private: `chmod 700 {h}`.")
|
|
84
|
+
return Check("home", "Data folder", "ok", f"{h} (private, mode {mode:o})")
|
|
85
|
+
return Check("home", "Data folder", "ok", str(h))
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
# --------------------------------------------------------------------------- secrets
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def check_secrets() -> Check:
|
|
92
|
+
"""Which secrets backend holds the encryption key, and can it be used right now."""
|
|
93
|
+
from . import secret_backends as sb
|
|
94
|
+
from .crypto import KeyUnavailable
|
|
95
|
+
title = "Secrets backend"
|
|
96
|
+
keychain = sb.KeyringBackend().usable()
|
|
97
|
+
try:
|
|
98
|
+
be = sb.select()
|
|
99
|
+
except KeyUnavailable as e:
|
|
100
|
+
return Check("secrets", title, "fail", str(e),
|
|
101
|
+
"Unlock or start the keychain, then consider `freemyid secrets migrate "
|
|
102
|
+
"--to passphrase`.")
|
|
103
|
+
except ValueError as e: # unknown backend name in config.yaml
|
|
104
|
+
return Check("secrets", title, "fail", str(e),
|
|
105
|
+
"Fix `secrets_backend` in config.yaml (keyring, passphrase or "
|
|
106
|
+
"systemd-creds), or remove it for automatic choice.")
|
|
107
|
+
extra = "" if be.name == "keyring" else (
|
|
108
|
+
"; system keychain also available" if keychain else "; no system keychain")
|
|
109
|
+
if be.usable():
|
|
110
|
+
if be.name == "passphrase" and not os.environ.get(sb.PASSPHRASE_FILE_ENV):
|
|
111
|
+
return Check("secrets", title, "warn",
|
|
112
|
+
f"passphrase, typed at a prompt{extra}",
|
|
113
|
+
f"Scheduled and headless runs can't type it: set "
|
|
114
|
+
f"{sb.PASSPHRASE_FILE_ENV} to a file only you can read (chmod 600), "
|
|
115
|
+
"or use systemd credentials.")
|
|
116
|
+
return Check("secrets", title, "ok", f"{be.name}{extra}")
|
|
117
|
+
fixes = {
|
|
118
|
+
"keyring": "Install and unlock a keychain (GNOME Keyring or KWallet), or run "
|
|
119
|
+
"`freemyid secrets migrate --to passphrase`.",
|
|
120
|
+
"passphrase": f"Set {sb.PASSPHRASE_FILE_ENV} to a file only you can read (chmod 600), "
|
|
121
|
+
"or run Free My ID in a terminal.",
|
|
122
|
+
"systemd-creds": f"Add LoadCredential={sb.CRED_NAME}:<path> to the service.",
|
|
123
|
+
}
|
|
124
|
+
return Check("secrets", title, "fail", f"{be.name} is selected but not usable here{extra}",
|
|
125
|
+
fixes.get(be.name, "Run `freemyid secrets status`."))
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
# --------------------------------------------------------------------------- data store
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def check_store(out: dict[str, Any]) -> Check:
|
|
132
|
+
"""Opens the encrypted database if it exists; puts the profiles in out['profiles']."""
|
|
133
|
+
if not paths.db_path().exists():
|
|
134
|
+
return Check("store", "Saved people", "warn", "nobody is set up yet",
|
|
135
|
+
"Run `freemyid setup` (or `freemyid setup --from config.yaml`).")
|
|
136
|
+
from .consent import consent_ok
|
|
137
|
+
from .crypto import KeyUnavailable
|
|
138
|
+
from .storage import Store
|
|
139
|
+
try:
|
|
140
|
+
st = Store()
|
|
141
|
+
except KeyUnavailable as e:
|
|
142
|
+
return Check("store", "Saved people", "fail", f"cannot unlock the data ({e})",
|
|
143
|
+
"Unlock your keychain, or restore the key; see the Secrets check.")
|
|
144
|
+
try:
|
|
145
|
+
profiles = st.list_profiles()
|
|
146
|
+
ok = sum(1 for p in profiles if consent_ok(p, st)[0])
|
|
147
|
+
finally:
|
|
148
|
+
st.close()
|
|
149
|
+
out["profiles"] = profiles
|
|
150
|
+
if not profiles:
|
|
151
|
+
return Check("store", "Saved people", "warn", "nobody is set up yet",
|
|
152
|
+
"Run `freemyid setup` (or `freemyid setup --from config.yaml`).")
|
|
153
|
+
if ok < len(profiles):
|
|
154
|
+
return Check("store", "Saved people", "warn",
|
|
155
|
+
f"{len(profiles)} saved, {len(profiles) - ok} without valid permission",
|
|
156
|
+
"Open Free My ID and give permission again (`freemyid profile list`).")
|
|
157
|
+
return Check("store", "Saved people", "ok", f"{len(profiles)} saved, all with permission")
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
# --------------------------------------------------------------------------- browser
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def playwright_browsers_dir() -> Path:
|
|
164
|
+
env = os.environ.get("PLAYWRIGHT_BROWSERS_PATH")
|
|
165
|
+
if env and env != "0":
|
|
166
|
+
return Path(env)
|
|
167
|
+
if env == "0":
|
|
168
|
+
try:
|
|
169
|
+
import playwright
|
|
170
|
+
return Path(playwright.__file__).parent / "driver" / "package" / ".local-browsers"
|
|
171
|
+
except ImportError:
|
|
172
|
+
pass
|
|
173
|
+
if sys.platform == "win32":
|
|
174
|
+
return Path(os.environ.get("LOCALAPPDATA", Path.home())) / "ms-playwright"
|
|
175
|
+
if sys.platform == "darwin":
|
|
176
|
+
return Path.home() / "Library" / "Caches" / "ms-playwright"
|
|
177
|
+
return Path(os.environ.get("XDG_CACHE_HOME", Path.home() / ".cache")) / "ms-playwright"
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
_CHANNEL_PATHS: dict[str, dict[str, list[str]]] = {
|
|
181
|
+
"chrome": {
|
|
182
|
+
"linux": ["google-chrome", "google-chrome-stable", "/opt/google/chrome/chrome"],
|
|
183
|
+
"darwin": ["/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"],
|
|
184
|
+
"win32": [r"%ProgramFiles%\Google\Chrome\Application\chrome.exe",
|
|
185
|
+
r"%ProgramFiles(x86)%\Google\Chrome\Application\chrome.exe",
|
|
186
|
+
r"%LOCALAPPDATA%\Google\Chrome\Application\chrome.exe"],
|
|
187
|
+
},
|
|
188
|
+
"msedge": {
|
|
189
|
+
"linux": ["microsoft-edge", "microsoft-edge-stable", "/opt/microsoft/msedge/msedge"],
|
|
190
|
+
"darwin": ["/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge"],
|
|
191
|
+
"win32": [r"%ProgramFiles(x86)%\Microsoft\Edge\Application\msedge.exe",
|
|
192
|
+
r"%ProgramFiles%\Microsoft\Edge\Application\msedge.exe"],
|
|
193
|
+
},
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def system_browser(channel: str | None) -> str | None:
|
|
198
|
+
if not channel:
|
|
199
|
+
return None
|
|
200
|
+
plat = "linux" if sys.platform.startswith("linux") else sys.platform
|
|
201
|
+
for cand in _CHANNEL_PATHS.get(channel, {}).get(plat, []):
|
|
202
|
+
c = os.path.expandvars(cand)
|
|
203
|
+
if os.sep in c or "/" in c:
|
|
204
|
+
if Path(c).is_file():
|
|
205
|
+
return c
|
|
206
|
+
elif shutil.which(c):
|
|
207
|
+
return shutil.which(c)
|
|
208
|
+
return None
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def check_browser() -> Check:
|
|
212
|
+
from . import config as cfgmod
|
|
213
|
+
cfg = cfgmod.load()
|
|
214
|
+
title = "Browser for automatic removals"
|
|
215
|
+
if not getattr(cfg, "browser_automation", True):
|
|
216
|
+
return Check("browser", title, "ok", "automatic tier is turned off (email and "
|
|
217
|
+
"assisted removals still work)")
|
|
218
|
+
try:
|
|
219
|
+
importlib.import_module("playwright")
|
|
220
|
+
except ImportError:
|
|
221
|
+
return Check("browser", title, "warn", "Playwright is not installed",
|
|
222
|
+
"`pipx inject freemyid playwright` (or reinstall Free My ID), then "
|
|
223
|
+
"`playwright install chromium`.")
|
|
224
|
+
exe = os.environ.get("FREEMYID_BROWSER_EXECUTABLE")
|
|
225
|
+
if exe:
|
|
226
|
+
if Path(exe).is_file():
|
|
227
|
+
return Check("browser", title, "ok", f"FREEMYID_BROWSER_EXECUTABLE ({exe})")
|
|
228
|
+
return Check("browser", title, "fail", f"FREEMYID_BROWSER_EXECUTABLE points to a "
|
|
229
|
+
f"missing file ({exe})", "Fix or unset FREEMYID_BROWSER_EXECUTABLE.")
|
|
230
|
+
channels = (["msedge"] if sys.platform == "win32" else []) + [cfg.browser_channel or ""]
|
|
231
|
+
for ch in dict.fromkeys(c for c in channels if c):
|
|
232
|
+
if system_browser(ch):
|
|
233
|
+
return Check("browser", title, "ok", f"system browser ({ch})")
|
|
234
|
+
d = playwright_browsers_dir()
|
|
235
|
+
found = sorted(d.glob("chromium-*")) + sorted(d.glob("chromium_headless_shell-*")) \
|
|
236
|
+
if d.is_dir() else []
|
|
237
|
+
if found:
|
|
238
|
+
return Check("browser", title, "ok", f"Playwright Chromium ({found[-1].name})")
|
|
239
|
+
return Check("browser", title, "warn", "no Chrome, Edge or Playwright Chromium found",
|
|
240
|
+
"Run `playwright install chromium` (email and assisted removals work "
|
|
241
|
+
"without it).")
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
# --------------------------------------------------------------------------- scheduler
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def check_scheduler() -> Check:
|
|
248
|
+
from . import scheduler
|
|
249
|
+
title = "Background checks"
|
|
250
|
+
try:
|
|
251
|
+
detail = scheduler.status()
|
|
252
|
+
except Exception as e: # noqa: BLE001
|
|
253
|
+
return Check("scheduler", title, "warn", f"could not read status ({type(e).__name__})",
|
|
254
|
+
"Run `freemyid schedule install`.")
|
|
255
|
+
low = detail.lower()
|
|
256
|
+
if "not supported" in low:
|
|
257
|
+
return Check("scheduler", title, "warn", detail,
|
|
258
|
+
"Use cron or a container scheduler to run `freemyid run --headless`.")
|
|
259
|
+
if "not installed" in low:
|
|
260
|
+
return Check("scheduler", title, "warn", "not installed",
|
|
261
|
+
"Run `freemyid schedule install` (or run `freemyid run --headless` from "
|
|
262
|
+
"cron/systemd yourself).")
|
|
263
|
+
return Check("scheduler", title, "ok", detail)
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
# --------------------------------------------------------------------------- mailbox
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def _is_loopback(host: str) -> bool:
|
|
270
|
+
return host in ("localhost", "127.0.0.1", "::1")
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def probe(host: str, port: int, tls: bool, timeout: float = NET_TIMEOUT_S) -> str | None:
|
|
274
|
+
"""TCP connect (and a TLS handshake when `tls`). None if fine, else a short reason.
|
|
275
|
+
|
|
276
|
+
No login: nothing but the TLS hello is sent.
|
|
277
|
+
"""
|
|
278
|
+
try:
|
|
279
|
+
with socket.create_connection((host, port), timeout=timeout) as s:
|
|
280
|
+
if tls:
|
|
281
|
+
ctx = ssl.create_default_context()
|
|
282
|
+
with ctx.wrap_socket(s, server_hostname=host):
|
|
283
|
+
pass
|
|
284
|
+
except ssl.SSLError as e:
|
|
285
|
+
return f"TLS failed ({type(e).__name__})"
|
|
286
|
+
except (OSError, RuntimeError) as e:
|
|
287
|
+
return f"cannot connect ({type(e).__name__})"
|
|
288
|
+
return None
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _mailbox_targets(mb: MailboxConfig) -> list[tuple[str, str, int, bool]]:
|
|
292
|
+
imap_tls = mb.imap_ssl and not _is_loopback(mb.imap_host)
|
|
293
|
+
smtp_tls = (not mb.smtp_starttls and mb.smtp_port == 465
|
|
294
|
+
and not _is_loopback(mb.smtp_host))
|
|
295
|
+
return [("incoming", mb.imap_host, mb.imap_port, imap_tls),
|
|
296
|
+
("outgoing", mb.smtp_host, mb.smtp_port, smtp_tls)]
|
|
297
|
+
|
|
298
|
+
|
|
299
|
+
def check_mailboxes(profiles: list[Profile] | None, network: bool = True,
|
|
300
|
+
prober: Callable[[str, int, bool], str | None] | None = None) -> Check:
|
|
301
|
+
title = "Removal mailbox"
|
|
302
|
+
with_mb = [p for p in (profiles or []) if p.mailbox]
|
|
303
|
+
if not with_mb:
|
|
304
|
+
return Check("mailbox", title, "warn", "no removal mailbox set up",
|
|
305
|
+
"Add one with `freemyid profile edit <name>` so removal emails can be "
|
|
306
|
+
"sent and replies read.")
|
|
307
|
+
if not network:
|
|
308
|
+
return Check("mailbox", title, "ok", f"{len(with_mb)} configured (not contacted: "
|
|
309
|
+
"--offline)")
|
|
310
|
+
from .netpolicy import NetPolicy
|
|
311
|
+
pol = NetPolicy(None, with_mb, allow_loopback=True)
|
|
312
|
+
prober = prober or probe
|
|
313
|
+
problems: list[str] = []
|
|
314
|
+
for p in with_mb:
|
|
315
|
+
assert p.mailbox is not None
|
|
316
|
+
for which, host, port, tls in _mailbox_targets(p.mailbox):
|
|
317
|
+
if not pol.is_allowed(host):
|
|
318
|
+
problems.append(f"{p.label}: {which} server not allowed")
|
|
319
|
+
continue
|
|
320
|
+
why = prober(host, port, tls)
|
|
321
|
+
if why:
|
|
322
|
+
problems.append(f"{p.label}: {which} {host}:{port} {why}")
|
|
323
|
+
if problems:
|
|
324
|
+
return Check("mailbox", title, "fail", "; ".join(problems),
|
|
325
|
+
"Check the internet connection and the mail server names "
|
|
326
|
+
"(`freemyid profile edit <name>`); for Proton, start Proton Mail Bridge.")
|
|
327
|
+
return Check("mailbox", title, "ok", f"{len(with_mb)} reachable (no login attempted)")
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
# --------------------------------------------------------------------------- registry
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def check_registry(today: date | None = None) -> Check:
|
|
334
|
+
from . import config as cfgmod
|
|
335
|
+
from . import registry as regmod
|
|
336
|
+
title = "Broker list"
|
|
337
|
+
reg = regmod.load_registry()
|
|
338
|
+
age = ((today or date.today()) - reg.updated).days
|
|
339
|
+
detail = (f"{regmod.registry_source()} list, version {reg.version}, "
|
|
340
|
+
f"{len(reg.brokers)} brokers, updated {age} day(s) ago")
|
|
341
|
+
fix = ("Run `freemyid registry update`." if cfgmod.load().registry_url else
|
|
342
|
+
"Install the latest Free My ID version to get the newest broker list.")
|
|
343
|
+
if regmod.load_warnings:
|
|
344
|
+
return Check("registry", title, "warn", detail + "; " + "; ".join(regmod.load_warnings),
|
|
345
|
+
fix)
|
|
346
|
+
if age > REGISTRY_WARN_DAYS:
|
|
347
|
+
return Check("registry", title, "warn", detail, fix)
|
|
348
|
+
return Check("registry", title, "ok", detail)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
# --------------------------------------------------------------------------- all
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def run_checks(network: bool = True,
|
|
355
|
+
prober: Callable[[str, int, bool], str | None] | None = None) -> list[Check]:
|
|
356
|
+
box: dict[str, Any] = {}
|
|
357
|
+
checks = [
|
|
358
|
+
_guard("python", "Python", check_python),
|
|
359
|
+
_guard("home", "Data folder", check_home),
|
|
360
|
+
_guard("secrets", "Secrets backend", check_secrets),
|
|
361
|
+
_guard("store", "Saved people", lambda: check_store(box)),
|
|
362
|
+
_guard("browser", "Browser for automatic removals", check_browser),
|
|
363
|
+
_guard("scheduler", "Background checks", check_scheduler),
|
|
364
|
+
_guard("mailbox", "Removal mailbox",
|
|
365
|
+
lambda: check_mailboxes(box.get("profiles"), network, prober)),
|
|
366
|
+
_guard("registry", "Broker list", check_registry),
|
|
367
|
+
]
|
|
368
|
+
return checks
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def doc(checks: list[Check]) -> dict[str, Any]:
|
|
372
|
+
counts = {s: sum(1 for c in checks if c.status == s) for s in ("ok", "warn", "fail")}
|
|
373
|
+
return {"schema": "doctor/1", "ok": counts["fail"] == 0, "counts": counts,
|
|
374
|
+
"checks": [c.as_dict() for c in checks]}
|
|
File without changes
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Dev fixtures: a fake registry, a local fixture web server and a fake inbox.
|
|
2
|
+
|
|
3
|
+
Used by tests/test_e2e_dryrun.py and `python -m freemyid.engine._dev.devrun`. Nothing here
|
|
4
|
+
talks to the internet:
|
|
5
|
+
|
|
6
|
+
* `fixture_registry()` fake brokers on reserved `.example` hosts (registry.yaml).
|
|
7
|
+
* `fixture_server()` context manager: serves `site/<host>/<path>` on 127.0.0.1:<random>
|
|
8
|
+
and sets FREEMYID_DEV_FIXTURES=<base url> while active. With that set,
|
|
9
|
+
BrowserManager answers every allowlisted request from this server
|
|
10
|
+
(route.fetch -> route.fulfill) and the runner/outbox use FakeMailer.
|
|
11
|
+
* `fixture_inbox()` broker_id -> raw .eml messages for FakeMailer.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import contextlib
|
|
16
|
+
import os
|
|
17
|
+
import threading
|
|
18
|
+
from collections.abc import Iterator
|
|
19
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from urllib.parse import unquote, urlsplit
|
|
22
|
+
|
|
23
|
+
from ...models import Registry
|
|
24
|
+
from ..outbox import DEV_FIXTURES_ENV
|
|
25
|
+
|
|
26
|
+
DEV_DIR = Path(__file__).resolve().parent
|
|
27
|
+
SITE_DIR = DEV_DIR / "site"
|
|
28
|
+
INBOX_DIR = DEV_DIR / "inbox"
|
|
29
|
+
REGISTRY_PATH = DEV_DIR / "registry.yaml"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def fixture_registry() -> Registry:
|
|
33
|
+
from ...registry import load_registry
|
|
34
|
+
return load_registry(REGISTRY_PATH)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def fixture_inbox() -> dict[str, list[bytes]]:
|
|
38
|
+
out: dict[str, list[bytes]] = {}
|
|
39
|
+
for f in sorted(INBOX_DIR.glob("*.eml")):
|
|
40
|
+
out.setdefault(f.stem.split(".")[0], []).append(f.read_bytes())
|
|
41
|
+
return out
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def resolve(path: str) -> Path | None:
|
|
45
|
+
"""Map '/<host>/<path>' to a file under SITE_DIR (path, path.html, path/index.html)."""
|
|
46
|
+
rel = unquote(urlsplit(path).path).lstrip("/")
|
|
47
|
+
if not rel or "\\" in rel or any(p in ("..", "") for p in rel.rstrip("/").split("/")):
|
|
48
|
+
return None
|
|
49
|
+
base = (SITE_DIR / rel.rstrip("/")).resolve()
|
|
50
|
+
if SITE_DIR.resolve() not in base.parents:
|
|
51
|
+
return None
|
|
52
|
+
for cand in (base, base.with_name(base.name + ".html"), base / "index.html"):
|
|
53
|
+
if cand.is_file():
|
|
54
|
+
return cand
|
|
55
|
+
return None
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class _Handler(BaseHTTPRequestHandler):
|
|
59
|
+
def _serve(self) -> None:
|
|
60
|
+
if self.command == "POST":
|
|
61
|
+
n = int(self.headers.get("Content-Length") or 0)
|
|
62
|
+
self.rfile.read(min(n, 1_000_000))
|
|
63
|
+
f = resolve(self.path)
|
|
64
|
+
if f is None:
|
|
65
|
+
body = b"<html><body><h1>Not found</h1></body></html>"
|
|
66
|
+
self.send_response(404)
|
|
67
|
+
else:
|
|
68
|
+
body = f.read_bytes()
|
|
69
|
+
self.send_response(200)
|
|
70
|
+
ctype = "text/html; charset=utf-8" if f is None or f.suffix in (".html", "") \
|
|
71
|
+
else "application/octet-stream"
|
|
72
|
+
self.send_header("Content-Type", ctype)
|
|
73
|
+
self.send_header("Content-Length", str(len(body)))
|
|
74
|
+
self.end_headers()
|
|
75
|
+
self.wfile.write(body)
|
|
76
|
+
|
|
77
|
+
do_GET = _serve
|
|
78
|
+
do_POST = _serve
|
|
79
|
+
|
|
80
|
+
def log_message(self, format: str, *args: object) -> None:
|
|
81
|
+
return
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@contextlib.contextmanager
|
|
85
|
+
def fixture_server() -> Iterator[str]:
|
|
86
|
+
"""Serve the fixture site on 127.0.0.1 and point the engine at it. Yields the base URL."""
|
|
87
|
+
srv = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)
|
|
88
|
+
t = threading.Thread(target=srv.serve_forever, daemon=True)
|
|
89
|
+
t.start()
|
|
90
|
+
base = f"http://127.0.0.1:{srv.server_address[1]}"
|
|
91
|
+
old = os.environ.get(DEV_FIXTURES_ENV)
|
|
92
|
+
os.environ[DEV_FIXTURES_ENV] = base
|
|
93
|
+
try:
|
|
94
|
+
yield base
|
|
95
|
+
finally:
|
|
96
|
+
if old is None:
|
|
97
|
+
os.environ.pop(DEV_FIXTURES_ENV, None)
|
|
98
|
+
else:
|
|
99
|
+
os.environ[DEV_FIXTURES_ENV] = old
|
|
100
|
+
srv.shutdown()
|
|
101
|
+
srv.server_close()
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Run the whole loop end to end against the dev fixtures. No internet, no real email.
|
|
2
|
+
|
|
3
|
+
.venv/bin/python -m freemyid.engine._dev.devrun # `freemyid run --dry-run`
|
|
4
|
+
.venv/bin/python -m freemyid.engine._dev.devrun --no-mailbox # D2 manual-send path
|
|
5
|
+
.venv/bin/python -m freemyid.engine._dev.devrun --submit # fixture "submissions"
|
|
6
|
+
.venv/bin/python -m freemyid.engine._dev.devrun --scheduled # quiet D4 run
|
|
7
|
+
|
|
8
|
+
It makes a throwaway FREEMYID_HOME (weak test key, fake person Jane Q Testperson), installs
|
|
9
|
+
the fixture registry as that home's registry cache, starts the fixture server and calls the
|
|
10
|
+
real CLI. `--submit` enables "real" submissions in the throwaway home only: every request is
|
|
11
|
+
answered by the local fixture server and mail goes to FakeMailer.
|
|
12
|
+
"""
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import argparse
|
|
16
|
+
import os
|
|
17
|
+
import shutil
|
|
18
|
+
import sys
|
|
19
|
+
import tempfile
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _setup_home(home: Path, *, mailbox: bool, submit: bool) -> None:
|
|
24
|
+
os.environ["FREEMYID_HOME"] = str(home)
|
|
25
|
+
os.environ.setdefault("FREEMYID_TEST_KEY", "devrun-weak-key-fake-data-only")
|
|
26
|
+
os.environ["FREEMYID_ALLOW_TEST_KEY"] = "1"
|
|
27
|
+
from freemyid import config as cfgmod
|
|
28
|
+
from freemyid import paths
|
|
29
|
+
from freemyid.consent import CURRENT_TEXT_VERSION
|
|
30
|
+
from freemyid.models import Address, Consent, ConsentBasis, MailboxConfig, PersonName, Profile
|
|
31
|
+
from freemyid.storage import Store
|
|
32
|
+
|
|
33
|
+
from . import REGISTRY_PATH
|
|
34
|
+
shutil.copyfile(REGISTRY_PATH, paths.registry_cache_path())
|
|
35
|
+
cfg = cfgmod.Config(min_delay_s=0, max_delay_s=0, max_retries=1,
|
|
36
|
+
real_submissions_enabled=submit)
|
|
37
|
+
cfgmod.save(cfg)
|
|
38
|
+
store = Store()
|
|
39
|
+
if not store.list_profiles():
|
|
40
|
+
mb = MailboxConfig(address="removals.testperson@mail.example",
|
|
41
|
+
imap_host="imap.mail.example", smtp_host="smtp.mail.example",
|
|
42
|
+
username="removals.testperson@mail.example") if mailbox else None
|
|
43
|
+
store.save_profile(Profile(
|
|
44
|
+
label="Tester",
|
|
45
|
+
consent=Consent(basis=ConsentBasis.SELF, attested_by="Jane Q Testperson",
|
|
46
|
+
text_version=CURRENT_TEXT_VERSION),
|
|
47
|
+
name=PersonName(first="Jane", middle="Q", last="Testperson"),
|
|
48
|
+
addresses=[Address(city="Springfield", state="IL", zip="62704", current=True)],
|
|
49
|
+
age=46, relatives=["John Testperson"], mailbox=mb))
|
|
50
|
+
store.close()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _show(home: Path) -> None:
|
|
54
|
+
from freemyid.storage import Store
|
|
55
|
+
store = Store()
|
|
56
|
+
print("\n--- statuses (broker: state / tier)")
|
|
57
|
+
for st in sorted(store.list_statuses(), key=lambda s: s.broker_id):
|
|
58
|
+
print(f" {st.broker_id:12} {st.state.value:22} {st.tier.value if st.tier else '-'}")
|
|
59
|
+
print("--- open to-dos")
|
|
60
|
+
for t in store.list_todos():
|
|
61
|
+
print(f" {t.kind.value:20} {t.broker_id or '-':12} {t.title} "
|
|
62
|
+
f"[copy: {', '.join(t.copy_fields)}]")
|
|
63
|
+
print("--- requests")
|
|
64
|
+
for r in store.list_requests():
|
|
65
|
+
print(f" {r.broker_id:12} {r.kind.value:12} {r.status.value:20} dry_run={r.dry_run}")
|
|
66
|
+
n = sum(len(store.list_evidence(p.id)) for p in store.list_profiles())
|
|
67
|
+
print(f"--- evidence: {n} item(s)")
|
|
68
|
+
store.close()
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def main(argv: list[str] | None = None) -> int:
|
|
72
|
+
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
73
|
+
ap.add_argument("--home", type=Path, help="FREEMYID_HOME to use (default: new temp dir)")
|
|
74
|
+
ap.add_argument("--no-mailbox", action="store_true")
|
|
75
|
+
ap.add_argument("--submit", action="store_true", help="fixture submissions, not dry run")
|
|
76
|
+
ap.add_argument("--scheduled", action="store_true")
|
|
77
|
+
ap.add_argument("--keep", action="store_true", help="keep the temp home")
|
|
78
|
+
a = ap.parse_args(argv)
|
|
79
|
+
home = a.home or Path(tempfile.mkdtemp(prefix="freemyid-devrun-"))
|
|
80
|
+
_setup_home(home, mailbox=not a.no_mailbox, submit=a.submit)
|
|
81
|
+
|
|
82
|
+
from freemyid.cli import app
|
|
83
|
+
|
|
84
|
+
from . import fixture_server
|
|
85
|
+
args = ["run"] + ([] if a.submit else ["--dry-run"]) + (["--scheduled"] if a.scheduled
|
|
86
|
+
else [])
|
|
87
|
+
code = 0
|
|
88
|
+
with fixture_server() as base:
|
|
89
|
+
print(f"fixture server: {base} home: {home}")
|
|
90
|
+
try:
|
|
91
|
+
app(args, standalone_mode=False)
|
|
92
|
+
except SystemExit as e:
|
|
93
|
+
code = int(e.code or 0)
|
|
94
|
+
_show(home)
|
|
95
|
+
if not a.keep and not a.home:
|
|
96
|
+
shutil.rmtree(home, ignore_errors=True)
|
|
97
|
+
return code
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
if __name__ == "__main__":
|
|
101
|
+
sys.exit(main())
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
From: Dev Auto People <noreply@devauto.example>
|
|
2
|
+
To: removals.testperson@mail.example
|
|
3
|
+
Subject: Confirm your opt-out
|
|
4
|
+
Content-Type: text/html; charset=utf-8
|
|
5
|
+
|
|
6
|
+
<p>Click to confirm: <a href="https://devauto.example/confirm?token=dev123">confirm</a></p>
|
|
7
|
+
<p>Not this one: <a href="https://evil.example/steal">x</a></p>
|