freemyid 0.2.0.dev1__py3-none-any.whl

freemyid/netpolicy.py

@@ -0,0 +1,145 @@
+"""Network allowlist. OWNER: orchestrator.
+
+The ONLY hosts Free My ID may contact: broker domains from the registry, the profile's own
+mail servers, the configured registry hosts, and user-approved extras. The engine must call
+`is_allowed` for every browser request (Playwright route) and before any httpx/SMTP/IMAP.
+
+Loopback (localhost, 127.0.0.1, ::1) is NOT allowed by default: broker-page JS in the
+engine browser must never reach the local wizard. Callers that legitimately need it (the
+wizard, tests) opt in with `NetPolicy(..., allow_loopback=True)` or `policy.with_loopback()`.
+A profile's own mailbox host (e.g. Proton Bridge on 127.0.0.1) is still allowed for mail,
+but `for_broker` never hands loopback to a browser run unless loopback was opted in.
+"""
+from __future__ import annotations
+
+import copy
+from urllib.parse import urlsplit
+
+from . import config
+from .models import Profile, Registry
+
+LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"})
+
+
+def _norm(host: str) -> str:
+    return host.strip().lower().rstrip(".").strip("[]")
+
+
+def _host_ok(host: str, allowed: set[str]) -> bool:
+    host = _norm(host)
+    return bool(host) and any(host == a or host.endswith("." + a) for a in allowed)
+
+
+def _is_loopback(host: str) -> bool:
+    h = _norm(host)
+    return h in LOOPBACK_HOSTS or h.endswith(".localhost") or h.startswith("127.")
+
+
+def host_of(url_or_host: str) -> str | None:
+    """Hostname of a URL (or a bare host); None if it is unsafe or unparsable.
+
+    Rejects backslashes (urlsplit reads `https://evil.com\\@broker.com` as broker.com,
+    browsers read it as evil.com), whitespace and control characters.
+    """
+    s = url_or_host
+    if not s or "\\" in s or any(c.isspace() or ord(c) < 0x20 or ord(c) == 0x7F for c in s):
+        return None
+    if "://" in s:
+        try:
+            host = urlsplit(s).hostname
+        except ValueError:
+            return None
+    else:
+        host = s
+    return _norm(host) if host else None
+
+
+class NetPolicy:
+    def __init__(self, registry: Registry | None, profiles: list[Profile] | None = None,
+                 *, allow_loopback: bool = False) -> None:
+        cfg = config.load()
+        self.allow_loopback = allow_loopback
+        self.allowed: set[str] = set()
+        for b in registry.brokers if registry else []:
+            self.allowed.update(_norm(d) for d in b.domains)
+            self.allowed.update(_norm(d) for d in b.confirmation_domains)
+        for p in profiles or []:
+            if p.mailbox:
+                self.allowed.update({_norm(p.mailbox.imap_host), _norm(p.mailbox.smtp_host)})
+        for u in (cfg.registry_url, cfg.registry_sha256_url):
+            if u:
+                self.allowed.add(host_of(u) or "")
+        self.allowed.update(_norm(h) for h in cfg.extra_allowed_hosts)
+        if allow_loopback:
+            self.allowed.update(LOOPBACK_HOSTS)
+        self.allowed.discard("")
+
+    def with_loopback(self) -> NetPolicy:
+        """Copy of this policy that also allows localhost/127.0.0.1/::1 (wizard, tests)."""
+        other = copy.copy(self)
+        other.allowed = set(self.allowed) | LOOPBACK_HOSTS
+        other.allow_loopback = True
+        return other
+
+    def is_allowed(self, url_or_host: str) -> bool:
+        host = host_of(url_or_host)
+        return bool(host) and _host_ok(str(host), self.allowed)
+
+    def for_broker(self, broker_domains: list[str], extra: list[str] | None = None) -> set[str]:
+        """Narrow allowlist for a single broker run: (broker + extras) ∩ global allowlist.
+
+        Hosts not covered by the global allowlist are dropped, and loopback is dropped
+        unless this policy opted in to it.
+        """
+        out: set[str] = set()
+        for d in [*broker_domains, *(extra or [])]:
+            h = host_of(d)
+            if not h or not _host_ok(h, self.allowed):
+                continue
+            if _is_loopback(h) and not self.allow_loopback:
+                continue
+            out.add(h)
+        return out
+
+
+# --------------------------------------------------------------------------- CAPTCHA frames
+# SEC-36. The human-in-the-loop windows (D9 "Fill it in for me", D10 listing finder) ask the
+# person to tick the site's "I'm not a robot" box. That box is served by a CAPTCHA provider,
+# not the broker. These hosts are allowed ONLY in those headed windows, ONLY as embedded
+# sub-resources / sub-frames (never a main-frame navigation), and only https. Free My ID never
+# reads, solves or automates them; it just stops blocking the widget the person clicks.
+CAPTCHA_HOSTS: dict[str, tuple[str, ...]] = {
+    # host (exact or subdomain)  -> allowed path prefixes ("" = any path)
+    "www.google.com": ("/recaptcha/",),
+    "www.gstatic.com": ("/recaptcha/",),
+    "www.recaptcha.net": ("/recaptcha/",),
+    "hcaptcha.com": ("",),
+    "challenges.cloudflare.com": ("",),
+}
+
+
+def captcha_subresource_allowed(url: str, *, is_main_frame_navigation: bool) -> bool:
+    """True if `url` is a CAPTCHA widget resource a headed fill/finder window may load."""
+    if is_main_frame_navigation:
+        return False
+    host = host_of(url)
+    if not host:
+        return False
+    try:
+        parts = urlsplit(url)
+    except ValueError:
+        return False
+    if parts.scheme != "https" or parts.username or parts.password or parts.port not in (
+            None, 443):
+        return False
+    path = parts.path or "/"
+    for h, prefixes in CAPTCHA_HOSTS.items():
+        if host == h or (h == "hcaptcha.com" and host.endswith(".hcaptcha.com")):
+            return any(path.startswith(p) for p in prefixes)
+    return False
+
+
+def captcha_hosts() -> set[str]:
+    """Hostnames to add to the DNS/resolver allowlist of a headed fill/finder window."""
+    return set(CAPTCHA_HOSTS) | {"js.hcaptcha.com", "newassets.hcaptcha.com",
+                                 "api.hcaptcha.com", "imgs.hcaptcha.com"}

freemyid/notify.py

@@ -0,0 +1,58 @@
+"""Desktop notifications. Bodies must be PII-free (counts and first names at most)."""
+from __future__ import annotations
+
+import asyncio
+import logging
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+
+class LogNotifier:
+    """Fallback: write to the log only."""
+
+    def notify(self, title: str, body: str, url: str | None = None) -> None:
+        log.info("notification: %s - %s", title, body)
+
+
+def _app_icon() -> Path | None:
+    """The Free My ID icon, found next to the package (installed exe) or in the repo."""
+    here = Path(__file__).resolve().parent
+    for p in (here / "wizard" / "static" / "freemyid.png", here / "wizard" / "static" / "freemyid.ico",
+              here.parents[1] / "packaging" / "windows" / "freemyid.ico"):
+        if p.is_file():
+            return p
+    return None
+
+
+class DesktopNotifier:
+    def __init__(self) -> None:
+        from desktop_notifier import DesktopNotifier as _DN
+        icon = _app_icon()
+        if icon is not None:
+            from desktop_notifier import Icon
+            self._dn = _DN(app_name="Free My ID", app_icon=Icon(path=icon))
+        else:
+            self._dn = _DN(app_name="Free My ID")
+        self._fallback = LogNotifier()
+
+    def notify(self, title: str, body: str, url: str | None = None) -> None:
+        self._fallback.notify(title, body, url)
+        try:
+            coro = self._dn.send(title=title, message=body)
+            try:
+                asyncio.get_running_loop()
+            except RuntimeError:
+                asyncio.run(coro)
+            else:   # called from async code: fire and forget on the running loop
+                asyncio.ensure_future(coro)
+        except Exception as e:  # noqa: BLE001  headless boxes have no notification daemon
+            log.info("desktop notification unavailable: %s", type(e).__name__)
+
+
+def get_notifier():
+    try:
+        return DesktopNotifier()
+    except Exception as e:  # noqa: BLE001
+        log.info("desktop-notifier not usable (%s); logging instead", type(e).__name__)
+        return LogNotifier()

freemyid/paths.py

@@ -0,0 +1,83 @@
+"""Filesystem locations. OWNER: orchestrator."""
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+
+def home() -> Path:
+    """~/.freemyid (or %APPDATA%\\Free My ID on Windows). Override with FREEMYID_HOME."""
+    if env := os.environ.get("FREEMYID_HOME"):
+        p = Path(env)
+    elif sys.platform == "win32":
+        p = Path(os.environ.get("APPDATA", Path.home() / "AppData" / "Roaming")) / "FreeMyID"
+    else:
+        p = Path.home() / ".freemyid"
+    created = not p.exists()
+    p.mkdir(parents=True, exist_ok=True, mode=0o700)
+    marker = p / MARKER
+    # Mark only folders Free My ID created itself (or that already hold its database), so
+    # `freemyid wipe` can never mistake an arbitrary folder for Free My ID's own (SEC-20).
+    if not marker.exists() and (created or (p / "freemyid.db").exists()):
+        try:
+            marker.write_text("This folder belongs to Free My ID. `freemyid wipe` deletes it.\n")
+        except OSError:
+            pass
+    return p
+
+
+MARKER = ".freemyid-home"
+# Every top-level entry Free My ID may create in home(); `freemyid wipe` deletes only these.
+OWNED_ENTRIES = ("freemyid.db", "freemyid.db-wal", "freemyid.db-shm", "evidence", "artifacts", "logs",
+                 "config.yaml", "registry.yaml", "app-instance.json", "freemyid-task.xml", ".fill-confirmed", MARKER)
+
+
+def is_safe_wipe_target(p: Path) -> bool:
+    """True only for a folder that is clearly Free My ID's own and not a home/system root."""
+    p = p.resolve()
+    user_home = Path.home().resolve()
+    if p == user_home or p in user_home.parents or p.parent == p or len(p.parts) <= 2:
+        return False
+    return (p / MARKER).exists() or (p / "freemyid.db").exists()
+
+
+def db_path() -> Path:
+    return home() / "freemyid.db"
+
+
+def evidence_dir() -> Path:
+    d = home() / "evidence"
+    d.mkdir(exist_ok=True, mode=0o700)
+    return d
+
+
+def artifacts_dir() -> Path:
+    """Generated letters / call scripts / reports (encrypted unless user exports)."""
+    d = home() / "artifacts"
+    d.mkdir(exist_ok=True, mode=0o700)
+    return d
+
+
+def logs_dir() -> Path:
+    d = home() / "logs"
+    d.mkdir(exist_ok=True, mode=0o700)
+    return d
+
+
+def config_path() -> Path:
+    return home() / "config.yaml"
+
+
+def registry_cache_path() -> Path:
+    return home() / "registry.yaml"
+
+
+def bundled_data_dir() -> Path:
+    """Registry/schema shipped with the package (wheel or PyInstaller) or repo checkout."""
+    here = Path(__file__).resolve().parent
+    if (here / "_data").exists():
+        return here / "_data"
+    if getattr(sys, "_MEIPASS", None):
+        return Path(sys._MEIPASS) / "freemyid" / "_data"  # type: ignore[attr-defined]
+    return here.parent.parent  # repo root: brokers/, schema/

freemyid/registry.py

@@ -0,0 +1,275 @@
+"""Broker + state-law registry loading, validation and updates. OWNER: orchestrator.
+
+Precedence: FREEMYID_HOME/registry.yaml (validated update or installed family registry)
+  > registry.yaml next to the installed exe (family installer; frozen builds only)
+  > bundled brokers/registry.yaml (full registry; family/private builds only)
+  > bundled brokers/registry.sample.yaml (small public sample; always shipped).
+`registry_source()` tells the UI/CLI which one is in use: "update" | "family" | "full" | "sample".
+Updates: fetched ONLY from config.registry_url (the user's own GitHub repo), validated
+against the pydantic model + JSON Schema, optionally pinned by sha256, then atomically
+swapped in. A failed validation never replaces the working copy.
+If the preferred registry is unreadable or invalid, `load_registry()` falls back to the next
+source (so a bad update never leaves the app with no broker list) and records a warning in
+`load_warnings`. YAML aliases are refused (alias-expansion "billion laughs" protection).
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import re
+from pathlib import Path
+
+import httpx
+import jsonschema
+import yaml
+
+from . import config, paths
+from .models import Registry, StateLaw, StateLawRegistry
+from .netpolicy import NetPolicy
+
+log = logging.getLogger(__name__)
+
+
+class RegistryError(RuntimeError):
+    pass
+
+
+class _NoAliasLoader(yaml.SafeLoader):
+    """SafeLoader that refuses aliases to lists/maps. Expanding those (the json round-trip
+    below) turns a tiny file into gigabytes ("billion laughs"). Aliases to plain scalars
+    (the registry reuses a source URL this way) cannot grow and stay allowed."""
+
+    def compose_node(self, parent, index):  # type: ignore[no-untyped-def]
+        if self.check_event(yaml.AliasEvent):
+            anchor = self.peek_event().anchor
+            if not isinstance(self.anchors.get(anchor), yaml.ScalarNode):
+                raise yaml.YAMLError("YAML aliases to lists or maps are not allowed in a registry")
+        return super().compose_node(parent, index)
+
+
+SOURCE_UPDATE, SOURCE_FAMILY, SOURCE_FULL, SOURCE_SAMPLE = "update", "family", "full", "sample"
+
+
+def bundled_full_registry_path() -> Path:
+    """Full maintained registry. Not in the public repo; present in family/private builds."""
+    return paths.bundled_data_dir() / "brokers" / "registry.yaml"
+
+
+def bundled_sample_registry_path() -> Path:
+    """Small public sample registry; always shipped."""
+    return paths.bundled_data_dir() / "brokers" / "registry.sample.yaml"
+
+
+def bundled_registry_path() -> Path:
+    """The bundled registry in use: the full one if this build carries it, else the sample."""
+    full = bundled_full_registry_path()
+    return full if full.exists() else bundled_sample_registry_path()
+
+
+def _candidates() -> list[tuple[str, Path]]:
+    """Existing registries in precedence order; the sample is always last."""
+    out: list[tuple[str, Path]] = []
+    cached = paths.registry_cache_path()
+    if cached.exists():
+        out.append((SOURCE_UPDATE, cached))
+    family = find_family_registry()
+    if family is not None:
+        out.append((SOURCE_FAMILY, family))
+    full = bundled_full_registry_path()
+    if full.exists():
+        out.append((SOURCE_FULL, full))
+    out.append((SOURCE_SAMPLE, bundled_sample_registry_path()))
+    return out
+
+
+def _active_registry() -> tuple[str, Path]:
+    return _candidates()[0]
+
+
+# (preferred candidate at load time, source actually loaded): a fallback is only reported
+# while the same preferred registry is still in place.
+_loaded: tuple[tuple[str, Path], str] | None = None
+load_warnings: list[str] = []
+
+
+def registry_source() -> str:
+    """Which registry `load_registry()` used last (or would use): "update"
+    (FREEMYID_HOME/registry.yaml, from the update feed or an installed family registry),
+    "family" (registry.yaml next to the installed exe), "full" (bundled full registry) or
+    "sample" (public sample). After a fallback this names the source actually loaded."""
+    preferred = _active_registry()
+    if _loaded is not None and _loaded[0] == preferred:
+        return _loaded[1]
+    return preferred[0]
+
+
+def bundled_state_laws_path() -> Path:
+    return paths.bundled_data_dir() / "brokers" / "state_laws.yaml"
+
+
+def schema_path() -> Path:
+    return paths.bundled_data_dir() / "schema" / "registry.schema.json"
+
+
+def json_schema() -> dict:
+    return Registry.model_json_schema()
+
+
+def validate_text(text: str) -> Registry:
+    try:
+        raw = yaml.load(text, Loader=_NoAliasLoader)
+    except yaml.YAMLError as e:
+        raise RegistryError(f"registry is not valid YAML: {e}") from e
+    sp = schema_path()
+    schema = json.loads(sp.read_text(encoding="utf-8")) if sp.exists() else json_schema()
+    try:
+        jsonschema.validate(json.loads(json.dumps(raw, default=str)), schema)
+        return Registry.model_validate(raw)
+    except (jsonschema.ValidationError, ValueError) as e:
+        raise RegistryError(f"registry failed validation: {e}") from e
+
+
+def _load_file(path: Path) -> Registry:
+    try:
+        if path.stat().st_size > MAX_REGISTRY_BYTES:
+            raise RegistryError(f"registry is larger than {MAX_REGISTRY_BYTES} bytes; refusing")
+        text = path.read_text(encoding="utf-8")
+    except (OSError, UnicodeDecodeError) as e:
+        raise RegistryError(f"registry could not be read: {e}") from e
+    return validate_text(text)
+
+
+def load_registry(path: Path | None = None) -> Registry:
+    """Load an explicit registry file, or the best available one with fallback."""
+    global _loaded
+    if path is not None:
+        return _load_file(path)
+    load_warnings.clear()
+    err: RegistryError | None = None
+    cands = _candidates()
+    for source, p in cands:
+        try:
+            reg = _load_file(p)
+        except RegistryError as e:
+            msg = f"{source} registry unusable, trying the next one: {e}"
+            log.warning(msg)
+            load_warnings.append(msg)
+            err = e
+            continue
+        _loaded = (cands[0], source)
+        return reg
+    assert err is not None
+    raise err
+
+
+def load_state_laws(path: Path | None = None) -> dict[str, StateLaw]:
+    p = path or bundled_state_laws_path()
+    if not p.exists():
+        return {}
+    reg = StateLawRegistry.model_validate(yaml.safe_load(p.read_text(encoding="utf-8")))
+    return {s.state: s for s in reg.states}
+
+
+MAX_REGISTRY_BYTES = 5 * 1024 * 1024
+MAX_PIN_BYTES = 4096
+_SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
+
+
+def _check_url(url: str, what: str, policy: NetPolicy) -> None:
+    if not url.lower().startswith("https://"):
+        raise RegistryError(f"{what} must be https")
+    if not policy.is_allowed(url):
+        raise RegistryError(f"{what} host is not on the network allowlist")
+
+
+def _fetch(c: httpx.Client, url: str, cap: int, what: str) -> bytes:
+    """GET url with no redirects, streaming, refusing bodies over `cap` bytes."""
+    try:
+        with c.stream("GET", url) as resp:
+            if resp.status_code != 200:
+                raise RegistryError(f"{what} fetch failed: HTTP {resp.status_code}")
+            declared = resp.headers.get("content-length")
+            if declared and declared.isdigit() and int(declared) > cap:
+                raise RegistryError(f"{what} is larger than {cap} bytes; refusing")
+            buf = bytearray()
+            for chunk in resp.iter_bytes():
+                buf.extend(chunk)
+                if len(buf) > cap:
+                    raise RegistryError(f"{what} is larger than {cap} bytes; refusing")
+            return bytes(buf)
+    except httpx.HTTPError as e:
+        raise RegistryError(f"{what} fetch failed: {type(e).__name__}") from e
+
+
+def update_registry(timeout: float = 30.0, *,
+                    transport: httpx.BaseTransport | None = None) -> tuple[str, int]:
+    """Pull, validate, pin, swap. Returns (version, broker_count).
+
+    Both URLs must be https and allowed by NetPolicy; redirects are refused and bodies
+    are capped at MAX_REGISTRY_BYTES. `transport` is for tests (httpx.MockTransport).
+    """
+    cfg = config.load()
+    if not cfg.registry_url:
+        raise RegistryError("registry_url is not configured (freemyid config set registry_url ...)")
+    policy = NetPolicy(None)   # registry hosts + user extras only; no broker hosts needed
+    _check_url(cfg.registry_url, "registry_url", policy)
+    if cfg.registry_sha256_url:
+        _check_url(cfg.registry_sha256_url, "registry_sha256_url", policy)
+    with httpx.Client(timeout=timeout, follow_redirects=False, transport=transport) as c:
+        body = _fetch(c, cfg.registry_url, MAX_REGISTRY_BYTES, "registry")
+        if cfg.registry_sha256_url:
+            pin = _fetch(c, cfg.registry_sha256_url, MAX_PIN_BYTES, "registry sha256 pin")
+            fields = pin.decode("ascii", errors="replace").split()
+            want = fields[0].strip().lower() if fields else ""
+            if not _SHA256_RE.match(want):
+                raise RegistryError("registry sha256 pin is empty or malformed; refusing update")
+            if want != hashlib.sha256(body).hexdigest():
+                raise RegistryError("registry sha256 mismatch; refusing update")
+    reg = install_registry_bytes(body)
+    return reg.version, len(reg.brokers)
+
+
+def install_registry_bytes(body: bytes) -> Registry:
+    """Validate a registry and atomically install it as FREEMYID_HOME/registry.yaml.
+
+    Shared by `update_registry` and family-bundle import, so both get the same checks:
+    size cap, UTF-8, YAML, JSON Schema + pydantic. On any failure nothing is replaced.
+    """
+    if len(body) > MAX_REGISTRY_BYTES:
+        raise RegistryError(f"registry is larger than {MAX_REGISTRY_BYTES} bytes; refusing")
+    try:
+        text = body.decode("utf-8")
+    except UnicodeDecodeError as e:
+        raise RegistryError("registry is not valid UTF-8") from e
+    reg = validate_text(text)
+    dest = paths.registry_cache_path()
+    tmp = dest.with_suffix(".tmp")
+    tmp.write_bytes(body)   # exact bytes: text mode would turn \r\n into \r\r\n on Windows
+    tmp.replace(dest)
+    return reg
+
+
+FAMILY_REGISTRY_NAME = "registry.yaml"
+
+
+def find_family_registry() -> Path | None:
+    """A registry.yaml shipped next to the installed app (family installer), if any.
+
+    Only checked for frozen (installed) builds, mirroring wizard.flows.find_family_bundle.
+    """
+    import sys
+    if not getattr(sys, "frozen", False):
+        return None
+    p = Path(sys.executable).parent / FAMILY_REGISTRY_NAME
+    return p if p.is_file() else None
+
+
+def install_family_registry(src: Path | bytes) -> tuple[str, int]:
+    """Install a family-bundle registry (file path or raw bytes) through the update checks."""
+    if isinstance(src, Path):
+        if src.stat().st_size > MAX_REGISTRY_BYTES:
+            raise RegistryError(f"registry is larger than {MAX_REGISTRY_BYTES} bytes; refusing")
+        src = src.read_bytes()
+    reg = install_registry_bytes(src)
+    return reg.version, len(reg.brokers)