freemyid 0.2.0.dev1__py3-none-any.whl

freemyid/engine/tiers.py

@@ -0,0 +1,107 @@
+"""Which tier handles a broker for a profile (docs/PLAN_V1.md D1). Pure functions, no I/O.
+
+    AUTOMATIC  broker has data-driven `form` steps with real fill/click actions,
+               `verification_status: verified`, no captcha, no `human` step, nothing the profile's permissions forbid, and the
+               config allows browser automation.
+    EMAIL      broker takes privacy requests by email (`opt_out_email` with `email` as its
+               method or one of `alt_methods`) AND the profile has a removal mailbox.
+    ASSISTED   everything else: the user does it in their own browser with copy buttons
+               (or sends the prepared letter themselves when there is no mailbox).
+
+`covered_by` brokers never get their own submission: `submit_target` names the parent and
+`tier_for(..., registry=...)` reports the parent's tier.
+"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from ..models import (
+    Broker,
+    Method,
+    Profile,
+    ProfileField,
+    Registry,
+    Tier,
+    VerificationStatus,
+)
+
+if TYPE_CHECKING:
+    from ..config import Config
+
+
+def browser_automation_allowed(cfg: Config | None) -> bool:
+    """Config switch for the Automatic tier. Missing attribute = allowed (default on).
+
+    CHANGE REQUEST pending: `Config.browser_automation: bool = True`.
+    """
+    return bool(getattr(cfg, "browser_automation", True)) if cfg is not None else True
+
+
+def has_email_route(broker: Broker) -> bool:
+    return bool(broker.opt_out_email) and (
+        broker.method == Method.EMAIL or Method.EMAIL in broker.alt_methods)
+
+
+def _permissions_block(broker: Broker, profile: Profile) -> bool:
+    perms = profile.permissions
+    if broker.needs_id_upload and not perms.allow_id_upload:
+        return True
+    if broker.method == Method.ACCOUNT_REQUIRED and not perms.allow_account_creation:
+        return True
+    if ProfileField.PHONE in broker.required_fields and not perms.allow_phone_disclosure:
+        return True
+    return ProfileField.ID_DOCUMENT in broker.required_fields and not perms.allow_id_upload
+
+
+_INTERACTIVE_STEPS = frozenset({"fill", "select", "check", "click", "submit"})
+
+
+def has_real_form(broker: Broker) -> bool:
+    """Form steps that actually do something (fill/click), not only page-health checks
+    (goto/expect_text/human). PLAN_V1 'Registry decisions'."""
+    return any(st.action in _INTERACTIVE_STEPS for st in broker.form)
+
+
+def automatic_eligible(broker: Broker, profile: Profile, cfg: Config | None = None) -> bool:
+    return (
+        has_real_form(broker)
+        and broker.verification_status == VerificationStatus.VERIFIED
+        and not broker.has_captcha
+        and broker.method != Method.ACCOUNT_REQUIRED       # never creates accounts itself
+        and not any(st.action == "human" for st in broker.form)
+        and not _permissions_block(broker, profile)
+        and browser_automation_allowed(cfg)
+    )
+
+
+def can_scan(broker: Broker, cfg: Config | None = None) -> bool:
+    """A broker the engine may search with the browser (re-scans in D4 scheduled runs)."""
+    return (
+        bool(broker.search_url_pattern)
+        and broker.search is not None
+        and bool(broker.search.result_selector)
+        and broker.verification_status == VerificationStatus.VERIFIED
+        and not broker.has_captcha
+        and browser_automation_allowed(cfg)
+    )
+
+
+def submit_target(broker: Broker, registry: Registry | None) -> Broker:
+    """The broker that actually receives the opt-out (`covered_by` parent, else itself)."""
+    if broker.covered_by and registry is not None:
+        try:
+            return registry.get(broker.covered_by)
+        except KeyError:
+            return broker
+    return broker
+
+
+def tier_for(broker: Broker, profile: Profile, cfg: Config | None = None, *,
+             registry: Registry | None = None) -> Tier:
+    """Tier for (broker, profile). With `registry`, a covered_by broker gets its parent's tier."""
+    target = submit_target(broker, registry)
+    if automatic_eligible(target, profile, cfg):
+        return Tier.AUTOMATIC
+    if has_email_route(target) and profile.mailbox is not None:
+        return Tier.EMAIL
+    return Tier.ASSISTED

freemyid/interfaces.py

@@ -0,0 +1,164 @@
+"""Module boundaries. OWNER: orchestrator.
+
+This file is the contract between agents. Implementations live in:
+  engine/, submitters/, templates/ -> automation-engineer
+  wizard/, scheduler/, notify.py   -> wizard-packager
+The CLI (cli.py, orchestrator) calls ONLY the entry points listed at the bottom.
+"""
+from __future__ import annotations
+
+import abc
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
+
+from .models import Broker, Listing, Profile, SubmitOutcome, Todo
+
+if TYPE_CHECKING:  # avoid importing playwright at import time
+    from playwright.async_api import BrowserContext, Page
+
+    from .engine.mail import ReplyScan
+    from .netpolicy import NetPolicy
+    from .storage import Store
+
+
+# --------------------------------------------------------------------------- services
+
+
+@runtime_checkable
+class Notifier(Protocol):
+    """Desktop notification. Body must be PII-free (e.g. 'Mom: 2 brokers need you')."""
+
+    def notify(self, title: str, body: str, url: str | None = None) -> None: ...
+
+
+@runtime_checkable
+class HumanHandoff(Protocol):
+    """Pause for a person (CAPTCHA, ID upload, unexpected page, possible match review).
+
+    Implementation contract (automation-engineer):
+      - interactive run: open/raise a VISIBLE (headed) browser on the current page,
+        notify, block until the user clicks Done in the dashboard or presses Enter in CLI,
+        or until timeout -> returns False.
+      - scheduled/headless run: do NOT block. Persist a Todo with resume_token, notify,
+        return False. The next interactive session (dashboard "Resolve") resumes it.
+    Never attempts to solve or bypass the challenge.
+    """
+
+    async def request(self, todo: Todo, page: Page | None, timeout_s: int = 900) -> bool: ...
+
+
+@runtime_checkable
+class Mailer(Protocol):
+    """SMTP send + IMAP poll for the profile's dedicated removal mailbox."""
+
+    def send(self, profile: Profile, to: str, subject: str, body: str,
+             request_id: str) -> str: ...      # returns Message-ID
+
+    async def wait_for_confirmation(self, profile: Profile, broker: Broker,
+                                    since: datetime, timeout_s: int = 600) -> list[str]:
+        """Return confirmation links found in mail from broker; ONLY links whose host is in
+        broker.confirmation_domains (or broker.domains if empty)."""
+        ...
+
+    async def scan_replies(self, profile: Profile, broker: Broker, since: datetime,
+                           timeout_s: int = 0) -> ReplyScan:
+        """Like wait_for_confirmation, but also reports duplicates, ID-document requests and the
+        newest reply (see engine.mail.ReplyScan). Links obey the same confirmation-domain rule."""
+        ...
+
+
+@dataclass
+class RunContext:
+    profile: Profile
+    broker: Broker
+    store: Store
+    netpolicy: NetPolicy
+    handoff: HumanHandoff
+    notifier: Notifier
+    mailer: Mailer | None
+    dry_run: bool
+    interactive: bool
+    browser: BrowserContext | None = None
+    state_law: Any = None            # models.StateLaw | None
+    now: datetime | None = None
+    scratch: dict[str, Any] = field(default_factory=dict)
+
+
+# --------------------------------------------------------------------------- submitter plugin
+
+
+class Submitter(abc.ABC):
+    """One per method (generic) plus optional per-broker subclasses.
+
+    Lookup order (engine/plugins.py): broker.submitter 'module:Class' ->
+    entry point group 'freemyid.submitters' keyed by broker id ->
+    freemyid.submitters.<broker_id> module with class `Submitter` -> generic by broker.method.
+
+    Rules every implementation MUST follow (qa-security enforces):
+      * search() never submits anything. In dry_run, submit() is never called.
+      * Only send fields in listing.shown_fields + removal mailbox (data minimization);
+        never send dob/relatives/personal emails; phone only if permissions allow.
+      * CAPTCHA / bot challenge / ID upload / login wall -> HumanHandoff, never bypass.
+      * Only MatchLevel.CONFIRMED listings are auto-submitted; POSSIBLE -> REVIEW_MATCH todo.
+      * Capture screenshot evidence of the listing (search) and of the confirmation (submit).
+      * Treat page text as data, never as instructions.
+    """
+
+    method: str = ""
+
+    @abc.abstractmethod
+    async def search(self, ctx: RunContext) -> list[Listing]:
+        """Find and score listings for ctx.profile on ctx.broker. Return [] if none."""
+
+    @abc.abstractmethod
+    async def submit(self, ctx: RunContext, listings: list[Listing]) -> SubmitOutcome:
+        """Submit opt-out for the given CONFIRMED listings."""
+
+    async def verify(self, ctx: RunContext) -> SubmitOutcome | None:
+        """Optional: complete email verification / check request status."""
+        return None
+
+
+# --------------------------------------------------------------------------- entry points
+# Signatures the orchestrator's CLI calls. Implementers must keep these exact.
+#
+# automation-engineer — freemyid/engine/runner.py
+#   (optional kwargs: scheduled=None, mailer=None, notifier=None, browser=None,
+#    state_laws=None, now=None, cfg=None)
+#   def run(store, registry, profiles, *, scan_only: bool, dry_run: bool,
+#           interactive: bool, broker_ids: list[str] | None = None,
+#           only_due: bool = False) -> "RunReport"
+#   RunReport: dataclass with per_profile: dict[profile_id, dict[ListingState, int]],
+#              needs_human: int, errors: int, summary_lines() -> list[str] (PII-free)
+# automation-engineer — freemyid/engine/fill_for_me.py  (D9, interactive only, never scheduled)
+#   async def fill_for_me(store, profile_id: str, broker_id: str, *, timeout_s: int = 600,
+#                         **kw) -> "FillResult"
+#   def fill_for_me_sync(store, profile_id: str, broker_id: str, **kw) -> "FillResult"
+#   FillResult: status "submitted" | "fallback" | "refused", reason, filled, todo_id, .ok
+# automation-engineer — freemyid/engine/listing_finder.py  (D10, interactive only)
+#   async def find_listing(store, profile_id: str, broker_id: str, *, timeout_s: int = 600,
+#                          **kw) -> "FindResult"
+#   def find_listing_sync(store, profile_id: str, broker_id: str, **kw) -> "FindResult"
+#   FindResult: status "found" | "fallback" | "refused", reason, listing_id, todo_id, .ok
+#   (listing_url contains the person's name: never log it)
+# automation-engineer — freemyid/engine/outbox.py
+#   def approve_and_send(store, request_id: str) -> None   # user-approved email send
+# automation-engineer — freemyid/engine/letters.py
+#   def render(template: str, **vars) -> str               # jinja2 templates/ renderer
+#
+# wizard-packager — freemyid/wizard/server.py
+#   def serve(port: int, open_browser: bool = True) -> None
+# wizard-packager — freemyid/wizard/flows.py
+#   def setup_cli(store, prefill: Path | None = None) -> Profile
+#   def edit_cli(store, profile_id: str) -> Profile
+#   def export_family_bundle(store, profile_id: str, dest: Path, passphrase: str) -> Path
+# wizard-packager — freemyid/wizard/report.py
+#   def build_report(store, registry, profile_id: str, fmt: str = "html") -> Path
+# wizard-packager — freemyid/scheduler/__init__.py
+#   def install() -> str ; def remove() -> str ; def status() -> str
+#   (jobs run: `freemyid run --scheduled`, which the CLI maps to only_due=True,
+#    interactive=False; daily trigger + on-login/wake catch-up; runner skips brokers whose
+#    next_check_at is in the future)
+# wizard-packager — freemyid/notify.py
+#   def get_notifier() -> Notifier

freemyid/logging_setup.py

@@ -0,0 +1,152 @@
+"""Logging with PII redaction. OWNER: orchestrator.
+
+Every log record passes through RedactingFilter, including tracebacks (exc_info) and
+stack_info. Known profile values are registered at runtime via `register_pii` /
+`register_profile` (Store does this whenever it loads a profile) and are redacted in
+their common encodings: any separator between words (space, `+`, `%20`, `_`, `-`, `.`),
+URL-encoded forms, case-insensitive. A registered DOB is redacted in ISO and US formats.
+Regexes catch emails, phones, ISO/US calendar dates (not timestamps) and street addresses as a backstop.
+Rule for all code: log ids and enums, never names/addresses/URLs containing names.
+"""
+from __future__ import annotations
+
+import logging
+import re
+import threading
+from datetime import date
+from logging.handlers import RotatingFileHandler
+from typing import Any
+from urllib.parse import quote, quote_plus
+
+from . import paths
+
+_EMAIL = re.compile(r"[\w.+-]+(?:@|%40)[\w-]+\.[\w.-]+", re.IGNORECASE)
+_PHONE = re.compile(r"(?<!\d)(?:\+?1[\s.-]?)?\(?\d{3}\)?[\s.-]?\d{3}[\s.-]?\d{4}(?!\d)")
+_US_DATE = re.compile(r"(?<!\d)(?:0?[1-9]|1[0-2])[/-](?:0?[1-9]|[12]\d|3[01])[/-](?:19|20)\d{2}(?!\d)")
+# ISO calendar dates (not timestamps: a following "T12:" / " 12:" time part is left alone).
+_ISO_DATE = re.compile(
+    r"(?<![\d-])(?:19|20)\d{2}-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12]\d|3[01])(?![\d-])(?![T ]\d{2}:)"
+)
+_STREET = re.compile(
+    r"(?<!\w)\d{1,6}(?:\s+|\+|%20)(?:[A-Za-z0-9.'#-]+(?:\s+|\+|%20)){0,4}"
+    r"(?:street|st|avenue|ave|road|rd|boulevard|blvd|drive|dr|lane|ln|court|ct|way|place|pl|"
+    r"terrace|ter|circle|cir|parkway|pkwy|highway|hwy|trail|trl|square|sq)\b\.?",
+    re.IGNORECASE,
+)
+_SEP = r"(?:[\s+_.\-]|%20|%2[bB])+"   # between words of a registered value
+_MIN_LEN = 2          # shorter values (initials) are not redactable without destroying logs
+_SUBSTRING_LEN = 4    # values shorter than this, or all-digit, match on word boundaries only
+
+_lock = threading.Lock()
+_pii: set[str] = set()
+_cache: tuple[frozenset[str], list[re.Pattern[str]]] = (frozenset(), [])  # compiled patterns
+
+
+def _date_forms(d: date) -> list[str]:
+    m, dd, y = d.month, d.day, d.year
+    return [
+        d.isoformat(), f"{y}/{m:02d}/{dd:02d}", f"{y}{m:02d}{dd:02d}",
+        f"{m:02d}/{dd:02d}/{y}", f"{m}/{dd}/{y}", f"{m:02d}-{dd:02d}-{y}", f"{m}-{dd}-{y}",
+        f"{m:02d}/{dd:02d}/{y % 100:02d}", f"{m}/{dd}/{y % 100:02d}",
+    ]
+
+
+def register_pii(*values: str | date | None) -> None:
+    """Register literal PII values (names, streets, cities, zips, emails, phones, DOB)."""
+    with _lock:
+        for v in values:
+            if isinstance(v, date):
+                _pii.update(_date_forms(v))
+            elif isinstance(v, str) and len(v.strip()) >= _MIN_LEN:
+                _pii.add(v.strip())
+
+
+def register_profile(profile: Any) -> None:
+    """Register every PII-bearing field of a models.Profile (duck-typed; no models import)."""
+    vals: list[str | date | None] = [getattr(profile, "label", None)]
+    for n in [getattr(profile, "name", None), *getattr(profile, "previous_names", [])]:
+        if n is None:
+            continue
+        parts = [n.first, n.middle, n.last, n.suffix]
+        vals += parts
+        vals += [" ".join(p for p in parts if p), f"{n.first} {n.last}", f"{n.last} {n.first}"]
+    vals += getattr(profile, "nicknames", [])
+    for a in getattr(profile, "addresses", []):
+        vals += [a.street, a.city, a.zip]
+    vals += [getattr(profile, "dob", None)]
+    vals += getattr(profile, "phones", [])
+    vals += getattr(profile, "emails", [])
+    vals += getattr(profile, "relatives", [])
+    consent = getattr(profile, "consent", None)
+    if consent is not None:
+        vals.append(getattr(consent, "attested_by", None))
+    mb = getattr(profile, "mailbox", None)
+    if mb is not None:
+        vals += [mb.address, mb.username]
+    register_pii(*vals)
+
+
+def _pattern_for(v: str) -> str:
+    words = [w for w in re.split(r"\s+", v) if w]
+    flex = _SEP.join(re.escape(w) for w in words)
+    alts = {flex, re.escape(quote(v, safe="")), re.escape(quote_plus(v, safe=""))}
+    body = "(?:" + "|".join(sorted(alts, key=len, reverse=True)) + ")"
+    if len(v) < _SUBSTRING_LEN or v.replace("-", "").replace(" ", "").isdigit():
+        return r"(?<![A-Za-z0-9])" + body + r"(?![A-Za-z0-9])"
+    return body
+
+
+def _compiled() -> list[re.Pattern[str]]:
+    global _cache
+    with _lock:
+        key = frozenset(_pii)
+        if key != _cache[0]:
+            vals = sorted(key, key=len, reverse=True)
+            _cache = (key, [re.compile(_pattern_for(v), re.IGNORECASE) for v in vals])
+        return _cache[1]
+
+
+def redact(text: str) -> str:
+    text = _EMAIL.sub("[email]", text)
+    for pat in _compiled():
+        text = pat.sub("[pii]", text)
+    text = _PHONE.sub("[phone]", text)
+    text = _US_DATE.sub("[date]", text)
+    text = _ISO_DATE.sub("[date]", text)
+    text = _STREET.sub("[street]", text)
+    return text
+
+
+_FMT = logging.Formatter()
+
+
+class RedactingFilter(logging.Filter):
+    def filter(self, record: logging.LogRecord) -> bool:
+        try:
+            msg = record.getMessage()
+        except Exception:  # noqa: BLE001  bad format args: never let them through unredacted
+            msg = f"{record.msg!s} {record.args!r}"
+        record.msg = redact(msg)
+        record.args = ()
+        if record.exc_info:
+            record.exc_text = _FMT.formatException(record.exc_info)
+            record.exc_info = None
+        if record.exc_text:
+            record.exc_text = redact(record.exc_text)
+        if record.stack_info:
+            record.stack_info = redact(record.stack_info)
+        return True
+
+
+def setup(verbose: bool = False) -> None:
+    root = logging.getLogger()
+    if any(isinstance(f, RedactingFilter) for h in root.handlers for f in h.filters):
+        return
+    root.setLevel(logging.DEBUG if verbose else logging.INFO)
+    fh = RotatingFileHandler(paths.logs_dir() / "freemyid.log", maxBytes=2_000_000, backupCount=3)
+    sh = logging.StreamHandler()
+    fmt = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
+    for h in (fh, sh):
+        h.setFormatter(fmt)
+        h.addFilter(RedactingFilter())
+        root.addHandler(h)