atsfill 0.0.5.dev143__tar.gz → 0.0.5.dev145__tar.gz

{atsfill-0.0.5.dev143 → atsfill-0.0.5.dev145}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: atsfill
-Version: 0.0.5.dev143
+Version: 0.0.5.dev145
 Summary: Read, dry-run and submit job application forms on hiring systems: Ashby, Greenhouse, Teamtailor, Welcome to the Jungle.
 Keywords: job-applications,ats,ashby,greenhouse,teamtailor,ai-agents,cli
 Author: shbernal
@@ -153,7 +153,7 @@ The core owns every rule about when a send is allowed; an adapter only knows its
 Every result has a `status` (`ready`, `refused`, `submitted`, `blocked`, `unknown`, `unsupported`) and, when it is not clean, a `reason` code.
 The exit code is 0 for `ready` and `submitted`, 1 otherwise, 2 for a usage error.
 
-Each `read`, `fill` and `send` sends one anonymous event (the system, the command, the status, a hash of the posting, never answers or URLs): [what is sent and how to turn it off](docs/telemetry.md).
+Each `read`, `fill` and `send` sends one anonymous event (the system, the command, the status, a hash of the posting, never answers or URLs); `atsfill report` sends a failed run's URL and page, scrubbed of your answers, when you ask: [what is sent and how to turn it off](docs/telemetry.md).
 
 ## More
 

{atsfill-0.0.5.dev143 → atsfill-0.0.5.dev145}/README.md

@@ -124,7 +124,7 @@ The core owns every rule about when a send is allowed; an adapter only knows its
 Every result has a `status` (`ready`, `refused`, `submitted`, `blocked`, `unknown`, `unsupported`) and, when it is not clean, a `reason` code.
 The exit code is 0 for `ready` and `submitted`, 1 otherwise, 2 for a usage error.
 
-Each `read`, `fill` and `send` sends one anonymous event (the system, the command, the status, a hash of the posting, never answers or URLs): [what is sent and how to turn it off](docs/telemetry.md).
+Each `read`, `fill` and `send` sends one anonymous event (the system, the command, the status, a hash of the posting, never answers or URLs); `atsfill report` sends a failed run's URL and page, scrubbed of your answers, when you ask: [what is sent and how to turn it off](docs/telemetry.md).
 
 ## More
 

{atsfill-0.0.5.dev143 → atsfill-0.0.5.dev145}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "atsfill"
-version = "0.0.5.dev143"
+version = "0.0.5.dev145"
 description = "Read, dry-run and submit job application forms on hiring systems: Ashby, Greenhouse, Teamtailor, Welcome to the Jungle."
 readme = "README.md"
 license = "MIT"

{atsfill-0.0.5.dev143 → atsfill-0.0.5.dev145}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "atsfill"
-version = "0.0.5.dev143"
+version = "0.0.5.dev145"
 description = "Read, dry-run and submit job application forms on hiring systems: Ashby, Greenhouse, Teamtailor, Welcome to the Jungle."
 readme = "README.md"
 license = "MIT"

{atsfill-0.0.5.dev143 → atsfill-0.0.5.dev145}/src/atsfill/cli.py

@@ -8,7 +8,7 @@ from atsfill import core
 from atsfill.adapters import adapters
 from atsfill.core import browser, paths
 from atsfill.core.adapter import NeedsAccount, SignUpDetails
-from atsfill.core.results import Result, SessionReport, Status, StatusReport
+from atsfill.core.results import ReportResult, Result, SessionReport, Status, StatusReport
 
 OK = {Status.READY, Status.SUBMITTED}
 
@@ -83,6 +83,9 @@ def parser() -> argparse.ArgumentParser:
         "--resend", action="store_true", help="send even though an earlier send may have gone"
     )
     sub.add_parser("status", help="version, paths, browser, and each adapter's state")
+    sub.add_parser(
+        "report", help="send the last failed run's URL and page, scrubbed, to the maintainer"
+    )
     sub.add_parser("install-browser", help="download the Camoufox build atsfill is pinned to")
     accounts = [a.name for a in adapters() if isinstance(a, NeedsAccount)]
     for name, text in (
@@ -169,6 +172,10 @@ def run(args: argparse.Namespace) -> int:
             return 1 if signed_out or report.browser != report.browser_pinned else 0
         case "install-browser":
             return browser.install()
+        case "report":
+            sent = core.report(data_dir=args.data_dir)
+            emit(sent, False)
+            return 0 if sent.status == "sent" else 1
         case "login" | "logout":
             where = {"user": args.user, "url": args.url, "data_dir": args.data_dir}
             try:
@@ -197,7 +204,9 @@ def run(args: argparse.Namespace) -> int:
 
 
 def emit(
-    value: Result | StatusReport | SessionReport, verbose: bool, command: str | None = None
+    value: Result | StatusReport | SessionReport | ReportResult,
+    verbose: bool,
+    command: str | None = None,
 ) -> None:
     """Prints the whole value with `verbose`; else what its consumer acts on, on one line."""
     data = dataclasses.asdict(value)

{atsfill-0.0.5.dev143 → atsfill-0.0.5.dev145}/src/atsfill/core/__init__.py

@@ -2,7 +2,7 @@
 
 from typing import Any
 
-__all__ = ["fill", "login", "logout", "read", "send", "status"]
+__all__ = ["fill", "login", "logout", "read", "report", "send", "status"]
 
 
 def __getattr__(name: str) -> Any:

{atsfill-0.0.5.dev143 → atsfill-0.0.5.dev145}/src/atsfill/core/accounts.py

@@ -8,9 +8,11 @@ from pathlib import Path
 from atsfill.core import browser
 from atsfill.core.adapter import (
     CreatesAccount,
+    CreatesAccountByOtp,
     FinishesSignUp,
     NeedsAccount,
     SignsIn,
+    SignsInByOtp,
     SignUpDetails,
     Unsupported,
 )
@@ -47,6 +49,12 @@ def email(user_dir: Path, adapter: str, scope: str | None) -> str | None:
     return emails.get(account_key(adapter, scope))
 
 
+def emails(user_dir: Path) -> list[str]:
+    """Every account's email, for a snapshot to scrub."""
+    path = user_dir / EMAILS
+    return list(json.loads(path.read_text()).values()) if path.exists() else []
+
+
 def remember(user_dir: Path, adapter: str, scope: str | None, address: str) -> None:
     path = user_dir / EMAILS
     emails = json.loads(path.read_text()) if path.exists() else {}
@@ -55,21 +63,30 @@ def remember(user_dir: Path, adapter: str, scope: str | None, address: str) -> N
 
 def sign_in(
     page: Page,
-    adapter: SignsIn,
+    adapter: SignsIn | SignsInByOtp,
     scope: str | None,
     address: str,
-    password: str,
+    password: str | None,
     session: Path,
     details: SignUpDetails | None = None,
     create: bool = False,
 ) -> None:
     """Signs in from the system's login page and stores the session: with `create`, creates
     the account from the user's sign-up `details`; without, `details` finish a sign-up the
-    account was left in, where the adapter knows that step.
+    account was left in, where the adapter knows that step. An account without a password
+    (`password` None) signs in by OTP, and creates the same way.
     Raises what the adapter raises, without the password in it."""
     page.goto(adapter.login_url(scope))
     try:
-        if create:
+        if isinstance(adapter, SignsInByOtp):
+            if create:
+                assert isinstance(adapter, CreatesAccountByOtp) and details is not None
+                adapter.sign_up_by_otp(page, address, details)
+            else:
+                adapter.sign_in_by_otp(page, address)
+        elif password is None:
+            raise AssertionError(f"{adapter.name} signs in with a password")
+        elif create:
             assert isinstance(adapter, CreatesAccount) and details is not None
             adapter.sign_up(page, address, password, details)
         elif details is not None and isinstance(adapter, FinishesSignUp):
@@ -77,7 +94,7 @@ def sign_in(
         else:
             adapter.sign_in(page, address, password)
     except Exception as e:
-        if password in str(e):
+        if password and password in str(e):
             raise redacted(e, password) from None
         raise
     if not adapter.signed_in(page.request, scope):

{atsfill-0.0.5.dev143 → atsfill-0.0.5.dev145}/src/atsfill/core/adapter.py

@@ -42,7 +42,7 @@ def out_of_scope(field_id: str, what: str) -> str:
 
 class Refused(Exception):
     """The system turned down a sign-in or a sign-up for a reason it states:
-    `email_in_use`, `email_domain_rejected`, `password_rule`, `wrong_password`.
+    `email_in_use`, `email_domain_rejected`, `password_rule`, `wrong_password`, `no_account`.
 
     The message is the result's reason, `code` or `code: detail`.
     """
@@ -125,6 +125,18 @@ class SignsIn(NeedsAccount, Protocol):
         ...
 
 
+@runtime_checkable
+class SignsInByOtp(NeedsAccount, Protocol):
+    """An adapter atsfill can sign in through with an email and the OTP its system emails each
+    time, an account without a password (ADR 0015)."""
+
+    def sign_in_by_otp(self, page: Page, email: str) -> None:
+        """Signs in on the page `login_url` loaded, typing the OTP with `otp.enter`. Raises as
+        `sign_in` does, and `Refused("no_account")` before the click that would turn an unknown
+        email's sign-in into a sign-up."""
+        ...
+
+
 ADDRESS = ("street", "city", "state", "zip", "country")
 PHONE = re.compile(r"\+[1-9][0-9]{6,14}")  # E.164
 COUNTRY = re.compile(r"[A-Z]{2}")  # ISO 3166 alpha-2
@@ -183,6 +195,20 @@ class CreatesAccount(SignsIn, Protocol):
         ...
 
 
+@runtime_checkable
+class CreatesAccountByOtp(SignsInByOtp, Protocol):
+    """An adapter atsfill can create an account without a password through. Account creation is
+    a side effect it declares (ADR 0008)."""
+
+    asks: set[str]  # the `SignUpDetails` fields its sign-up requires
+
+    def sign_up_by_otp(self, page: Page, email: str, details: SignUpDetails) -> None:
+        """Creates the account from the page `login_url` loaded, or signs in where the email
+        has one, signed in once it returns. Raises as `sign_in_by_otp` does, `no_account` aside;
+        the details are given as `sign_up` gives them."""
+        ...
+
+
 @runtime_checkable
 class FinishesSignUp(CreatesAccount, Protocol):
     """An adapter whose system can keep an account half made: signed up, but a step the

{atsfill-0.0.5.dev143 → atsfill-0.0.5.dev145}/src/atsfill/core/commands.py

@@ -1,6 +1,7 @@
 import shlex
 from collections.abc import Callable, Generator
-from contextlib import contextmanager
+from contextlib import contextmanager, suppress
+from dataclasses import dataclass
 from importlib.metadata import version
 from pathlib import Path
 
@@ -15,18 +16,21 @@ from atsfill.core import (
     passwords,
     paths,
     registry,
+    snapshot,
     telemetry,
 )
 from atsfill.core.adapter import (
     Adapter,
     Blocked,
     CreatesAccount,
+    CreatesAccountByOtp,
     ListsShownFields,
     NeedsAccount,
     NotTaken,
     ReadsByRequest,
     Refused,
     SignsIn,
+    SignsInByOtp,
     SignUpDetails,
     Unsupported,
 )
@@ -35,6 +39,7 @@ from atsfill.core.results import (
     AdapterStatus,
     Answer,
     Field,
+    ReportResult,
     Result,
     SessionReport,
     Status,
@@ -44,18 +49,26 @@ from atsfill.core.results import (
 type Sessions = dict[str, Path]
 
 
+@dataclass
+class Seen:
+    """What a command saw before it ended, for a result it did not get to make."""
+
+    adapter: str | None = None  # the adapter detected
+    page: tuple[str, str] | None = None  # a failed run's page, for its report (ADR 0016)
+
+
 def _detect(candidates: list[Adapter], url: str, page: Page | None = None) -> Adapter | None:
     return next((a for a in candidates if a.detect(url, page)), None)
 
 
-def login_command(adapter: str, user: str, url: str | None) -> str:
+def login_command(adapter: str, user: str, url: str | None, *more: str) -> str:
     """The `login` command line for one account: its user, and a posting where it is scoped."""
     command = ["atsfill", "login", adapter]
     if user != paths.DEFAULT_USER:
         command += ["--user", user]
     if url:
         command += ["--url", url]
-    return shlex.join(command)
+    return shlex.join([*command, *more])
 
 
 def _login_required(url: str, adapter: NeedsAccount, user: str) -> Result:
@@ -63,45 +76,57 @@ def _login_required(url: str, adapter: NeedsAccount, user: str) -> Result:
     return Result(Status.BLOCKED, url, adapter.name, reason=f"login_required: run `{command}`")
 
 
-type Credentials = tuple[str, str]  # email, password
+type Credentials = tuple[str, str | None]  # email, password: None for an account without one
 # Refusals the page states before any account exists: the generated password goes.
 NO_ACCOUNT = {"email_in_use", "email_domain_rejected", "password_rule", "sign_up_details"}
 
 
-def _email_link(reason: str | None, email: str, command: str) -> str | None:
-    """Names the address a verification link went to and the command to run once clicked."""
-    if reason and reason.split(":")[0] == "email_link":
+def _next_step(
+    reason: str | None, email: str, adapter: str, user: str, url: str | None
+) -> str | None:
+    """Names the command to run next: once the verification link sent to the address is
+    clicked, or `--create` for an email the system has no account for."""
+    code = (reason or "").split(":")[0]
+    if code == "email_link":
+        command = login_command(adapter, user, url)
         return f"email_link: click the link sent to {email}, then run `{command}`"
+    if code == "no_account":
+        command = login_command(adapter, user, url, "--email", email, "--create")
+        return f"no_account: {email} has no {adapter} account; run `{command}`"
     return reason
 
 
 def _credentials(
     adapter: NeedsAccount, user: str, mine: Path, scope: str | None, given: str | None = None
 ) -> Credentials | None:
-    """The account's email and a password from its password source, where atsfill can sign in
+    """The account's email and a password from its password source, or no password for an
+    account without one when the user has an OTP command for it, where atsfill can sign in
     through the adapter; None otherwise."""
-    if not isinstance(adapter, SignsIn):
+    if not isinstance(adapter, (SignsIn, SignsInByOtp)):
         return None
     if not (email := accounts.email(mine, adapter.name, scope)):
         return None
+    if isinstance(adapter, SignsInByOtp):
+        return (email, None) if otp.command(user, adapter.name) else None
     password = passwords.password(user, mine, adapter.name, scope, email, given)
     return (email, password) if password else None
 
 
 @contextmanager
 def _opened(
-    data: Path, user: str, url: str, sessions: Sessions | None, detected: list[str]
+    data: Path, user: str, url: str, sessions: Sessions | None, seen: Seen
 ) -> Generator[tuple[Page, Adapter] | Result]:
     """The loaded page and the adapter that claims it by its URL, else by the URL a redirect
     lands on, else by its page; or the result that ends the command first. An adapter that
     needs an account gets the user's session, and only that adapter's; signed out with a
-    password source, atsfill signs in again (ADR 0015). The adapter's name goes in `detected`
+    password source, or an OTP command for an account without a password, atsfill signs in
+    again (ADR 0015). The adapter's name goes in `seen`
     once known. The adapters are fresh: none keeps state from another
     posting (a survey or the adapter gate runs many commands in one process)."""
     candidates = adapters()
     adapter = _detect(candidates, url)
     if adapter:
-        detected.append(adapter.name)
+        seen.adapter = adapter.name
     mine = paths.user_dir(data, user)
     session, credentials = None, None
     if isinstance(adapter, NeedsAccount):
@@ -129,7 +154,7 @@ def _opened(
         if adapter is None and (
             adapter := _detect(candidates, page.url) or _detect(candidates, url, page)
         ):
-            detected.append(adapter.name)
+            seen.adapter = adapter.name
         if adapter is None:
             yield _no_adapter(url)
             return
@@ -151,8 +176,8 @@ def _sign_in_again(
     url: str,
     credentials: Credentials | None,
 ) -> Result | None:
-    """Signs in with the account's password source, stores the session and loads the posting
-    again; the result that ends the command instead, `login_required` without a source."""
+    """Signs in as `_credentials` allows, stores the session and loads the posting again; the
+    result that ends the command instead, `login_required` where it does not."""
     result = Result(Status.BLOCKED, url, adapter.name)
     with _stops(result):
         scope = adapter.account_scope(url)
@@ -161,14 +186,15 @@ def _sign_in_again(
         except passwords.PasswordError as e:
             result.reason = f"password_source: {e}"
             return result
-        if credentials is None or not isinstance(adapter, SignsIn):
+        if credentials is None or not isinstance(adapter, (SignsIn, SignsInByOtp)):
             return _login_required(url, adapter, user)
         session = accounts.session_file(mine, adapter.name, scope)
         accounts.sign_in(page, adapter, scope, *credentials, session)
         page.goto(url)
         return None
-    command = login_command(adapter.name, user, url if adapter.per_employer else None)
-    result.reason = _email_link(result.reason, credentials[0] if credentials else "", command)
+    email = credentials[0] if credentials else ""
+    scoped = url if adapter.per_employer else None
+    result.reason = _next_step(result.reason, email, adapter.name, user, scoped)
     return result
 
 
@@ -247,18 +273,41 @@ def _stops(result: Result) -> Generator[None]:
         result.status, result.reason = Status.BLOCKED, f"overlaid: {e}"
 
 
-def _in_budget(url: str, command: Callable[[list[str]], Result]) -> Result:
+def _in_budget(url: str, seen: Seen, command: Callable[[], Result]) -> Result:
     """Runs a command to a status whatever happens: past the click it reports `unknown` itself,
     before it an error the adapter did not expect means the page is not one it handles. The
     result keeps the adapter detected before the stop."""
-    detected: list[str] = []
     try:
-        return command(detected)
+        return command()
     except OutOfTime as e:
-        return Result(Status.BLOCKED, url, (detected or [None])[-1], f"time_budget: {e}")
+        return Result(Status.BLOCKED, url, seen.adapter, f"time_budget: {e}")
     except Exception as e:
         reason = f"adapter_error: {type(e).__name__}: {e}"
-        return Result(Status.UNSUPPORTED, url, (detected or [None])[-1], reason)
+        return Result(Status.UNSUPPORTED, url, seen.adapter, reason)
+
+
+def _kept(page: Page, seen: Seen, run: Callable[[], Result]) -> Result:
+    """The run's result; a failed run's page goes in `seen`, values stripped, when telemetry
+    is on."""
+    try:
+        result = run()
+    except Exception:
+        _capture(page, seen)
+        raise
+    if telemetry.verdict(result) == "fail":
+        _capture(page, seen)
+    return result
+
+
+def _capture(page: Page, seen: Seen) -> None:
+    with suppress(Exception):  # a page the budget closed has nothing to keep
+        if telemetry.level() != "off":
+            seen.page = snapshot.capture(page)
+
+
+def _secrets(data: Path, user: str, answers: dict[str, Answer], result: Result) -> set[str]:
+    emails = accounts.emails(paths.user_dir(data, user))
+    return snapshot.secrets(answers, result.fields, emails)
 
 
 def _fill(page: Page, adapter: Adapter, url: str, answers: dict[str, Answer], user: str) -> Result:
@@ -281,18 +330,20 @@ def _fill(page: Page, adapter: Adapter, url: str, answers: dict[str, Answer], us
     return result
 
 
-def _browser_read(
-    data: Path, user: str, url: str, sessions: Sessions | None, detected: list[str]
-) -> Result:
-    with _opened(data, user, url, sessions, detected) as opened:
+def _browser_read(data: Path, user: str, url: str, sessions: Sessions | None, seen: Seen) -> Result:
+    with _opened(data, user, url, sessions, seen) as opened:
         if isinstance(opened, Result):
             return opened
         page, adapter = opened
-        result = _result(adapter, url, user)
-        with _stops(result):
-            _listed(result, adapter.browser_read(page))
-            _unfillable(result)
-        return result
+
+        def run() -> Result:
+            result = _result(adapter, url, user)
+            with _stops(result):
+                _listed(result, adapter.browser_read(page))
+                _unfillable(result)
+            return result
+
+        return _kept(page, seen, run)
 
 
 def _form_command(
@@ -303,46 +354,59 @@ def _form_command(
     sessions: Sessions | None,
     send: bool,
     resend: bool,
-    detected: list[str],
+    seen: Seen,
 ) -> Result:
     mine = paths.user_dir(data, user)  # before the page takes the lock its first use needs
-    with _opened(data, user, url, sessions, detected) as opened:
+    with _opened(data, user, url, sessions, seen) as opened:
         if isinstance(opened, Result):
             return opened
         page, adapter = opened
-        posting = adapter.posting(page.url)  # past any redirect a link goes through
-        if send and not paths.submit_enabled(adapter.name, user):
-            where = "" if user == paths.DEFAULT_USER else f" under [users.{user}]"
-            reason = (
-                f"submit_not_enabled: add {adapter.name} to `submit-allowed`{where}"
-                f" in {paths.config_file()}"
-            )
-            return Result(Status.REFUSED, url, adapter.name, reason=reason)
-        if send and not resend and attempts.blocks_send(mine, posting):
-            last = attempts.last(mine, posting)
-            reason = f"already_sent: last attempt ended {last}; pass --resend"
-            return Result(Status.REFUSED, url, adapter.name, reason=reason)
-        result = _fill(page, adapter, url, answers, user)
-        if result.status is not Status.READY:
-            return result
-        if not send:
-            return result
-        attempts.record(mine, posting, attempts.PENDING, url=url)
-        try:
-            result.evidence = adapter.submit(page)
-            if result.evidence:
-                result.status = Status.SUBMITTED
-            else:
-                result.status, result.reason = Status.UNKNOWN, "no_evidence"
-        except Blocked as e:
-            result.status, result.reason = Status.BLOCKED, str(e)
-        except Exception as e:  # after the click, nothing tells a failed send from a sent one
-            result.status = Status.UNKNOWN
-            result.reason = f"error_after_click: {type(e).__name__}: {e}"
-        attempts.record(
-            mine, posting, result.status, evidence=result.evidence, reason=result.reason
+        return _kept(
+            page, seen, lambda: _on_page(page, adapter, mine, user, url, answers, send, resend)
         )
+
+
+def _on_page(
+    page: Page,
+    adapter: Adapter,
+    mine: Path,
+    user: str,
+    url: str,
+    answers: dict[str, Answer],
+    send: bool,
+    resend: bool,
+) -> Result:
+    posting = adapter.posting(page.url)  # past any redirect a link goes through
+    if send and not paths.submit_enabled(adapter.name, user):
+        where = "" if user == paths.DEFAULT_USER else f" under [users.{user}]"
+        reason = (
+            f"submit_not_enabled: add {adapter.name} to `submit-allowed`{where}"
+            f" in {paths.config_file()}"
+        )
+        return Result(Status.REFUSED, url, adapter.name, reason=reason)
+    if send and not resend and attempts.blocks_send(mine, posting):
+        last = attempts.last(mine, posting)
+        reason = f"already_sent: last attempt ended {last}; pass --resend"
+        return Result(Status.REFUSED, url, adapter.name, reason=reason)
+    result = _fill(page, adapter, url, answers, user)
+    if result.status is not Status.READY:
+        return result
+    if not send:
         return result
+    attempts.record(mine, posting, attempts.PENDING, url=url)
+    try:
+        result.evidence = adapter.submit(page)
+        if result.evidence:
+            result.status = Status.SUBMITTED
+        else:
+            result.status, result.reason = Status.UNKNOWN, "no_evidence"
+    except Blocked as e:
+        result.status, result.reason = Status.BLOCKED, str(e)
+    except Exception as e:  # after the click, nothing tells a failed send from a sent one
+        result.status = Status.UNKNOWN
+        result.reason = f"error_after_click: {type(e).__name__}: {e}"
+    attempts.record(mine, posting, result.status, evidence=result.evidence, reason=result.reason)
+    return result
 
 
 def read(
@@ -354,11 +418,12 @@ def read(
 ) -> Result:
     """Lists the form's fields by a request read when an adapter has one (ADR 0011), else by a
     browser read."""
-    data = paths.data_dir(data_dir)
-    return telemetry.sent("read", _read(data, user, url, sessions), data)
+    data, seen = paths.data_dir(data_dir), Seen()
+    result = _read(data, user, url, sessions, seen)
+    return telemetry.sent("read", result, data, seen.page, _secrets(data, user, {}, result))
 
 
-def _read(data: Path, user: str, url: str, sessions: Sessions | None) -> Result:
+def _read(data: Path, user: str, url: str, sessions: Sessions | None, seen: Seen) -> Result:
     for adapter in adapters():
         if not isinstance(adapter, ReadsByRequest):
             continue
@@ -370,7 +435,7 @@ def _read(data: Path, user: str, url: str, sessions: Sessions | None) -> Result:
                 return Result(Status.UNSUPPORTED, url, adapter.name, f"adapter_error: {e}")
             _unfillable(result)
             return result
-    return _in_budget(url, lambda seen: _browser_read(data, user, url, sessions, seen))
+    return _in_budget(url, seen, lambda: _browser_read(data, user, url, sessions, seen))
 
 
 def fill(
@@ -381,11 +446,11 @@ def fill(
     data_dir: Path | None = None,
     sessions: Sessions | None = None,
 ) -> Result:
-    data = paths.data_dir(data_dir)
+    data, seen = paths.data_dir(data_dir), Seen()
     result = _in_budget(
-        url, lambda seen: _form_command(data, user, url, answers, sessions, False, False, seen)
+        url, seen, lambda: _form_command(data, user, url, answers, sessions, False, False, seen)
     )
-    return telemetry.sent("fill", result, data)
+    return telemetry.sent("fill", result, data, seen.page, _secrets(data, user, answers, result))
 
 
 def send(
@@ -397,11 +462,16 @@ def send(
     data_dir: Path | None = None,
     sessions: Sessions | None = None,
 ) -> Result:
-    data = paths.data_dir(data_dir)
+    data, seen = paths.data_dir(data_dir), Seen()
     result = _in_budget(
-        url, lambda seen: _form_command(data, user, url, answers, sessions, True, resend, seen)
+        url, seen, lambda: _form_command(data, user, url, answers, sessions, True, resend, seen)
     )
-    return telemetry.sent("send", result, data)
+    return telemetry.sent("send", result, data, seen.page, _secrets(data, user, answers, result))
+
+
+def report(*, data_dir: Path | None = None) -> ReportResult:
+    """Sends the last failed run's URL and scrubbed page, once (ADR 0016)."""
+    return telemetry.report(paths.data_dir(data_dir))
 
 
 def _sessions(data: Path, users: list[str], candidates: list[Adapter]) -> list[SessionReport]:
@@ -478,7 +548,9 @@ def login(
     creates the account with a password atsfill generates and stores first (a side effect),
     giving the sign-up the user's `details`; with a password already stored, the account may
     exist, so it signs in with that password and keeps the account's email instead, the
-    `details` finishing a sign-up the account was left in."""
+    `details` finishing a sign-up the account was left in. An account without a password signs
+    in through the user's OTP command in place of a password source, and is created the same
+    way."""
     account, data = _account(adapter), paths.data_dir(data_dir)
     scope = _scope(account, url)
     mine = paths.user_dir(data, user)
@@ -488,8 +560,11 @@ def login(
         raise ValueError("creating an account needs --email")
     if details and not create:
         raise ValueError("--details goes with --create")
+    by_otp = isinstance(account, SignsInByOtp)
+    if password and by_otp:
+        raise ValueError(f"{adapter} signs in with an emailed OTP, not a password")
     try:
-        kept = create and passwords.stored(mine, adapter, scope) is not None
+        kept = create and not by_otp and passwords.stored(mine, adapter, scope) is not None
     except passwords.PasswordError as e:
         report.status, report.reason = Status.BLOCKED, f"password_source: {e}"
         return report
@@ -523,11 +598,8 @@ def login(
         )
     else:
         result = _headless(data, user, account, scope, credentials, session, details)
-        command = login_command(adapter, user, url)
-        report.status, report.reason = (
-            result.status,
-            _email_link(result.reason, credentials[0], command),
-        )
+        report.status = result.status
+        report.reason = _next_step(result.reason, credentials[0], adapter, user, url)
     if report.reason is None:
         report.session = report.status = "signed_in"
     return report
@@ -544,8 +616,9 @@ def _create(
 ) -> SessionReport:
     """Creates the account: refused before the browser opens when `details` lack one the
     sign-up asks; the generated password is stored before the click, and deleted only on a
-    refusal the page states before any account exists."""
-    if not isinstance(account, CreatesAccount):
+    refusal the page states before any account exists. An account without a password gets
+    none."""
+    if not isinstance(account, (CreatesAccount, CreatesAccountByOtp)):
         report.status = Status.UNSUPPORTED
         report.reason = f"no_sign_up: atsfill cannot create a {account.name} account"
         return report
@@ -553,9 +626,10 @@ def _create(
         report.status = Status.REFUSED
         report.reason = f"sign_up_details: no {', '.join(missing)}"
         return report
-    password = passwords.generate()
+    password = None if isinstance(account, CreatesAccountByOtp) else passwords.generate()
     try:
-        passwords.store(mine, account.name, report.scope, password)
+        if password:
+            passwords.store(mine, account.name, report.scope, password)
     except passwords.PasswordError as e:
         report.status, report.reason = Status.BLOCKED, f"password_source: {e}"
         return report
@@ -563,10 +637,14 @@ def _create(
     result = _headless(
         data, report.user, account, report.scope, credentials, report.path, details, create=True
     )
-    command = login_command(account.name, report.user, url)
-    report.status, report.reason = result.status, _email_link(result.reason, email, command)
-    report.password = "stored"
-    if result.status is Status.REFUSED and (result.reason or "").split(":")[0] in NO_ACCOUNT:
+    report.status = result.status
+    report.reason = _next_step(result.reason, email, account.name, report.user, url)
+    report.password = "stored" if password else "none"
+    if (
+        password
+        and result.status is Status.REFUSED
+        and (result.reason or "").split(":")[0] in NO_ACCOUNT
+    ):
         passwords.delete(mine, account.name, report.scope)
         report.password = "deleted"
     if report.reason is None:
@@ -586,16 +664,16 @@ def _headless(
 ) -> Result:
     """Signs in, or creates the account with `create`, through the adapter, an OTP it meets
     from the user's `otp-commands` entry: `ready` once the session is stored."""
-    assert isinstance(account, SignsIn)
+    assert isinstance(account, (SignsIn, SignsInByOtp))
     url = account.login_url(scope)
 
-    def run(_: list[str]) -> Result:
+    def run() -> Result:
         result = Result(Status.READY, url, account.name)
         with open_page(data, None) as page, _stops(result), otp.source(user, account.name, scope):
             accounts.sign_in(page, account, scope, *credentials, session, details, create)
         return result
 
-    return _in_budget(url, run)
+    return _in_budget(url, Seen(), run)
 
 
 def logout(

{atsfill-0.0.5.dev143 → atsfill-0.0.5.dev145}/src/atsfill/core/results.py

@@ -102,6 +102,14 @@ class Result:
     side_effects: list[SideEffect] = field(default_factory=list)
     evidence: str | None = None
     notice: str | None = None  # telemetry's, the first time it is on (ADR 0016)
+    report: str | None = None  # how to send a failed run's report, when it was not sent
+
+
+@dataclass
+class ReportResult:
+    status: str  # sent | refused | blocked
+    reason: str | None = None
+    url: str | None = None  # the failed run's posting
 
 
 @dataclass
@@ -113,7 +121,7 @@ class SessionReport:
     path: Path
     status: str | None = None  # a login's outcome: signed_in | refused | blocked | unsupported
     reason: str | None = None  # why a login stored nothing, `code` or `code: detail`
-    password: str | None = None  # after `--create`: stored | deleted, or kept
+    password: str | None = None  # after `--create`: stored | deleted | kept, or none (OTP only)
 
 
 @dataclass

atsfill-0.0.5.dev145/src/atsfill/core/snapshot.py

@@ -0,0 +1,75 @@
+"""A failed run's page, for `detailed` telemetry and `atsfill report` (ADR 0016): the DOM
+without scripts and styles, every value a field holds stripped in the page, then every answer,
+uploaded file name and account email replaced, here in the core, before it is kept."""
+
+import html
+import re
+from collections.abc import Iterable
+from pathlib import Path
+
+from atsfill.core.browser import Page
+from atsfill.core.results import Answer, Field
+
+MAX_BYTES = 800 * 1024  # the SDK drops an event above 900 KiB; room for its other properties
+MIN_SECRET = 3  # shorter answers ("No", "5") would wipe the page's own words, and tell little
+NAME_WORDS = 4  # an answer of up to this many words is scrubbed word by word too
+SCRUBBED = "[scrubbed]"
+TRUNCATED = "<!-- truncated -->"
+EMAIL = re.compile(r"[\w.+-]+@[\w-]+(?:\.[\w-]+)+")
+
+# The page and the failing step's subtree, the smallest element holding every form control.
+CLEAN = """() => {
+    const root = document.documentElement.cloneNode(true);
+    root.querySelectorAll("script, style, noscript, template, svg, link, meta").forEach(
+        e => e.remove());
+    for (const e of root.querySelectorAll("*")) {
+        for (const a of ["value", "checked", "selected", "aria-valuenow", "aria-valuetext"])
+            e.removeAttribute(a);
+    }
+    root.querySelectorAll("textarea, [contenteditable]").forEach(e => { e.textContent = ""; });
+    const controls = [...root.querySelectorAll(
+        "input, select, textarea, button, [role=combobox], [role=textbox], [contenteditable]")];
+    let step = controls[0] ?? root.querySelector("body") ?? root;
+    while (!controls.every(c => step.contains(c))) step = step.parentElement;
+    return [root.outerHTML, step.outerHTML];
+}"""
+
+
+def capture(page: Page) -> tuple[str, str]:
+    """The page and its failing step, values stripped, not yet scrubbed of secrets."""
+    whole, step = page.evaluate(CLEAN)
+    return whole, step
+
+
+def secrets(answers: dict[str, Answer], fields: Iterable[Field], emails: Iterable[str]) -> set[str]:
+    """What may not leave the machine: each answer, a file answer's name and stem, each
+    prefilled value and each account email; each word of a short one, a name the page may
+    show in part."""
+    out = set(emails)
+    values = [*answers.values(), *(f.prefilled for f in fields if f.prefilled)]
+    for value in values:
+        for v in [value] if isinstance(value, str) else value:
+            out |= {v, Path(v).name, Path(v).stem} if "/" in v or "\\" in v else {v}
+            if len(words := v.split()) <= NAME_WORDS:
+                out |= set(words)
+    return {s.strip() for s in out if len(s.strip()) >= MIN_SECRET}
+
+
+def scrub(text: str, secrets: Iterable[str]) -> str:
+    """The text with each secret, as typed or HTML-escaped, and every email address replaced."""
+    forms = {f for s in secrets for f in (s, html.escape(s), html.escape(s, quote=False))}
+    if forms:
+        alternatives = "|".join(re.escape(f) for f in sorted(forms, key=len, reverse=True))
+        whole_word = re.compile(rf"(?<![^\W_])(?:{alternatives})(?![^\W_])", re.IGNORECASE)
+        text = whole_word.sub(SCRUBBED, text)
+    return EMAIL.sub(SCRUBBED, text)
+
+
+def kept(captured: tuple[str, str], secrets: Iterable[str]) -> str:
+    """The scrubbed page, or its failing step when the page is over the limit, cut at it."""
+    whole, step = (scrub(c, secrets) for c in captured)
+    for snapshot in (whole, step):
+        if len(snapshot.encode()) <= MAX_BYTES:
+            return snapshot
+    cut = step.encode()[: MAX_BYTES - len(TRUNCATED)].decode(errors="ignore")
+    return cut + TRUNCATED

{atsfill-0.0.5.dev143 → atsfill-0.0.5.dev145}/src/atsfill/core/telemetry.py

@@ -1,6 +1,7 @@
 """Anonymous telemetry (ADR 0016): one `run` event per `read`, `fill` and `send`, built from
-the command's result. Never in the way: any failure is dropped and the result goes back as it
-came, but for the first-run notice."""
+the command's result, and for a failed run a `report` event with its URL and scrubbed page, at
+`detailed` or by `atsfill report`. Never in the way: any failure is dropped and the result goes
+back as it came, but for the notices."""
 
 import hashlib
 import json
@@ -9,6 +10,7 @@ import os
 import subprocess
 import sys
 import uuid
+from collections.abc import Iterable
 from contextlib import suppress
 from functools import cache
 from importlib.metadata import PackageNotFoundError, distribution, version
@@ -18,8 +20,8 @@ from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
 from posthog import Posthog
 
 from atsfill.adapters import adapters
-from atsfill.core import paths, registry
-from atsfill.core.results import Result, Status
+from atsfill.core import paths, registry, snapshot
+from atsfill.core.results import ReportResult, Result, Status
 
 PROJECT_KEY = "phc_xmrdiXySbbZCdUzdbdNUtjMco3rLL2Ko5j4hFrX9fUQ8"  # ingestion only, safe to ship
 HOST = "https://eu.i.posthog.com"
@@ -29,10 +31,16 @@ FLUSH_TIMEOUT_SECONDS = 2  # the SDK's shutdown() waits without one
 GIT_TIMEOUT_SECONDS = 2
 ROOT = Path(__file__).parents[3]  # the checkout, for an editable install
 NOTICE_FILE = "telemetry-notice"  # in the data directory, once the notice was shown
+FAILED_RUN = "failed-run.json"  # in the data directory, the last failed run's report event
+DOCS = "https://github.com/shbernal/atsfill/blob/main/docs/telemetry.md"
 NOTICE = (
     "telemetry: atsfill sends an anonymous event per read, fill and send, no answers or URLs;"
     ' set telemetry = "off" in {config} or DO_NOT_TRACK=1 to stop it:'
-    " https://github.com/shbernal/atsfill/blob/main/docs/telemetry.md"
+    " " + DOCS
+)
+REPORT = (
+    "run `atsfill report` to send this run's URL and page, scrubbed of your answers,"
+    " to the maintainer: " + DOCS
 )
 # Reason codes that say nothing about the adapter: the user's input or a fact about the posting
 EXCLUDED = (
@@ -132,17 +140,23 @@ def event(command: str, result: Result) -> dict:
     return {k: v for k, v in properties.items() if v is not None}
 
 
-def _capture(properties: dict) -> None:
+def _capture(events: list[tuple[str, dict]]) -> None:
+    """Sends the events; raises when the SDK reports a batch it could not send."""
+    failed: list[Exception] = []
     client = Posthog(
         PROJECT_KEY,
         host=HOST,
         timeout=REQUEST_TIMEOUT_SECONDS,
         max_retries=MAX_RETRIES,
         disable_geoip=True,
+        on_error=lambda e, _: failed.append(e),
     )
-    properties = {**properties, "$process_person_profile": False}
-    client.capture("run", distinct_id=str(uuid.uuid4()), properties=properties)
+    for name, properties in events:
+        properties = {**properties, "$process_person_profile": False}
+        client.capture(name, distinct_id=str(uuid.uuid4()), properties=properties)
     client.flush(timeout_seconds=FLUSH_TIMEOUT_SECONDS)
+    if failed:
+        raise failed[0]
 
 
 def _notice(data: Path) -> str | None:
@@ -154,10 +168,68 @@ def _notice(data: Path) -> str | None:
     return NOTICE.format(config=paths.config_file())
 
 
-def sent(command: str, result: Result, data: Path) -> Result:
-    """The result, its event sent at `basic` and `detailed`."""
+def _report(
+    run: dict, result: Result, captured: tuple[str, str] | None, secrets: Iterable[str]
+) -> dict:
+    """The `report` event: the run's, plus its URL, its reason's detail and its page, each
+    scrubbed of the secrets."""
+    detail = (result.reason or "").partition(":")[2].strip()
+    report = {
+        **run,
+        "url": result.url,
+        "detail": snapshot.scrub(detail, secrets) or None,
+        "snapshot": snapshot.kept(captured, secrets) if captured else None,
+    }
+    return {k: v for k, v in report.items() if v is not None}
+
+
+def sent(
+    command: str,
+    result: Result,
+    data: Path,
+    captured: tuple[str, str] | None = None,
+    secrets: Iterable[str] = (),
+) -> Result:
+    """The result, its event sent at `basic` and `detailed`. A failed run is kept for `atsfill
+    report`, and its report sent along at `detailed`, else named in the result."""
     with suppress(Exception):  # never in the way
-        if level() != "off":
-            result.notice = _notice(data)
-            _capture(event(command, result))
+        if (at := level()) == "off":
+            return result
+        result.notice = _notice(data)
+        run = event(command, result)
+        events = [("run", run | {"url": result.url} if at == "detailed" else run)]
+        if verdict(result) == "fail":
+            report = _report(run, result, captured, secrets)
+            (data / FAILED_RUN).write_text(json.dumps({"event": report, "reported": False}))
+            if at == "detailed":
+                events.append(("report", report))
+            else:
+                result.report = REPORT
+        _capture(events)
+        if len(events) > 1:
+            _mark_reported(data)
     return result
+
+
+def _mark_reported(data: Path) -> None:
+    path = data / FAILED_RUN
+    path.write_text(json.dumps(json.loads(path.read_text()) | {"reported": True}))
+
+
+def report(data: Path) -> ReportResult:
+    """Sends the last failed run's report, once, whatever the level but `off`."""
+    if level() == "off":
+        return ReportResult("refused", "telemetry_off: telemetry is off, nothing is sent")
+    path = data / FAILED_RUN
+    if not path.exists():
+        return ReportResult("refused", "no_failed_run: no failed run to report")
+    kept = json.loads(path.read_text())
+    report, url = kept["event"], kept["event"]["url"]
+    if kept["reported"]:
+        return ReportResult("refused", "already_reported: the last failed run was sent", url)
+    try:
+        _capture([("report", report)])
+    except Exception as e:
+        return ReportResult("blocked", f"not_sent: {type(e).__name__}: {e}", url)
+    _mark_reported(data)
+    return ReportResult("sent", url=url)