procesio-cli 0.1.1__py3-none-any.whl

procesio_cli/tools/web/handlers/contacts.py

@@ -0,0 +1,74 @@
+"""page-contacts - the contact details a public page publishes, without a browser.
+
+For link-in-bio pages (Linktree, Beacons, Stan, ...) and personal sites: one plain
+HTTP GET with a browser User-Agent, then the shared extractor
+(tools._lib.contacts) over mailto links, anchors, visible text and the profile
+links that link-in-bio services embed as JSON. No session, no login, nothing is
+clicked or submitted, and no address is guessed - only what the page shows.
+
+A page that answers 4xx/5xx is a normal outcome when scanning many creators, so it
+comes back as ``reachable: false`` rather than an error. A network failure (DNS,
+timeout, TLS) is an error.
+"""
+from __future__ import annotations
+
+import argparse
+
+from tools._lib import contacts as extractor
+from tools.web.actiondef import ActionDef
+from tools.web.errors import UsageError
+from tools.web.handlers.archive import _HEADERS
+
+MAX_BYTES = 2_000_000
+
+
+def _fetch(url: str, timeout: int) -> tuple[str, int, str, str]:
+    """GET url. Returns (text, status, content_type, final_url)."""
+    import requests
+    try:
+        resp = requests.get(url, headers=_HEADERS, timeout=timeout, allow_redirects=True)
+    except Exception as exc:  # noqa: BLE001 - network shape varies by failure
+        raise UsageError(f"fetch failed for {url!r}: {exc}")
+    body = resp.content[:MAX_BYTES]
+    encoding = resp.encoding or "utf-8"
+    try:
+        text = body.decode(encoding, errors="replace")
+    except LookupError:
+        text = body.decode("utf-8", errors="replace")
+    return text, resp.status_code, resp.headers.get("Content-Type", ""), resp.url
+
+
+def page_contacts(args) -> dict:
+    url = (args.url or "").strip()
+    if not url:
+        raise UsageError("--url is required")
+    if not url.lower().startswith(("http://", "https://")):
+        url = "https://" + url
+    text, status, content_type, final_url = _fetch(url, args.timeout)
+    out = {"url": url, "final_url": final_url, "status": status,
+           "content_type": content_type, "reachable": 200 <= status < 400}
+    if not out["reachable"]:
+        out.update(extractor.empty_result())
+        out["title"] = ""
+        return out
+    ctype = content_type.lower()
+    if "html" in ctype or "<html" in text[:2000].lower():
+        out.update(extractor.extract_html(text, base_url=final_url))
+    else:
+        out.update(extractor.extract(text))
+        out["title"] = ""
+    return out
+
+
+def _args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--url", required=True, help="Public page to read (link-in-bio, personal site)")
+    p.add_argument("--timeout", type=int, default=20, help="Request timeout in seconds (default 20)")
+
+
+ACTIONS = {
+    "page-contacts": ActionDef(
+        page_contacts, _args, needs_driver=False,
+        description=("Emails, profile links and affiliate links a public page "
+                     "publishes (one plain GET; no browser, no session)."),
+    ),
+}

procesio_cli/tools/web/handlers/sessmgmt.py

@@ -0,0 +1,120 @@
+"""Session-management handlers: save-session, list-sessions, delete-session.
+
+These do NOT receive an injected driver (``needs_driver=False``):
+  * ``save-session`` is interactive - it launches its own HEADED browser via the
+    public API and blocks for a human ENTER before saving storageState.
+  * ``list-sessions`` / ``delete-session`` only touch the sessions directory.
+"""
+from __future__ import annotations
+
+import argparse
+
+from tools.web import api, sessions
+from tools.web.actiondef import ActionDef
+
+
+def _release_owning_broker(name: str) -> str | None:
+    """If a serializing broker owns this session (whatsapp/ryver/fgo/mirro),
+    stop it first: its warm browser may hold the profile, and capturing or
+    deleting a session under it would race the broker. Returns the owning
+    tool's name when a running broker was stopped, else None. Best-effort —
+    a broker that cannot be reached must not block a login capture."""
+    try:
+        from tools._lib.webserialize import client as ws_client
+        from tools._lib.webserialize import registry as ws_registry
+    except Exception:  # noqa: BLE001 - lib unavailable must never block
+        return None
+    cfg = ws_registry.owner_of_session(name)
+    if cfg is None:
+        return None
+    try:
+        if ws_client.ping(cfg) is None:
+            return None
+        ws_client.stop(cfg)
+        return cfg.tool
+    except Exception:  # noqa: BLE001
+        return None
+
+
+def save_session(args) -> dict:
+    """Capture a login session interactively (headed browser + ENTER signal)."""
+    stopped = _release_owning_broker(args.name)
+    out = api.save_session(args.name, args.url, channel=args.channel,
+                           persistent=args.persistent,
+                           wait_seconds=getattr(args, "wait_seconds", None),
+                           wait_until_url_excludes=getattr(
+                               args, "wait_until_url_excludes", None))
+    if stopped:
+        out["broker_stopped"] = stopped
+        out["note"] = (f"the {stopped} serializing broker was stopped to free "
+                       "the session; it auto-restarts on that tool's next command")
+    return out
+
+
+def _save_session_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--name", required=True,
+                   help="Session name (letters/digits/._-; how you'll reference it later)")
+    p.add_argument("--url", required=True,
+                   help="Login URL to open in the headed browser")
+    p.add_argument("--channel", default=None,
+                   help="Optional Chromium channel: 'chrome' or 'msedge' "
+                        "(use a real browser instead of the bundled Chromium)")
+    p.add_argument("--persistent", action="store_true",
+                   help="Save a PERSISTENT browser profile (user-data-dir) instead "
+                        "of a storageState snapshot. Required for IndexedDB-auth "
+                        "sites like WhatsApp Web whose login a snapshot can't restore.")
+    p.add_argument("--wait-seconds", type=int, default=None,
+                   help="Wait this long for the login instead of blocking on ENTER. "
+                        "REQUIRED when an agent launches this flow: agent stdin is "
+                        "EOF, so the ENTER wait returns instantly and saves a "
+                        "half-authenticated profile.")
+    p.add_argument("--wait-until-url-excludes", default=None,
+                   help="With --wait-seconds, finish as soon as the address bar "
+                        "leaves this host (e.g. 'accounts.google.com'), instead of "
+                        "waiting the full time.")
+
+
+def list_sessions(args) -> dict:
+    items = sessions.list_all()
+    return {
+        "sessions": items,
+        "count": len(items),
+        "dir": str(sessions.sessions_dir()),
+    }
+
+
+def delete_session(args) -> dict:
+    sessions.validate_name(args.name)
+    confirm = getattr(args, "confirm", False)
+    # Only disturb the broker when we are actually going to delete; a preview
+    # must not have side effects.
+    stopped = _release_owning_broker(args.name) if confirm else None
+    out = sessions.delete(args.name, confirm=confirm)
+    if stopped:
+        out["broker_stopped"] = stopped
+    return out
+
+
+def _delete_session_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--name", required=True, help="Session name to delete")
+    p.add_argument("--confirm", action="store_true",
+                   help="Actually delete. Without it this only PREVIEWS what "
+                        "would be removed - deleting a session destroys a login "
+                        "that only a human at a browser can re-establish.")
+
+
+ACTIONS = {
+    "save-session": ActionDef(
+        save_session, _save_session_args, needs_driver=False,
+        description="Interactively capture a login session (headed browser; press ENTER to save).",
+    ),
+    "list-sessions": ActionDef(
+        list_sessions, needs_driver=False,
+        description="List saved sessions (names + metadata only, never contents).",
+    ),
+    "delete-session": ActionDef(
+        delete_session, _delete_session_args, needs_driver=False,
+        description="Delete ALL of a saved session's artifacts (storageState + "
+                    "persistent profile + cookie sidecar). Previews unless --confirm.",
+    ),
+}

procesio_cli/tools/web/main.py

@@ -0,0 +1,128 @@
+"""web tool entrypoint - action dispatch (Playwright-driven, no HTTP API).
+
+Drives web apps that have no API, reusing saved login sessions (Playwright
+storageState). Sessions are sensitive auth material stored in the central
+git-ignored user-data folder (context-state-knowledge/sessions/) - see
+sessions.py and tools/web/sessions/README.md.
+
+Actions:
+  save-session      capture a login session interactively (headed browser)
+  list-sessions     list saved sessions (metadata only)
+  delete-session    remove a saved session
+  run               run a declarative step list against a saved session
+  get-text          load a session, open a URL, return visible text
+  screenshot        load a session, open a URL, save a screenshot
+  archive-document  capture a published document (original + self-contained + extraction)
+
+Playwright is imported lazily (inside the driver) so this tool and its tests
+load even before the browser binaries are installed.
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+TOOL_ROOT = Path(__file__).resolve().parent
+FRAMEWORK_ROOT = TOOL_ROOT.parents[1]
+_VENV_PY = FRAMEWORK_ROOT / ".venv" / "Scripts" / "python.exe"
+if _VENV_PY.exists() and Path(sys.executable).resolve() != _VENV_PY.resolve():
+    import subprocess
+    sys.exit(subprocess.run([str(_VENV_PY), __file__, *sys.argv[1:]]).returncode)
+
+import argparse  # noqa: E402
+
+if str(FRAMEWORK_ROOT) not in sys.path:
+    sys.path.insert(0, str(FRAMEWORK_ROOT))
+
+from tools._lib.io import emit, fail  # noqa: E402
+from tools.web import errors  # noqa: E402
+from tools.web.actiondef import ActionDef  # noqa: E402
+from tools.web.driver import PlaywrightDriver  # noqa: E402
+
+
+class _Parser(argparse.ArgumentParser):
+    def error(self, message: str):
+        raise errors.UsageError(message)
+
+
+def collect_actions() -> dict[str, ActionDef]:
+    actions: dict[str, ActionDef] = {}
+    from tools.web.handlers import archive, browse, contacts, sessmgmt
+    actions.update(sessmgmt.ACTIONS)
+    actions.update(browse.ACTIONS)
+    actions.update(archive.ACTIONS)
+    actions.update(contacts.ACTIONS)
+    return actions
+
+
+ACTIONS = collect_actions()
+
+
+def _default_driver_factory(*, storage_state=None, headless=True, channel=None,
+                            user_data_dir=None, extra_args=None):
+    """Production factory injected into driver-based handlers. Returns an
+    UNSTARTED PlaywrightDriver; the API layer calls .start().
+
+    Accepts ``user_data_dir`` so the API layer can drive a PERSISTENT profile
+    session (auto-detected from a ``<name>.profile`` dir) through the CLI, not
+    just storageState sessions. Also accepts ``extra_args`` because the
+    persistent-profile path in ``api.open_session`` forwards it to the factory
+    (matches ``api._default_factory``); without it, ``web run/get-text/screenshot``
+    on any persistent profile raised an unexpected-keyword TypeError."""
+    return PlaywrightDriver(storage_state=storage_state, headless=headless,
+                            channel=channel, user_data_dir=user_data_dir,
+                            extra_args=extra_args)
+
+
+def dispatch(action: str, argv: list[str], *, driver_factory=None,
+             allow_service: bool = True) -> dict:
+    if action not in ACTIONS:
+        raise errors.UsageError(
+            f"unknown action: {action}. Known: {', '.join(sorted(ACTIONS))}"
+        )
+    defn = ACTIONS[action]
+    parser = _Parser(prog=f"web {action}", description=defn.description)
+    defn.add_args(parser)
+    parsed = parser.parse_args(argv)
+
+    # BROKER-OWNED SESSIONS: when the target session belongs to a serializing
+    # broker (whatsapp/ryver/fgo/mirro — see tools/_lib/webserialize/registry),
+    # the generic browser actions are translated to a step list and executed
+    # INSIDE that broker, serialized with the owning tool's own commands. A
+    # direct run against an owned profile could race the broker's warm browser
+    # and invalidate the session. Unowned sessions behave exactly as before.
+    if defn.needs_driver:
+        from tools.web import brokered
+        cfg = brokered.eligible(parsed, driver_factory=driver_factory,
+                                allow_service=allow_service)
+        if cfg is not None:
+            return brokered.route(
+                cfg, action, parsed,
+                dispatch_direct=lambda: dispatch(action, argv,
+                                                 allow_service=False),
+            )
+
+    if defn.needs_driver:
+        factory = driver_factory or _default_driver_factory
+        return defn.func(factory, parsed)
+    return defn.func(parsed)
+
+
+def main(argv=None) -> None:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    if not argv or argv[0] in ("-h", "--help"):
+        print("Actions:", file=sys.stderr)
+        for name in sorted(ACTIONS):
+            print(f"  {name:<16} {ACTIONS[name].description}", file=sys.stderr)
+        sys.exit(0)
+    action, rest = argv[0], argv[1:]
+    try:
+        result = dispatch(action, rest)
+    except Exception as exc:  # noqa: BLE001
+        code, message, details, exit_code = errors.classify(exc)
+        fail(code, message, details, exit_code)
+    emit(result)
+
+
+if __name__ == "__main__":
+    main()

procesio_cli/tools/web/sessions.py

@@ -0,0 +1,253 @@
+"""Saved-session storage for the web tool.
+
+A *session* is a Playwright ``storageState`` JSON file (cookies + localStorage)
+captured after a human logged in once. It lets later headless runs reuse that
+authentication. These files contain live auth material and are therefore
+**sensitive**:
+
+  * They live under ``context-state-knowledge/sessions/<name>.json`` - the
+    central, git-ignored user-data folder (framework/user-data separation;
+    moved from tools/web/sessions/ on 2026-07-05).
+  * That folder is never committed (see the repo .gitignore).
+  * They are never printed or logged. ``list-sessions`` reports names and
+    metadata (size, mtime) only - never file contents.
+
+Session names are validated to a safe charset so a name can never escape the
+sessions directory (no path traversal, no separators).
+"""
+from __future__ import annotations
+
+import json
+import re
+import shutil
+import time
+from pathlib import Path
+
+from tools.web.errors import SessionError, UsageError
+from tools._lib import userdata
+
+# Saved sessions are USER data (live auth for the user's accounts), so they live
+# in the central user-data folder - context-state-knowledge/sessions/ - not in
+# the tool dir (framework/user-data separation; migrated 2026-07-05, see
+# tools/web/sessions/README.md breadcrumb). Resolved via tools/_lib/userdata.py,
+# relocatable with AAT_USERDATA_DIR.
+SESSIONS_DIR = userdata.sessions_dir()
+
+_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$")
+
+
+def validate_name(name: str) -> str:
+    """Return the name if it is a safe session identifier, else raise UsageError.
+
+    Allowed: letters, digits, dot, dash, underscore; must start alphanumeric;
+    1-64 chars. This blocks path traversal ('..', '/', '\\') by construction.
+    """
+    if not isinstance(name, str) or not _NAME_RE.match(name):
+        raise UsageError(
+            f"invalid session name {name!r}: use 1-64 chars of "
+            "letters/digits/._- starting with a letter or digit"
+        )
+    return name
+
+
+def sessions_dir() -> Path:
+    SESSIONS_DIR.mkdir(parents=True, exist_ok=True)
+    return SESSIONS_DIR
+
+
+def session_path(name: str) -> Path:
+    """Absolute path to a session file. Validates the name first."""
+    validate_name(name)
+    return sessions_dir() / f"{name}.json"
+
+
+def exists(name: str) -> bool:
+    return session_path(name).exists()
+
+
+# --- persistent browser profiles ------------------------------------------
+# Some sites (notably WhatsApp Web) keep their auth in IndexedDB, which a
+# storageState snapshot cannot reliably restore — WhatsApp rejects a
+# deserialized copy with "a database error occurred". For those, the web tool
+# uses a PERSISTENT browser profile (a Chromium user-data-dir): the native
+# on-disk profile is reused as-is, so the login survives across runs.
+
+def profile_dir(name: str) -> Path:
+    """Absolute path to a session's persistent browser-profile directory."""
+    validate_name(name)
+    return sessions_dir() / f"{name}.profile"
+
+
+def profile_exists(name: str) -> bool:
+    """True if a non-empty persistent profile directory exists for the name."""
+    d = profile_dir(name)
+    return d.is_dir() and any(d.iterdir())
+
+
+def require_profile(name: str) -> Path:
+    """Return an existing persistent profile dir or raise SessionError."""
+    if not profile_exists(name):
+        raise SessionError(
+            f"no persistent profile for {name!r}. Capture one with: "
+            f"python scripts/run-tool.py web save-session --name {name} "
+            f"--url <login-url> --persistent"
+        )
+    return profile_dir(name)
+
+
+def require(name: str) -> Path:
+    """Return the path to an existing session or raise SessionError."""
+    p = session_path(name)
+    if not p.exists():
+        raise SessionError(
+            f"no saved session named {name!r}. Capture one with: "
+            f"python scripts/run-tool.py web save-session --name {name} --url <login-url>"
+        )
+    return p
+
+
+def load_state(name: str) -> dict:
+    """Read and parse a session's storageState JSON. Raises SessionError on
+    a missing or corrupt file. The returned dict is the Playwright storage_state."""
+    p = require(name)
+    try:
+        # utf-8-sig tolerates a stray BOM if a session file was hand-edited.
+        return json.loads(p.read_text(encoding="utf-8-sig"))
+    except (ValueError, OSError) as exc:
+        raise SessionError(f"session {name!r} is unreadable/corrupt: {exc}") from exc
+
+
+def write_state(name: str, state: dict) -> Path:
+    """Persist a storageState dict to the session file. Returns the path."""
+    p = session_path(name)
+    p.write_text(json.dumps(state, ensure_ascii=False), encoding="utf-8")
+    return p
+
+
+def artifacts(name: str) -> list[tuple[str, Path]]:
+    """Every on-disk artifact belonging to a session, as (kind, path).
+
+    A session is up to THREE files, not one: the storageState snapshot, the
+    persistent browser profile, and the cookie sidecar. Anything that forgets
+    the last two leaves a stale login behind - ``open_session`` auto-detects
+    ``<name>.profile`` and will silently reuse it.
+    """
+    validate_name(name)
+    return [
+        ("state", session_path(name)),
+        ("profile", profile_dir(name)),
+        ("cookies", cookies_path(name)),
+    ]
+
+
+def present_artifacts(name: str) -> list[str]:
+    """Kinds that actually exist on disk for ``name``."""
+    out = []
+    for kind, path in artifacts(name):
+        if (profile_exists(name) if kind == "profile" else path.exists()):
+            out.append(kind)
+    return out
+
+
+def delete(name: str, *, confirm: bool = False) -> dict:
+    """Delete ALL of a session's artifacts (state + profile + cookie sidecar).
+
+    Deleting a session destroys a real login that only a human at a browser can
+    re-establish, so this REFUSES to act unless ``confirm=True``. Without it the
+    call is a non-destructive preview reporting ``would_remove`` - which also
+    serves as the dry run. Returns a report; never raises for a missing session.
+    """
+    validate_name(name)
+    present = present_artifacts(name)
+    if not present:
+        return {"session": name, "deleted": False, "existed": False, "removed": []}
+    if not confirm:
+        return {"session": name, "deleted": False, "existed": True, "removed": [],
+                "would_remove": present, "confirmation_required": True}
+
+    removed: list[str] = []
+    failed: dict[str, str] = {}
+    for kind, path in artifacts(name):
+        if kind not in present:
+            continue
+        try:
+            if kind == "profile":
+                shutil.rmtree(path)
+            else:
+                path.unlink()
+            removed.append(kind)
+        except OSError as exc:   # e.g. Windows lock: a browser still has the profile
+            failed[kind] = str(exc)
+
+    out = {"session": name, "existed": True, "removed": removed,
+           "deleted": bool(removed) and not failed}
+    if failed:
+        out["failed"] = failed
+        out["note"] = ("some artifacts could not be removed - a browser or "
+                       "serializing broker may still hold the profile; close it "
+                       "and retry")
+    return out
+
+
+# --- cookie sidecar (for persistent profiles) ------------------------------
+# A persistent Chromium profile persists localStorage + IndexedDB to disk, but
+# Chromium NEVER writes *session-scoped* cookies (no Expires/Max-Age) to the
+# profile — they live in memory and vanish when the browser closes. Apps that
+# authenticate with a session cookie (e.g. Ryver) therefore appear logged-out on
+# every reopen of an otherwise-valid profile. The fix: at save time, while the
+# browser is still alive, snapshot ALL cookies (session ones included) to this
+# sidecar; at open time, re-inject them into the fresh context before navigating.
+# The file holds live auth material — same sensitivity/gitignore as the profile.
+
+def cookies_path(name: str) -> Path:
+    """Absolute path to a persistent session's cookie sidecar JSON."""
+    validate_name(name)
+    return sessions_dir() / f"{name}.cookies.json"
+
+
+def cookies_exist(name: str) -> bool:
+    return cookies_path(name).exists()
+
+
+def write_cookies(name: str, cookies: list[dict]) -> Path:
+    """Persist a list of Playwright cookie dicts. Returns the path."""
+    p = cookies_path(name)
+    p.write_text(json.dumps(cookies, ensure_ascii=False), encoding="utf-8")
+    return p
+
+
+def load_cookies(name: str) -> list[dict]:
+    """Read a session's cookie sidecar. Returns [] if absent or unreadable
+    (best-effort: a corrupt sidecar must not block opening the profile)."""
+    p = cookies_path(name)
+    if not p.exists():
+        return []
+    try:
+        data = json.loads(p.read_text(encoding="utf-8-sig"))
+        return data if isinstance(data, list) else []
+    except (ValueError, OSError):
+        return []
+
+
+def _meta(p: Path, *, persistent: bool = False) -> dict:
+    st = p.stat()
+    return {
+        "name": p.stem,
+        "bytes": st.st_size,
+        "modified": time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(st.st_mtime)),
+        "persistent": persistent,
+    }
+
+
+def list_all() -> list[dict]:
+    """List saved sessions (metadata only, never contents). Sorted by name.
+
+    Includes BOTH storageState sessions (``<name>.json``) and persistent browser
+    profiles (``<name>.profile/``); the ``persistent`` flag tells them apart.
+    Without the profiles, a session saved via ``--persistent`` would be invisible
+    here even though it exists and is usable.
+    """
+    d = sessions_dir()
+    items = [_meta(p) for p in d.glob("*.json")]
+    items += [_meta(p, persistent=True) for p in d.glob("*.profile") if p.is_dir()]
+    return sorted(items, key=lambda i: i["name"])