ghost-api-cli 0.2.0__py3-none-any.whl

ghost_api/api.py

@@ -0,0 +1,264 @@
+"""Ghost API from Python: the same things the command line does, as functions.
+
+    from ghost_api.api import discover, fetch, diff
+
+    result = discover("https://quotes.toscrape.com/scroll", "list quotes with their authors")
+    print(result.status, [e["path"] for e in result.endpoints], result.client_files)
+
+    rows = fetch("quotes.toscrape.com/scroll", "get_quotes", pages=3)
+    changes = diff("quotes.toscrape.com/scroll", live=True)
+
+Every function respects the same rules as the CLI (robots.txt, rate limits, the network
+boundary, personal data hidden) and uses the project's settings (ghost-api.toml) unless
+`db` / `out` are given. Problems are exceptions: `NotAllowed`, `NoLLMKey`,
+`NothingRemembered`, `GhostApiError`.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from ghost_api import drift, workflows
+from ghost_api.config import Settings, default_db_path, load_env
+from ghost_api.events import Event, EventBus
+from ghost_api.generator import NothingToGenerate, generate_client, site_spec
+from ghost_api.memory.store import Store
+from ghost_api.pipeline import RunOptions
+from ghost_api.runtime import OperationError, SiteRuntime
+from ghost_api.site import SiteAmbiguous, resolve_site
+from ghost_api.workflows import NoLLMKey, NotAllowed, NothingRemembered
+
+__all__ = [
+    "GhostApiError",
+    "NoLLMKey",
+    "NotAllowed",
+    "NothingRemembered",
+    "RunResult",
+    "diff",
+    "discover",
+    "fetch",
+    "generate",
+    "operations",
+    "refresh",
+    "sites",
+]
+
+
+class GhostApiError(Exception):
+    """Something the caller can fix: an unknown site, nothing stored yet, a bad input."""
+
+
+@dataclass
+class RunResult:
+    status: str  # goal-covered, finished, captured, bot-protection, interrupted, failed...
+    reason: str
+    site: str
+    run_id: int | None
+    endpoints: list[dict[str, Any]]  # one summary per endpoint (path, status, params...)
+    client: dict[str, Any] | None  # class name, directory, files, methods
+    report: str
+    report_html: str
+    openapi: str
+    network: dict[str, Any] = field(default_factory=dict)  # the run's permissions summary
+    raw: dict[str, Any] = field(default_factory=dict)  # everything, as `--json` prints it
+
+    @property
+    def ok(self) -> bool:
+        return self.status not in ("failed", "llm-unavailable", "bot-protection", "interrupted")
+
+    @property
+    def client_files(self) -> list[str]:
+        if not self.client:
+            return []
+        return [str(Path(self.client["directory"]) / f) for f in self.client.get("files", [])]
+
+
+def _result(data: dict[str, Any]) -> RunResult:
+    return RunResult(
+        status=str(data.get("status", "failed")),
+        reason=str(data.get("reason", "")),
+        site=str(data.get("site", "")),
+        run_id=data.get("run_id"),
+        endpoints=list(data.get("endpoints", [])),
+        client=data.get("client"),
+        report=str(data.get("report", "")),
+        report_html=str(data.get("report_html", "")),
+        openapi=str(data.get("openapi", "")),
+        network=dict(data.get("network", {})),
+        raw=data,
+    )
+
+
+def _bus(on_event: Callable[[Event], None] | None) -> tuple[EventBus, workflows.RunCollector]:
+    load_env()
+    bus = EventBus()
+    collector = workflows.RunCollector()
+    bus.subscribe(collector)
+    if on_event is not None:
+        bus.subscribe(on_event)
+    return bus, collector
+
+
+def _db(db: Path | str | None) -> Path:
+    return Path(db) if db is not None else default_db_path()
+
+
+def _site(store: Store, name: str) -> str:
+    try:
+        return resolve_site(name, {r.domain for r in store.list_runs()})
+    except SiteAmbiguous as exc:
+        raise GhostApiError(str(exc)) from exc
+
+
+def discover(
+    url: str,
+    goal: str,
+    *,
+    use_agent: bool = True,
+    lang: str = "python",
+    out: Path | str = "output",
+    db: Path | str | None = None,
+    headed: bool = False,
+    max_steps: int = 40,
+    max_llm_calls: int = 50,
+    max_minutes: float = 15.0,
+    rate_limit: float = 1.0,
+    verify: bool = True,
+    probe: bool = True,
+    on_event: Callable[[Event], None] | None = None,
+) -> RunResult:
+    """Explore a site (with the AI agent unless `use_agent=False`), test what it found,
+    and write a client and report. `on_event` receives every live event."""
+    settings = Settings(
+        url=url, goal=goal, lang=lang, headed=headed, max_steps=max_steps,
+        max_llm_calls=max_llm_calls, max_minutes=max_minutes, rate_limit=rate_limit,
+        db_path=_db(db),
+    )  # fmt: skip
+    opts = RunOptions(lang=lang, headed=headed, max_steps=max_steps, max_minutes=max_minutes,
+                      no_probe=not probe, no_verify=not verify, out=Path(out))  # fmt: skip
+    bus, collector = _bus(on_event)
+    workflows.run(settings, opts, bus, use_agent=use_agent)
+    return _result(collector.data)
+
+
+def refresh(
+    site: str,
+    *,
+    use_llm: bool = True,
+    lang: str = "python",
+    out: Path | str = "output",
+    db: Path | str | None = None,
+    verify: bool = True,
+    on_event: Callable[[Event], None] | None = None,
+) -> RunResult:
+    """Re-check a known site from memory: replays first, the browser only if needed."""
+    path = _db(db)
+    with Store(path) as store:
+        name = _site(store, site)
+    opts = RunOptions(lang=lang, max_steps=10, no_verify=not verify, out=Path(out))
+    bus, collector = _bus(on_event)
+    workflows.refresh(path, name, opts, bus, use_llm=use_llm)
+    return _result(collector.data)
+
+
+def generate(
+    site: str, *, lang: str = "python", out: Path | str = "output", db: Path | str | None = None
+) -> dict[str, Any]:
+    """(Re)write a site's client from stored data (no browser, no LLM). Returns the client's
+    class name, files and version."""
+    with Store(_db(db)) as store:
+        name = _site(store, site)
+        try:
+            result = generate_client(store, name, Path(out), lang=lang)
+        except NothingToGenerate as exc:
+            raise GhostApiError(str(exc)) from exc
+    client = result.client
+    return {
+        "class_name": result.spec.class_name,
+        "directory": str(client.directory),
+        "files": [str(p) for p in (client.client_path, client.models_path, client.tests_path)],
+        "version": result.version.version if result.version else result.spec.version,
+    }
+
+
+def operations(site: str, *, db: Path | str | None = None) -> list[dict[str, Any]]:
+    """What can be fetched from a site: name, summary, endpoint, inputs (with examples)."""
+    with Store(_db(db)) as store:
+        name = _site(store, site)
+        try:
+            runtime = SiteRuntime(store, name)
+        except OperationError as exc:
+            raise GhostApiError(str(exc)) from exc
+    with runtime:
+        return [
+            {"name": op.name, "summary": op.summary, "endpoint": op.label,
+             "inputs": {i.name: i.example for i in op.inputs}, "pagination": op.pagination}
+            for op in runtime.operations()
+        ]  # fmt: skip
+
+
+def fetch(
+    site: str,
+    operation: str,
+    inputs: dict[str, str] | None = None,
+    *,
+    pages: int = 5,
+    max_rows: int = 1000,
+    db: Path | str | None = None,
+) -> list[Any]:
+    """Rows from one operation, following pages. Read-only, rate-limited, personal data
+    hidden (like `ghost-api fetch`)."""
+    with Store(_db(db)) as store:
+        name = _site(store, site)
+        try:
+            runtime = SiteRuntime(store, name)
+        except OperationError as exc:
+            raise GhostApiError(str(exc)) from exc
+    rows: list[Any] = []
+    with runtime:
+        try:
+            op = runtime.operation(operation)
+            for page in runtime.iterate(op, inputs or {}, pages, max_rows):
+                if not page.result.ok:
+                    detail = page.result.error or f"HTTP {page.result.status}"
+                    if not rows:
+                        raise GhostApiError(f"{operation} failed: {detail}")
+                    break
+                rows += page.records
+        except OperationError as exc:
+            raise GhostApiError(str(exc)) from exc
+    return rows
+
+
+def diff(site: str, *, live: bool = False, db: Path | str | None = None) -> dict[str, Any]:
+    """What changed in a site's API since the last client version: version, previous and
+    a list of changes ({"severity", "where", "text"})."""
+    with Store(_db(db)) as store:
+        name = _site(store, site)
+        try:
+            spec = site_spec(store, name)
+        except NothingToGenerate as exc:
+            raise GhostApiError(str(exc)) from exc
+        info = drift.version_for(store, name, spec)
+        changes = list(info.changes)
+        if live:
+            changes += drift.live_check(store, name)
+    return {
+        "site": name,
+        "version": info.version,
+        "previous": info.previous,
+        "changes": drift.changes_json(changes),
+        "breaking": sum(c.severity == "breaking" for c in changes),
+    }
+
+
+def sites(*, db: Path | str | None = None) -> list[str]:
+    """Sites explored so far, newest first."""
+    path = _db(db)
+    if not path.exists():
+        return []
+    with Store(path) as store:
+        return list(dict.fromkeys(r.domain for r in store.list_runs()))

ghost_api/audit.py

@@ -0,0 +1,78 @@
+"""Audit log: every request Ghost API sent itself, and every request its rules blocked.
+
+One JSON line per event, appended to `ghost_api_files/audit/<site>.jsonl` next to the
+database, so a team can answer "what did this tool send to that site, and when?".
+Only the method, host, path, the names of query parameters, the status and the decision
+are kept: never query values, bodies, headers or cookies.
+
+    {"at": "2026-10-17T09:12:03+00:00", "site": "example.com", "run": 12, "kind": "direct",
+     "method": "GET", "host": "example.com", "path": "/api/items", "query": ["page"],
+     "status": 200, "decision": "sent"}
+"""
+
+from __future__ import annotations
+
+import json
+import threading
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+from urllib.parse import parse_qsl, urlsplit
+
+from ghost_api.site import to_url_id
+
+_lock = threading.Lock()
+
+
+def audit_file(db_path: Path, site: str) -> Path:
+    return db_path.parent / f"{db_path.stem}_files" / "audit" / f"{to_url_id(site)}.jsonl"
+
+
+class AuditLog:
+    def __init__(self, db_path: Path, site: str, run_id: int | None = None) -> None:
+        self.path = audit_file(db_path, site)
+        self.site = site
+        self.run_id = run_id
+
+    def record(self, kind: str, method: str, url: str, status: int | None, decision: str) -> None:
+        parts = urlsplit(url)
+        entry: dict[str, Any] = {
+            "at": datetime.now(UTC).isoformat(timespec="seconds"),
+            "site": self.site,
+            "run": self.run_id,
+            "kind": kind,  # direct (endpoint tests, fetch, watch, serve) or blocked
+            "method": method.upper(),
+            "host": (parts.hostname or "").lower(),
+            "path": parts.path or "/",
+            "query": sorted({k for k, _ in parse_qsl(parts.query, keep_blank_values=True)}),
+            "status": status,
+            "decision": decision,
+        }
+        line = json.dumps(entry, ensure_ascii=False) + "\n"
+        with _lock:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            with self.path.open("a", encoding="utf-8") as handle:
+                handle.write(line)
+
+    def blocked(self, message: str) -> None:
+        """A rule stopped a request; the message names the host and path."""
+        entry = {
+            "at": datetime.now(UTC).isoformat(timespec="seconds"),
+            "site": self.site,
+            "run": self.run_id,
+            "kind": "blocked",
+            "decision": message[:300],
+        }
+        with _lock:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            with self.path.open("a", encoding="utf-8") as handle:
+                handle.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+
+def read(db_path: Path, site: str, limit: int | None = None) -> list[dict[str, Any]]:
+    path = audit_file(db_path, site)
+    if not path.exists():
+        return []
+    lines = path.read_text(encoding="utf-8").splitlines()
+    entries = [json.loads(line) for line in lines if line.strip()]
+    return entries[-limit:] if limit else entries

ghost_api/browser/auth_state.py

@@ -0,0 +1,172 @@
+"""Saved logins: the browser session (cookies and site storage) after you log in yourself.
+
+`ghost-api login URL` opens a visible browser; you log in by hand (Ghost API never sees or
+stores a password); the session is then saved per site. Later runs and refreshes open the
+browser already logged in, and fetch / watch / serve add the session's cookies to their
+direct calls. `ghost-api logout SITE` (or `forget`) deletes it.
+
+At rest:
+  - Windows: encrypted with DPAPI for your Windows account, so another account, or a copy
+    of the file on another machine, can't read it.
+  - Elsewhere: a file only your user can read (mode 600), not encrypted. Use disk
+    encryption, or log out when you are done.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import time
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlsplit
+
+from ghost_api.site import to_url_id
+
+MAGIC_DPAPI = b"GHOSTAPI-DPAPI1\n"
+MAGIC_PLAIN = b"GHOSTAPI-PLAIN1\n"
+
+
+class LoginError(Exception):
+    """A saved login can't be read (damaged, or made by another Windows account)."""
+
+
+def logins_dir(db_path: Path) -> Path:
+    """Next to the database, like screenshots: never in the output folder."""
+    return db_path.parent / f"{db_path.stem}_files" / "logins"
+
+
+def _file(db_path: Path, site: str) -> Path:
+    return logins_dir(db_path) / f"{to_url_id(site)}.session"
+
+
+def save(db_path: Path, site: str, state: dict[str, Any]) -> dict[str, Any]:
+    """Save a Playwright storage state; returns what was saved (no values)."""
+    meta = {
+        "site": site,
+        "saved_at": datetime.now(UTC).isoformat(timespec="seconds"),
+        "cookies": len(state.get("cookies", [])),
+        "domains": sorted({str(c.get("domain", "")).lstrip(".") for c in state.get("cookies", [])}),
+        "encrypted": sys.platform == "win32",
+    }
+    payload = json.dumps({"meta": meta, "state": state}).encode("utf-8")
+    path = _file(db_path, site)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    data = MAGIC_DPAPI + _protect(payload) if meta["encrypted"] else MAGIC_PLAIN + payload
+    # Created with owner-only permissions before any secret is written.
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    with os.fdopen(fd, "wb") as handle:
+        handle.write(data)
+    return meta
+
+
+def load(db_path: Path, site: str) -> tuple[dict[str, Any], dict[str, Any]] | None:
+    """(state, meta) for a site's saved login, or None when there is none."""
+    path = _file(db_path, site)
+    if not path.exists():
+        return None
+    data = path.read_bytes()
+    try:
+        if data.startswith(MAGIC_DPAPI):
+            payload = _unprotect(data[len(MAGIC_DPAPI) :])
+        elif data.startswith(MAGIC_PLAIN):
+            payload = data[len(MAGIC_PLAIN) :]
+        else:
+            raise LoginError("unknown format")
+        saved = json.loads(payload)
+        return saved["state"], saved["meta"]
+    except (OSError, ValueError, KeyError) as exc:
+        raise LoginError(
+            f"The saved login for {site} can't be read ({exc}). Run `ghost-api login` again."
+        ) from exc
+
+
+def delete(db_path: Path, site: str) -> bool:
+    path = _file(db_path, site)
+    if path.exists():
+        path.unlink()
+        return True
+    return False
+
+
+def saved_sites(db_path: Path) -> list[dict[str, Any]]:
+    out = []
+    for path in sorted(logins_dir(db_path).glob("*.session")):
+        try:
+            data = path.read_bytes()
+            if data.startswith(MAGIC_DPAPI):
+                data = MAGIC_PLAIN + _unprotect(data[len(MAGIC_DPAPI) :])
+            out.append(json.loads(data[len(MAGIC_PLAIN) :])["meta"])
+        except (OSError, ValueError, KeyError):
+            out.append({"site": path.stem, "saved_at": "?", "damaged": True})
+    return out
+
+
+def cookie_header(state: dict[str, Any], url: str, now: float | None = None) -> str:
+    """The Cookie header a browser would send to `url` with this session."""
+    parts = urlsplit(url)
+    host, path = (parts.hostname or "").lower(), parts.path or "/"
+    now = time.time() if now is None else now
+    pairs = []
+    for cookie in state.get("cookies", []):
+        domain = str(cookie.get("domain", "")).lower()
+        bare = domain.lstrip(".")
+        if not (host == bare or host.endswith("." + bare)):
+            continue
+        if not path.startswith(str(cookie.get("path") or "/")):
+            continue
+        if cookie.get("secure") and parts.scheme != "https":
+            continue
+        expires = cookie.get("expires", -1)
+        if isinstance(expires, int | float) and 0 < expires < now:
+            continue
+        pairs.append(f"{cookie['name']}={cookie['value']}")
+    return "; ".join(pairs)
+
+
+def age_days(meta: dict[str, Any]) -> float:
+    try:
+        saved = datetime.fromisoformat(str(meta["saved_at"]))
+    except (KeyError, ValueError):
+        return 0.0
+    return (datetime.now(UTC) - saved).total_seconds() / 86400
+
+
+# --- Windows DPAPI (crypt32), through ctypes: no extra dependency ---------------------------
+
+
+def _protect(data: bytes) -> bytes:
+    return _dpapi(data, protect=True)
+
+
+def _unprotect(data: bytes) -> bytes:
+    if sys.platform != "win32":
+        raise LoginError("this login was saved on Windows and can only be read there")
+    return _dpapi(data, protect=False)
+
+
+def _dpapi(data: bytes, protect: bool) -> bytes:
+    if sys.platform != "win32":  # also tells the type checker windll exists below
+        raise LoginError("Windows data protection is only available on Windows")
+    import ctypes
+    from ctypes import wintypes
+
+    class Blob(ctypes.Structure):
+        _fields_ = [("cbData", wintypes.DWORD), ("pbData", ctypes.POINTER(ctypes.c_char))]
+
+    buffer = ctypes.create_string_buffer(data, len(data))
+    blob_in = Blob(len(data), ctypes.cast(buffer, ctypes.POINTER(ctypes.c_char)))
+    blob_out = Blob()
+    crypt32 = ctypes.windll.crypt32
+    kernel32 = ctypes.windll.kernel32
+    call = crypt32.CryptProtectData if protect else crypt32.CryptUnprotectData
+    # CRYPTPROTECT_UI_FORBIDDEN = 0x1: never show a prompt.
+    ok = call(ctypes.byref(blob_in), None, None, None, None, 0x1, ctypes.byref(blob_out))
+    if not ok:
+        raise LoginError(f"Windows could not {'protect' if protect else 'unlock'} the login")
+    try:
+        return ctypes.string_at(blob_out.pbData, blob_out.cbData)
+    finally:
+        kernel32.LocalFree(blob_out.pbData)