ghost-api-cli 0.2.0__py3-none-any.whl

ghost_api/events.py

@@ -0,0 +1,213 @@
+"""Structured events emitted while a run happens.
+
+The pipeline never prints. It emits these events, and each interface renders them:
+the terminal (`ghost_api/ui/terminal.py`), the web dashboard (`ghost_api/web/`), and the
+database (`run_events`), so any past run can be replayed in full.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+import time
+from collections.abc import Callable
+from dataclasses import asdict, dataclass, field
+from typing import Any, ClassVar, Literal
+
+Phase = Literal["checks", "explore", "test", "generate", "verify", "learn", "done"]
+PHASES: tuple[Phase, ...] = ("checks", "explore", "test", "generate", "verify", "learn", "done")
+Level = Literal["info", "success", "warning", "error"]
+
+
+_log = logging.getLogger("ghost_api")
+
+
+@dataclass
+class Event:
+    kind: ClassVar[str] = "event"
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"type": self.kind, **asdict(self)}
+
+
+@dataclass
+class RunStarted(Event):
+    kind: ClassVar[str] = "run_started"
+    run_id: int
+    domain: str
+    start_url: str
+    goal: str
+    mode: Literal["run", "refresh"] = "run"
+    lang: str = "python"
+
+
+@dataclass
+class PhaseChanged(Event):
+    kind: ClassVar[str] = "phase"
+    phase: Phase
+
+
+@dataclass
+class Notice(Event):
+    """A message for the person: progress, warnings, errors (already plain-language)."""
+
+    kind: ClassVar[str] = "notice"
+    text: str
+    level: Level = "info"
+    detail: str = ""
+
+
+@dataclass
+class RequestCaptured(Event):
+    kind: ClassVar[str] = "request"
+    method: str
+    url: str  # redacted
+    status: int
+    third_party: bool = False
+
+
+@dataclass
+class AgentStep(Event):
+    kind: ClassVar[str] = "step"
+    step: int
+    action: str
+    args: dict[str, Any]
+    thought: str
+    result: str
+    new_endpoints: list[str] = field(default_factory=list)
+    requests: int = 0
+    screenshot: str | None = None  # file name in the run's screenshot folder
+    box: dict[str, float] | None = None  # element acted on: x, y, width, height (page px)
+
+
+@dataclass
+class Screenshot(Event):
+    """A browser screenshot outside agent steps (e.g. the first page load)."""
+
+    kind: ClassVar[str] = "screenshot"
+    name: str
+    url: str
+
+
+@dataclass
+class EndpointsUpdated(Event):
+    """The current endpoint map (summaries; full details come from the API/database)."""
+
+    kind: ClassVar[str] = "endpoints"
+    endpoints: list[dict[str, Any]]
+
+
+@dataclass
+class HypothesisTested(Event):
+    kind: ClassVar[str] = "hypothesis"
+    endpoint: str  # display label
+    text: str
+    result: Literal["confirmed", "rejected", "inconclusive"]
+    evidence: str
+
+
+@dataclass
+class ProbeFinished(Event):
+    kind: ClassVar[str] = "probe_done"
+    endpoints: int
+    direct_calls: int
+
+
+@dataclass
+class VerifyRound(Event):
+    kind: ClassVar[str] = "verify_round"
+    number: int
+    passed: int
+    failed: int
+    skipped: int
+    failures: list[dict[str, str]] = field(default_factory=list)  # name, message
+    repairs: list[dict[str, str]] = field(default_factory=list)  # method, kind, action
+
+
+@dataclass
+class ClientGenerated(Event):
+    kind: ClassVar[str] = "client"
+    class_name: str
+    lang: str
+    named_by_llm: bool
+    directory: str
+    methods: list[dict[str, Any]]  # name, args, returns, iterator
+    credentials: list[dict[str, str]] = field(default_factory=list)  # arg, env
+    skipped: list[str] = field(default_factory=list)
+    files: list[str] = field(default_factory=list)
+
+
+@dataclass
+class NetworkSummary(Event):
+    """What a run sent where, and which rules applied (the report's permissions section)."""
+
+    kind: ClassVar[str] = "network"
+    target: str  # the site's host
+    robots: str  # how robots.txt was checked ("allowed by robots.txt", "check skipped"...)
+    rate_limit: float  # requests per second, at most
+    local_target: bool  # private addresses allowed (the target itself is local)
+    saved_login: bool  # a saved login (ghost-api login) was used
+    api_calls_seen: int  # API calls the page made (kept after filtering)
+    direct_calls: int  # requests Ghost API sent itself (endpoint tests)
+    third_party_hosts: list[str] = field(default_factory=list)  # API hosts besides the site
+    blocked: list[str] = field(default_factory=list)  # requests the rules stopped
+    llm_calls: int = 0
+
+
+@dataclass
+class UsageUpdated(Event):
+    kind: ClassVar[str] = "usage"
+    answered: int
+    tokens: int
+    cache_hits: int = 0
+    retries: int = 0
+    fallbacks: int = 0
+    by_provider: dict[str, int] = field(default_factory=dict)
+    problems: list[str] = field(default_factory=list)
+
+
+@dataclass
+class RunFinished(Event):
+    kind: ClassVar[str] = "run_finished"
+    status: str
+    reason: str
+    seconds: float
+    requests: int
+    endpoints: int
+    report: str = ""
+    openapi: str = ""
+    report_html: str = ""
+    lessons_learned: int = 0
+    filtered: dict[str, int] = field(default_factory=dict)
+    generation_error: str = ""
+    exposure: str = ""  # "1 endpoint exposes personal data without a login: ..."
+
+
+Subscriber = Callable[[Event], object]  # return values are ignored
+
+
+class EventBus:
+    """Delivers events to subscribers, in order. Thread-safe; a failing subscriber never
+    breaks the run (its error is kept in `errors`)."""
+
+    def __init__(self) -> None:
+        self._subscribers: list[Subscriber] = []
+        self._lock = threading.Lock()
+        self.errors: list[str] = []
+        self.started = time.monotonic()
+
+    def subscribe(self, callback: Subscriber) -> None:
+        with self._lock:
+            self._subscribers.append(callback)
+
+    def emit(self, event: Event) -> None:
+        with self._lock:
+            subscribers = list(self._subscribers)
+        for callback in subscribers:
+            try:
+                callback(event)
+            except Exception as exc:  # noqa: BLE001 - interfaces must not stop the run
+                message = f"{type(exc).__name__}: {exc}"
+                if not self.errors:  # say it once, so a broken display can't flood the log
+                    _log.warning("A display or listener failed (the run continues): %s", message)
+                self.errors.append(message)

ghost_api/exitcodes.py

@@ -0,0 +1,21 @@
+"""Exit codes of the `ghost-api` command, for scripts and CI. Stable across releases."""
+
+from __future__ import annotations
+
+OK = 0  # done (a run may still have found nothing: read the JSON or the report)
+FAILED = 1  # the command failed: an error, failed checks, nothing stored to work on
+USAGE = 2  # wrong command line or setup: bad option, missing LLM key, bad project file
+NOT_ALLOWED = 3  # the site said no: robots.txt disallows it, or a bot-protection page
+INTERRUPTED = 130  # stopped with Ctrl+C
+
+_BY_STATUS = {
+    "failed": FAILED,
+    "llm-unavailable": FAILED,
+    "bot-protection": NOT_ALLOWED,
+    "interrupted": INTERRUPTED,
+}
+
+
+def for_run_status(status: str) -> int:
+    """The exit code for a run or refresh that ended with `status`."""
+    return _BY_STATUS.get(status, OK)

ghost_api/export/__init__.py

@@ -0,0 +1,71 @@
+"""Write report.md, report.html and openapi.yaml for a site (Stage 9, UI phase 5)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+
+from ghost_api.export.html import ViewerMissing, build_report_html
+from ghost_api.export.openapi import build_openapi, to_yaml
+from ghost_api.export.report import build_report
+from ghost_api.export.site_data import report_data
+from ghost_api.generator.naming import NamingPlan, apply_plan
+from ghost_api.generator.spec import ClientSpec
+from ghost_api.memory.store import Store
+
+__all__ = [
+    "Exports",
+    "write_exports",
+]
+
+
+@dataclass
+class Exports:
+    report: Path
+    openapi: Path
+    html: Path | None  # None if the report viewer is not built
+
+
+def write_exports(
+    store: Store,
+    domain: str,
+    run_id: int,
+    out_root: Path,
+    spec: ClientSpec | None = None,
+) -> Exports:
+    """Write the files into out_root/<domain>/ from what the database holds."""
+    run = store.get_run(run_id)
+    if run is None:
+        raise ValueError(f"run {run_id} not found")
+    endpoints = store.current_endpoints(domain)
+    plan = None
+    stored_plan = store.load_naming_plan(domain)
+    if stored_plan:
+        try:
+            plan = NamingPlan.model_validate_json(stored_plan)
+        except ValueError:
+            plan = None  # fall back to URL-based names
+    _, names = apply_plan(plan, endpoints, domain)
+
+    folder = out_root / domain
+    folder.mkdir(parents=True, exist_ok=True)
+    openapi_path = folder / "openapi.yaml"
+    openapi_path.write_text(
+        to_yaml(build_openapi(domain, run.start_url, endpoints, names)), encoding="utf-8"
+    )
+    report_path = folder / "report.md"
+    markdown = build_report(store, domain, run_id, spec)
+    report_path.write_text(markdown, encoding="utf-8")
+
+    generated_at = datetime.now(UTC).isoformat(timespec="seconds")
+    data = report_data(store, domain, run_id, folder, markdown, generated_at)
+    html_path: Path | None = None
+    try:
+        page = build_report_html(data)
+    except ViewerMissing:
+        pass  # a source checkout without the built viewer: report.md is still there
+    else:
+        html_path = folder / "report.html"
+        html_path.write_text(page, encoding="utf-8")
+    return Exports(report_path, openapi_path, html_path)

ghost_api/export/confidence.py

@@ -0,0 +1,45 @@
+"""How much to trust each endpoint, from evidence only.
+
+Points:
+    verified by live tests          +2
+    failed after repairs            -3
+    callable directly (replayable)  +1
+    pagination confirmed            +1
+    samples: 2-4 -> +1, 5+ -> +2
+
+High is 4 or more, Medium 2-3, Low below 2.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from ghost_api.analysis.models import Endpoint
+
+Level = Literal["High", "Medium", "Low"]
+
+
+def confidence_points(ep: Endpoint) -> int:
+    points = 0
+    if ep.status == "verified":
+        points += 2
+    elif ep.status == "failed":
+        points -= 3
+    if ep.replayable:
+        points += 1
+    if ep.pagination and ep.pagination.get("confirmed"):
+        points += 1
+    if ep.sample_count >= 5:
+        points += 2
+    elif ep.sample_count >= 2:
+        points += 1
+    return points
+
+
+def confidence(ep: Endpoint) -> Level:
+    points = confidence_points(ep)
+    if points >= 4:
+        return "High"
+    if points >= 2:
+        return "Medium"
+    return "Low"

ghost_api/export/html.py

@@ -0,0 +1,41 @@
+"""report.html: one self-contained page to share a site's results (no server needed).
+
+The page is the React report viewer built from web/ (`npm run build` writes
+viewer/report.html with all scripts and styles inlined). This module puts the site's
+data into it as JSON.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+TEMPLATE = Path(__file__).parent / "viewer" / "report.html"
+DATA_TAG = '<script id="ghost-report" type="application/json">__GHOST_REPORT__</script>'
+
+
+class ViewerMissing(RuntimeError):
+    """The report viewer was not built (run `npm run build` in web/)."""
+
+
+def embed_json(data: Any) -> str:
+    """JSON that is safe inside a <script> element: no '</script>', '<!--' or '&' tricks."""
+    text = json.dumps(data, ensure_ascii=False, separators=(",", ":"))
+    return (
+        text.replace("&", "\\u0026")
+        .replace("<", "\\u003c")
+        .replace(">", "\\u003e")
+        .replace("
", "\\u2028")
+        .replace("
", "\\u2029")
+    )
+
+
+def build_report_html(data: dict[str, Any], template: Path = TEMPLATE) -> str:
+    if not template.is_file():
+        raise ViewerMissing(f"report viewer not found at {template}")
+    page = template.read_text(encoding="utf-8")
+    if page.count(DATA_TAG) != 1:
+        raise ViewerMissing("report viewer has no data placeholder; rebuild it")
+    tag = DATA_TAG.replace("__GHOST_REPORT__", embed_json(data))
+    return page.replace(DATA_TAG, tag)

ghost_api/export/openapi.py

@@ -0,0 +1,201 @@
+"""Export the endpoint map as an OpenAPI 3.1 document (FR-30).
+
+3.1 is used because the inferred schemas are plain JSON Schema (for example
+`"type": ["null", "string"]`), which 3.1 supports and 3.0 does not.
+
+- Only read-only endpoints are exported, matching the generated client.
+- Secrets never appear: secret parameters and headers become security schemes.
+- Endpoints on other hosts (third-party APIs) get their own `servers` entry.
+- Each operation carries `x-ghost-api` with status, confidence and sample count.
+"""
+
+from __future__ import annotations
+
+import copy
+from datetime import UTC, datetime
+from typing import Any
+from urllib.parse import urlsplit
+
+import yaml
+
+from ghost_api import __version__
+from ghost_api.analysis.models import Endpoint
+from ghost_api.capture.filters import is_sensitive_name
+from ghost_api.capture.models import RawRequest
+from ghost_api.export.confidence import confidence
+from ghost_api.generator.naming import EndpointNames, pascal, snake
+from ghost_api.probe.replay import is_replayable
+
+OPENAPI_VERSION = "3.1.0"
+JSON_TYPES = {"integer", "number", "boolean", "string"}
+
+
+def build_openapi(
+    domain: str,
+    start_url: str,
+    endpoints: list[Endpoint],
+    names: dict[str, EndpointNames],
+) -> dict[str, Any]:
+    parts = urlsplit(start_url)
+    main_host = (parts.hostname or "").lower()
+    doc: dict[str, Any] = {
+        "openapi": OPENAPI_VERSION,
+        "info": {
+            "title": f"{domain} (unofficial)",
+            "version": datetime.now(UTC).strftime("%Y.%m.%d"),
+            "description": (
+                f"The internal API of {domain}, discovered by Ghost API {__version__} from the "
+                "site's own traffic. Unofficial: it can change without notice, and the site's "
+                "terms of service apply."
+            ),
+        },
+        "servers": [{"url": f"{parts.scheme}://{parts.netloc}"}],
+        "paths": {},
+        "components": {"schemas": {}, "securitySchemes": {}},
+    }
+    schemas: dict[str, Any] = doc["components"]["schemas"]
+    security_schemes: dict[str, Any] = doc["components"]["securitySchemes"]
+
+    for ep in endpoints:
+        if not _exportable(ep):
+            continue
+        n = names.get(ep.key)
+        operation: dict[str, Any] = {
+            "operationId": n.method_name if n else snake(f"{ep.method}_{ep.path_template}"),
+            "summary": n.summary if n else f"{ep.method} {ep.path_template}",
+            "parameters": [],
+            "responses": {},
+            "x-ghost-api": {
+                "status": ep.status,
+                "confidence": confidence(ep),
+                "samples": ep.sample_count,
+                "replayable": ep.replayable,
+                **({"pagination": ep.pagination} if ep.pagination else {}),
+                **({"note": ep.status_note} if ep.status_note else {}),
+            },
+        }
+        if ep.host != main_host:
+            operation["servers"] = [{"url": f"https://{ep.host}"}]
+
+        security: dict[str, list[str]] = {}
+        for pp in ep.path_params:
+            operation["parameters"].append(
+                {
+                    "name": pp.name,
+                    "in": "path",
+                    "required": True,
+                    "schema": _param_schema(pp.type, pp.examples),
+                }
+            )
+        for qp in ep.query_params:
+            if qp.sensitive or is_sensitive_name(qp.name):
+                scheme = _scheme_name(qp.name)
+                security_schemes[scheme] = {"type": "apiKey", "in": "query", "name": qp.name}
+                security[scheme] = []
+                continue
+            schema = _param_schema(qp.type, qp.examples)
+            is_page = bool(ep.pagination) and ep.pagination.get("param") == qp.name  # type: ignore[union-attr]
+            if qp.constant and qp.examples:
+                schema["default"] = _typed(qp.examples[0], qp.type)
+            elif is_page and qp.examples:
+                # Paging starts at the lowest value seen (page 1, offset 0...).
+                values = [_typed(e, qp.type) for e in qp.examples]
+                numbers = [v for v in values if isinstance(v, int | float)]
+                schema["default"] = min(numbers) if numbers else values[0]
+            operation["parameters"].append(
+                {
+                    "name": qp.name,
+                    "in": "query",
+                    "required": qp.required and not qp.constant and not is_page,
+                    "schema": schema,
+                }
+            )
+        for header in ep.required_headers:
+            if header.lower() == "cookie":
+                security_schemes["sessionCookie"] = {
+                    "type": "apiKey",
+                    "in": "cookie",
+                    "name": "session",
+                    "description": "Session cookie(s) from a logged-in browser.",
+                }
+                security["sessionCookie"] = []
+            elif header.lower() == "authorization":
+                kind = "basic" if ep.auth_type == "basic" else "bearer"
+                security_schemes[f"{kind}Auth"] = {"type": "http", "scheme": kind}
+                security[f"{kind}Auth"] = []
+            elif is_sensitive_name(header):
+                scheme = _scheme_name(header)
+                security_schemes[scheme] = {"type": "apiKey", "in": "header", "name": header}
+                security[scheme] = []
+            else:
+                operation["parameters"].append(
+                    {"name": header, "in": "header", "required": True, "schema": {"type": "string"}}
+                )
+        if security:
+            operation["security"] = [security]
+        if not operation["parameters"]:
+            del operation["parameters"]
+
+        if ep.request_schema is not None:
+            operation["requestBody"] = {
+                "required": True,
+                "content": {"application/json": {"schema": _clean(ep.request_schema)}},
+            }
+        model = n.response_model if n else pascal(f"{ep.method}_{ep.path_template}") + "Response"
+        if ep.response_schema is not None:
+            schemas[model] = _clean(ep.response_schema)
+            content = {"application/json": {"schema": {"$ref": f"#/components/schemas/{model}"}}}
+            operation["responses"]["200"] = {"description": "Success", "content": content}
+        else:
+            operation["responses"]["200"] = {"description": "Success (no JSON sample captured)"}
+
+        path_item = doc["paths"].setdefault(ep.path_template, {})
+        path_item[ep.method.lower()] = operation
+
+    if not schemas:
+        del doc["components"]["schemas"]
+    if not security_schemes:
+        del doc["components"]["securitySchemes"]
+    if not doc["components"]:
+        del doc["components"]
+    return doc
+
+
+def to_yaml(doc: dict[str, Any]) -> str:
+    return yaml.safe_dump(doc, sort_keys=False, allow_unicode=True, width=100)
+
+
+def _exportable(ep: Endpoint) -> bool:
+    probe = RawRequest(ep.method, f"https://{ep.host}{ep.path_template}", {}, None)
+    return is_replayable(probe) and ep.replayable is not False
+
+
+def _param_schema(json_type: str, examples: list[str]) -> dict[str, Any]:
+    schema: dict[str, Any] = {"type": json_type if json_type in JSON_TYPES else "string"}
+    if examples:
+        schema["examples"] = [_typed(e, json_type) for e in examples[:3]]
+    return schema
+
+
+def _typed(value: str, json_type: str) -> Any:
+    try:
+        if json_type == "integer":
+            return int(value)
+        if json_type == "number":
+            return float(value)
+    except ValueError:
+        return value
+    if json_type == "boolean":
+        return value.lower() == "true"
+    return value
+
+
+def _scheme_name(wire: str) -> str:
+    return pascal(wire)[:1].lower() + pascal(wire)[1:]
+
+
+def _clean(schema: dict[str, Any]) -> dict[str, Any]:
+    """A copy without genson's '$schema' marker (not allowed inside OpenAPI)."""
+    out = copy.deepcopy(schema)
+    out.pop("$schema", None)
+    return out