ghost-api-cli 0.2.0__py3-none-any.whl

ghost_api/export/report.py

@@ -0,0 +1,227 @@
+"""Write report.md for a run (FR-47, FR-48), from the database only.
+
+Sections: summary and usage, endpoints (status, confidence, samples), hypotheses with
+evidence, the agent's reasoning log, known issues, and lessons for next time.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from ghost_api import __version__
+from ghost_api.analysis.exposure import Exposure, exposures, summary_line
+from ghost_api.analysis.models import Endpoint
+from ghost_api.capture.personal import LABEL
+from ghost_api.export.confidence import confidence
+from ghost_api.generator.spec import ClientSpec
+from ghost_api.memory.store import RunRow, Store
+from ghost_api.site import site_host
+
+RESULT_MARK = {"confirmed": "confirmed", "rejected": "rejected", "inconclusive": "inconclusive"}
+
+
+def _network_section(net: dict[str, Any]) -> list[str]:
+    """What the run sent where, and which rules applied."""
+    rows = [
+        ("Site", f"`{net['target']}`" + (" (local target: private addresses allowed)"
+                                         if net.get("local_target") else "")),
+        ("robots.txt", _md(str(net.get("robots", "")))),
+        ("Rate limit", f"at most {net.get('rate_limit', 0)} request(s) per second per site"),
+        ("API calls the pages made", str(net.get("api_calls_seen", 0))),
+        ("Direct calls Ghost API sent", f"{net.get('direct_calls', 0)} (read-only endpoint tests)"),
+        ("Other API hosts", ", ".join(f"`{h}`" for h in net.get("third_party_hosts", []))
+         or "none"),
+        ("Saved login used", "yes" if net.get("saved_login") else "no"),
+        ("LLM calls", str(net.get("llm_calls", 0))),
+    ]  # fmt: skip
+    out = ["| | |", "|---|---|", *(f"| {k} | {v} |" for k, v in rows)]
+    blocked = net.get("blocked") or []
+    out += ["", f"**Blocked by the rules ({len(blocked)}):**" if blocked else
+            "Nothing was blocked by the network rules."]  # fmt: skip
+    out += [f"- {_md(b)}" for b in blocked[:30]]
+    return out
+
+
+def build_report(store: Store, domain: str, run_id: int, spec: ClientSpec | None = None) -> str:
+    run = store.get_run(run_id)
+    if run is None:
+        raise ValueError(f"run {run_id} not found")
+    endpoints = store.current_endpoints(domain)
+    hypotheses = store.hypotheses_for_run(run_id)
+    steps = store.steps_for_run(run_id)
+    lessons = store.lessons_for(domain, limit=10)
+
+    out: list[str] = [
+        f"# Ghost API report: {domain}",
+        "",
+        f"Generated by Ghost API {__version__} for run #{run.id}. Goal: _{_md(run.goal)}_",
+        "",
+        "## Summary",
+        "",
+        *_summary(run, endpoints, spec),
+        "",
+        "## Endpoints",
+        "",
+    ]
+    out += _endpoint_table(endpoints, domain) if endpoints else ["No endpoints were found."]
+    exposed = exposures(endpoints, store.requests_for_domain(domain))
+    if exposed:
+        out += ["", "## Personal data exposed", "", *_exposure_section(exposed)]
+    network = [e for e in store.events_for_run(run_id) if e.get("type") == "network"]
+    if network:
+        out += ["", "## Network and permissions", "", *_network_section(network[-1])]
+    out += ["", "## Hypotheses tested", ""]
+    out += _hypotheses(hypotheses) or ["No direct-call tests in this run."]
+    out += ["", "## How the agent explored", ""]
+    out += _steps(steps) or ["The agent was not used in this run."]
+    out += ["", "## Known issues", ""]
+    out += _issues(endpoints, hypotheses, spec) or ["None found."]
+    if lessons:
+        out += ["", "## Lessons remembered for next time", ""]
+        out += [f"- {_md(lesson)}" for lesson in lessons]
+    out += ["", "---", "", _CONFIDENCE_NOTE, ""]
+    return "\n".join(out)
+
+
+def _exposure_section(exposed: list[Exposure]) -> list[str]:
+    lines = [
+        f"**{summary_line(exposed)}.** Ghost API hid these values: they are not in its "
+        "database, this report or the generated code. Examples below are masked.",
+        "",
+        "| Endpoint | Without login | Field | What | Seen | Example |",
+        "|---|---|---|---|---|---|",
+    ]
+    for e in exposed:
+        for f in e.fields:
+            lines.append(
+                f"| `{_md(e.label)}` | {'yes' if e.open else 'no'} | `{_md(f['path'])}` "
+                f"| {LABEL.get(f['kind'], f['kind'])} ({f['severity']}) | {f['count']} "
+                f"| `{_md(f['example'])}` |"
+            )
+    lines += [
+        "",
+        "_What to do: if this is your site, stop returning these fields (or require a "
+        "login). If it isn't, don't collect or share them, and consider telling the site "
+        "owner privately (responsible disclosure)._",
+    ]
+    return lines
+
+
+_CONFIDENCE_NOTE = (
+    "_Confidence: verified by live tests +2, callable directly +1, pagination confirmed +1, "
+    "2-4 samples +1 or 5+ samples +2, failed -3. High is 4 or more, Medium 2-3, Low below 2._"
+)
+
+
+def _summary(run: RunRow, endpoints: list[Endpoint], spec: ClientSpec | None) -> list[str]:
+    counts = {
+        s: sum(e.status == s for e in endpoints) for s in ("verified", "unverified", "failed")
+    }
+    outcome = run.status
+    if run.status == "running" and run.finished_at is None:
+        outcome = "did not finish (stopped unexpectedly)"
+    lines = [
+        f"- **Outcome:** {outcome}",
+        f"- **Endpoints:** {len(endpoints)} ({counts['verified']} verified, "
+        f"{counts['unverified']} unverified, {counts['failed']} failed)",
+    ]
+    if spec is not None:
+        methods = ", ".join(f"`{m.name}()`" for m in spec.methods)
+        lines.append(f"- **Client:** `{spec.class_name}` with {methods}")
+    duration = _duration(run.started_at, run.finished_at)
+    lines.append(
+        f"- **Usage:** {run.llm_calls} LLM call{_s(run.llm_calls)}, {run.tokens:,} tokens, "
+        f"{run.request_count} captured request{_s(run.request_count)}"
+        f"{', ' + duration if duration else ''}, $0.00 (free tiers)"
+    )
+    return lines
+
+
+def _endpoint_table(endpoints: list[Endpoint], domain: str) -> list[str]:
+    rows = [
+        "| Endpoint | Status | Confidence | Samples | Auth | Paging | Notes |",
+        "|---|---|---|---|---|---|---|",
+    ]
+    host = site_host(domain)
+    for ep in endpoints:
+        label = ep.label if ep.host == host else f"{ep.method} {ep.host}{ep.path_template}"
+        paging = "-"
+        if ep.pagination:
+            mark = "" if ep.pagination.get("confirmed") else " (unconfirmed)"
+            paging = f"{ep.pagination['style']}: `{ep.pagination['param']}`{mark}"
+        rows.append(
+            f"| `{_md(label)}` | {ep.status} | {confidence(ep)} | {ep.sample_count} | "
+            f"{ep.auth_type or '-'} | {paging} | {_md(ep.status_note) or '-'} |"
+        )
+    return rows
+
+
+def _hypotheses(hypotheses: list[dict[str, Any]]) -> list[str]:
+    lines: list[str] = []
+    current = None
+    for h in hypotheses:
+        if h["endpoint_key"] != current:
+            current = h["endpoint_key"]
+            if lines:
+                lines.append("")
+            lines += [f"**`{_md(str(current))}`**", ""]
+        result = RESULT_MARK[str(h["result"])]
+        lines.append(f"- {result}: {_md(str(h['text']))} ({_md(str(h['evidence']))})")
+    return lines
+
+
+def _steps(steps: list[dict[str, Any]]) -> list[str]:
+    lines = []
+    for s in steps:
+        args = ", ".join(f"{k}={v!r}" for k, v in dict(s["args"]).items())
+        lines.append(f"{s['step_no']}. `{s['action']}({_md(args)})`: {_md(str(s['thought']))}")
+        lines.append(f"   - Result: {_md(str(s['result']))}")
+    return lines
+
+
+def _issues(
+    endpoints: list[Endpoint], hypotheses: list[dict[str, Any]], spec: ClientSpec | None
+) -> list[str]:
+    issues = []
+    for ep in endpoints:
+        if ep.status in ("failed", "unverified"):
+            issues.append(
+                f"- `{_md(ep.label)}` is {ep.status}: {_md(ep.status_note) or 'not tested'}"
+            )
+        elif ep.status == "discovered":
+            issues.append(
+                f"- `{_md(ep.label)}` has not been verified yet "
+                "(run `ghost-api verify` or `ghost-api refresh`)"
+            )
+        if ep.replayable is False:
+            issues.append(f"- `{_md(ep.label)}` cannot be called outside the browser")
+    for h in hypotheses:
+        if h["result"] == "inconclusive":
+            issues.append(f"- Inconclusive: {_md(str(h['text']))} ({_md(str(h['evidence']))})")
+    if spec is not None:
+        issues += [f"- Not in the client: {_md(s)}" for s in spec.skipped]
+        for cred in spec.credentials:
+            issues.append(f"- The client needs `{cred.arg}`: set `{cred.env}` or pass it in")
+    return issues
+
+
+def _duration(started: str, finished: str | None) -> str:
+    if not finished:
+        return ""
+    try:
+        seconds = (
+            datetime.fromisoformat(finished) - datetime.fromisoformat(started)
+        ).total_seconds()
+    except ValueError:
+        return ""
+    return f"{seconds:.0f}s"
+
+
+def _s(count: int) -> str:
+    return "" if count == 1 else "s"
+
+
+def _md(text: str) -> str:
+    """Safe inside a Markdown table cell or list item."""
+    return text.replace("|", "\\|").replace("\n", " ").strip()

ghost_api/export/site_data.py

@@ -0,0 +1,196 @@
+"""What is stored about a site, as plain JSON for the dashboard and report.html.
+
+The dashboard's API and the shareable report use these same shapes, so the React
+components that show them work in both places (see web/src/types.ts).
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any
+
+from ghost_api import __version__
+from ghost_api.analysis.analyzer import endpoint_keys
+from ghost_api.analysis.exposure import exposures
+from ghost_api.analysis.models import Endpoint
+from ghost_api.analysis.schema import field_paths, top_level_fields
+from ghost_api.export.confidence import confidence
+from ghost_api.generator.naming import apply_plan
+from ghost_api.memory.store import RunRow, Store
+from ghost_api.runtime import inputs_for, is_callable, load_plan
+from ghost_api.site import site_host
+
+SAFE_NAME = re.compile(r"^[A-Za-z0-9_.-]+$")
+
+# Code and spec files embedded in report.html (bigger files are listed, not embedded).
+MAX_EMBEDDED_FILE = 300_000
+EMBEDDED_KINDS = ("client", "models", "tests", "openapi")
+# Left out of the shared report: request events (many, with full URLs), endpoint lists
+# (the report has its own, with sensitive examples removed) and screenshots (not embedded).
+SKIPPED_EVENTS = {"request", "endpoints", "screenshot"}
+
+
+def display_label(ep: Endpoint, site: str) -> str:
+    """The label as the site sees it; `site` may be a host or host/section."""
+    host = site_host(site)
+    return ep.label if ep.host == host else f"{ep.method} {ep.host}{ep.path_template}"
+
+
+def endpoint_summary(ep: Endpoint, site: str) -> dict[str, Any]:
+    """Everything an interface needs to show an endpoint, as plain JSON."""
+    host = site_host(site)
+    params = [
+        {
+            "name": p.name,
+            "type": p.type,
+            "location": "path",
+            "kind": "required",
+            "examples": p.examples[:3],
+            "sensitive": False,
+        }
+        for p in ep.path_params
+    ] + [
+        {
+            "name": p.name,
+            "type": p.type,
+            "location": "query",
+            "kind": p.kind,
+            "examples": p.examples[:3],
+            "sensitive": p.sensitive,
+        }
+        for p in ep.query_params
+    ]
+    return {
+        "key": ep.key,
+        "label": display_label(ep, site),
+        "method": ep.method,
+        "host": ep.host,
+        "path": ep.path_template,
+        "third_party": ep.host != host,
+        "params": params,
+        "samples": ep.sample_count,
+        "statuses": ep.statuses,
+        "status": ep.status,
+        "status_note": ep.status_note,
+        "replayable": ep.replayable,
+        "auth_type": ep.auth_type,
+        "required_headers": ep.required_headers,
+        "pagination": ep.pagination,
+        "confidence": confidence(ep),
+        "fields": top_level_fields(ep.response_schema),
+        "field_paths": field_paths(ep.response_schema, limit=40),
+        "has_request_body": ep.request_schema is not None,
+    }
+
+
+def endpoint_details(store: Store, domain: str) -> list[dict[str, Any]]:
+    """Summaries plus schemas and example URLs, from the latest run that found endpoints."""
+    run_id = store.latest_endpoint_run(domain)
+    if run_id is None:
+        return []
+    details = []
+    endpoints = store.current_endpoints(domain)
+    requests = store.requests_for_domain(domain)
+    found = exposures(endpoints, requests)
+    exposed = {e.endpoint_key: e.to_dict() for e in found}
+    samples: dict[str, list[Any]] = {}
+    for key, req in zip(endpoint_keys(requests), requests, strict=False):
+        samples.setdefault(key, []).append(req)
+    allowed = store.allowed_endpoints(domain)
+    _, names = apply_plan(load_plan(store.load_naming_plan(domain)), endpoints, domain)
+    for ep in endpoints:
+        summary = endpoint_summary(ep, domain)
+        summary.update(
+            request_schema=ep.request_schema,
+            response_schema=ep.response_schema,
+            example_urls=ep.example_urls,
+            exposure=exposed.get(ep.key),
+            # For Try it / Get data: can Ghost API call it, and with which inputs.
+            callable=is_callable(ep, ep.key in allowed),
+            allowed=ep.key in allowed,
+            operation=names[ep.key].method_name,  # for fetch / watch / serve / mcp
+            inputs=[asdict(i) for i in inputs_for(ep, samples.get(ep.key, []))],
+        )
+        details.append(summary)
+    return details
+
+
+def run_dict(r: RunRow) -> dict[str, Any]:
+    return {
+        "id": r.id,
+        "domain": r.domain,
+        "start_url": r.start_url,
+        "goal": r.goal,
+        "status": r.status,
+        "started_at": r.started_at,
+        "finished_at": r.finished_at,
+        "requests": r.request_count,
+        "llm_calls": r.llm_calls,
+        "tokens": r.tokens,
+    }
+
+
+def file_kind(name: str) -> str:
+    if name.startswith("test_") or name.endswith(".test.ts"):
+        return "tests"
+    if name.endswith(("_client.py", "_client.ts")):
+        return "client"
+    if name == "models.py":
+        return "models"
+    if name.endswith((".yaml", ".yml")):
+        return "openapi"
+    if name.startswith("report."):
+        return "report"
+    return "other"
+
+
+def site_files(folder: Path) -> list[dict[str, Any]]:
+    if not folder.is_dir():
+        return []
+    return [
+        {"name": f.name, "size": f.stat().st_size, "kind": file_kind(f.name)}
+        for f in sorted(folder.iterdir())
+        if f.is_file()
+    ]
+
+
+def _public_endpoint(ep: dict[str, Any]) -> dict[str, Any]:
+    """Drop example values of sensitive parameters before sharing."""
+    params = [{**p, "examples": []} if p["sensitive"] else p for p in ep["params"]]
+    return {**ep, "params": params}
+
+
+def report_data(
+    store: Store, domain: str, run_id: int, folder: Path, report_md: str, generated_at: str
+) -> dict[str, Any]:
+    """Everything report.html shows. It is meant to be shared, so it holds no secrets:
+    stored requests are already redacted, sensitive examples are dropped, and request
+    events (full URLs) are left out."""
+    run = store.get_run(run_id)
+    if run is None:
+        raise ValueError(f"run {run_id} not found")
+    files = []
+    for f in site_files(folder):
+        if f["kind"] not in EMBEDDED_KINDS:
+            continue
+        path = folder / f["name"]
+        content = (
+            path.read_text(encoding="utf-8", errors="replace")
+            if f["size"] <= MAX_EMBEDDED_FILE
+            else None
+        )
+        files.append({**f, "content": content})
+    events = [e for e in store.events_for_run(run_id) if e.get("type") not in SKIPPED_EVENTS]
+    return {
+        "version": __version__,
+        "generated_at": generated_at,
+        "domain": domain,
+        "run": run_dict(run),
+        "endpoints": [_public_endpoint(e) for e in endpoint_details(store, domain)],
+        "events": events,
+        "lessons": store.lessons_for(domain, limit=20),
+        "files": files,
+        "report_md": report_md,
+    }

ghost_api/export/tabular.py

@@ -0,0 +1,88 @@
+"""Write rows of records as CSV, Excel or JSON (for `ghost-api fetch` and the dashboard).
+
+Rows come from `runtime.flatten`, so nested fields are already dotted columns
+("venue.name"). Columns keep the order they first appear in.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+from pathlib import Path
+from typing import Any, Literal
+
+from ghost_api.runtime import flatten
+
+Format = Literal["csv", "xlsx", "json"]
+FORMATS: tuple[Format, ...] = ("csv", "xlsx", "json")
+MEDIA_TYPES = {
+    "csv": "text/csv; charset=utf-8",
+    "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "json": "application/json",
+}
+EXCEL_MAX_CELL = 32_767  # Excel refuses longer cell text
+
+
+def columns(rows: list[dict[str, Any]]) -> list[str]:
+    seen: dict[str, None] = {}
+    for row in rows:
+        for key in row:
+            seen.setdefault(key, None)
+    return list(seen)
+
+
+def to_bytes(records: list[Any], fmt: Format) -> bytes:
+    """The records as a file's contents. JSON keeps the original nesting."""
+    if fmt == "json":
+        return json.dumps(records, ensure_ascii=False, indent=2).encode("utf-8")
+    rows = [flatten(r) for r in records]
+    cols = columns(rows)
+    if fmt == "csv":
+        buffer = io.StringIO()
+        writer = csv.DictWriter(buffer, fieldnames=cols, extrasaction="ignore")
+        writer.writeheader()
+        writer.writerows({k: _safe_text(v) for k, v in row.items()} for row in rows)
+        return buffer.getvalue().encode("utf-8-sig")  # the BOM makes Excel read UTF-8
+    return _xlsx(rows, cols)
+
+
+def write(records: list[Any], path: Path, fmt: Format) -> Path:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(to_bytes(records, fmt))
+    return path
+
+
+def _xlsx(rows: list[dict[str, Any]], cols: list[str]) -> bytes:
+    from openpyxl import Workbook
+    from openpyxl.styles import Font
+
+    book = Workbook()
+    sheet = book.active
+    assert sheet is not None
+    sheet.title = "data"
+    sheet.append(cols)
+    for cell in sheet[1]:
+        cell.font = Font(bold=True)
+    for row in rows:
+        sheet.append([_cell(row.get(c)) for c in cols])
+    sheet.freeze_panes = "A2"
+    for i, col in enumerate(cols, start=1):
+        width = max([len(col)] + [len(str(r.get(col) or "")) for r in rows[:200]])
+        sheet.column_dimensions[sheet.cell(1, i).column_letter].width = min(max(width, 8), 60)
+    buffer = io.BytesIO()
+    book.save(buffer)
+    return buffer.getvalue()
+
+
+def _cell(value: Any) -> Any:
+    safe = _safe_text(value)
+    return safe[:EXCEL_MAX_CELL] if isinstance(safe, str) else safe
+
+
+def _safe_text(value: Any) -> Any:
+    """Site data must never become a spreadsheet formula ("=HYPERLINK(...)")."""
+    if value is None or isinstance(value, bool | int | float):
+        return value
+    text = str(value)
+    return "'" + text if text[:1] in ("=", "+", "-", "@", "\t", "\r") else text