ghost-api-cli 0.2.0__py3-none-any.whl

ghost_api/browser/snapshot.py

@@ -0,0 +1,221 @@
+"""A compact, LLM-friendly view of the current page (FR-8, NFR-6).
+
+JavaScript in the page lists visible interactive elements and tags each with a
+`data-ghost-id` attribute so the agent can refer to it by number. Python then removes
+elements the agent must never use (FR-10, FR-11, NFR-21) and caps the list size.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import urlsplit
+
+from ghost_api.capture.filters import is_same_site
+from ghost_api.net import RobotsPolicy
+
+MAX_ELEMENTS = 60
+MAX_LABEL = 60
+MAX_TEXT = 400
+
+# Labels or URLs of actions that change server state, spend money or end the session.
+DESTRUCTIVE_RE = re.compile(
+    r"\b(delete|remove|destroy|erase|buy|purchase|checkout|check out|pay|payment|place order|"
+    r"order now|add to (cart|bag|basket)|subscribe|unsubscribe|log ?out|sign ?out|logout|"
+    r"signout|deactivate|close account|cancel (my )?(account|subscription|order)|donate|"
+    r"report|block|unfollow|follow|upvote|downvote|vote|flag|hide|reply|comment|send|share|"
+    r"invite|transfer|withdraw|confirm)\b",
+    re.I,
+)
+# Input types the agent must not fill in.
+FORBIDDEN_INPUT_TYPES = {"password", "file", "hidden"}
+
+SNAPSHOT_JS = r"""
+({ MAX_LABEL, MAX_TEXT }) => {
+  document.querySelectorAll('[data-ghost-id]').forEach(e => e.removeAttribute('data-ghost-id'));
+  const selector = [
+    'a[href]', 'button', 'input:not([type=hidden])', 'select', 'textarea', 'summary',
+    '[role=button]', '[role=link]', '[role=tab]', '[role=menuitem]', '[role=option]',
+    '[role=checkbox]', '[role=switch]', '[role=searchbox]', '[role=combobox]', '[onclick]',
+    '[contenteditable=true]'
+  ].join(',');
+  const clean = s => (s || '').replace(/\s+/g, ' ').trim();
+  const visible = el => {
+    const r = el.getBoundingClientRect();
+    if (r.width < 2 || r.height < 2) return false;
+    const st = getComputedStyle(el);
+    return st.visibility !== 'hidden' && st.display !== 'none' && Number(st.opacity) > 0.05;
+  };
+  const labelOf = el => {
+    const parts = [
+      el.getAttribute('aria-label'), el.innerText, el.getAttribute('placeholder'),
+      el.getAttribute('title'), el.getAttribute('alt'), el.getAttribute('name'),
+      el.tagName === 'INPUT' && ['submit', 'button'].includes(el.type) ? el.value : '',
+      el.querySelector && el.querySelector('img[alt]') ? el.querySelector('img[alt]').alt : ''
+    ];
+    for (const p of parts) { const c = clean(p); if (c) return c.slice(0, MAX_LABEL); }
+    return '';
+  };
+  const out = [];
+  let n = 0;
+  for (const el of document.querySelectorAll(selector)) {
+    if (!visible(el) || el.disabled) continue;
+    // Open links in this tab so the recorder sees their traffic.
+    if (el.tagName === 'A' && el.target) el.target = '_self';
+    const r = el.getBoundingClientRect();
+    n += 1;
+    el.setAttribute('data-ghost-id', String(n));
+    out.push({
+      id: n,
+      tag: el.tagName.toLowerCase(),
+      role: el.getAttribute('role') || '',
+      type: (el.getAttribute('type') || '').toLowerCase(),
+      label: labelOf(el),
+      href: el.tagName === 'A' ? el.href : '',
+      download: el.hasAttribute('download'),
+      inViewport: r.bottom > 0 && r.top < window.innerHeight,
+      top: r.top + window.scrollY
+    });
+  }
+  const doc = document.documentElement;
+  const maxScroll = doc.scrollHeight - window.innerHeight;
+  const scroll = maxScroll <= 5 ? 'no-scroll'
+    : window.scrollY <= 5 ? 'top'
+    : window.scrollY >= maxScroll - 5 ? 'bottom' : 'middle';
+  const main = document.querySelector('main, [role=main], #content, article') || document.body;
+  return {
+    url: location.href,
+    title: document.title,
+    scroll,
+    text: clean(main ? main.innerText : '').slice(0, MAX_TEXT),
+    elements: out
+  };
+}
+"""
+SNAPSHOT_ARGS = {"MAX_LABEL": MAX_LABEL, "MAX_TEXT": MAX_TEXT}
+
+
+@dataclass
+class Element:
+    id: int
+    kind: str  # link, button, input:text, select, ...
+    label: str
+    href: str = ""
+    in_viewport: bool = False
+
+    def describe(self) -> str:
+        text = (
+            f'[{self.id}] {self.kind} "{self.label}"' if self.label else f"[{self.id}] {self.kind}"
+        )
+        if self.href:
+            text += f" -> {_short_href(self.href)}"
+        return text
+
+
+@dataclass
+class PageSnapshot:
+    url: str
+    title: str
+    scroll: str
+    text: str
+    elements: list[Element] = field(default_factory=list)
+    blocked: int = 0  # elements removed for safety
+    has_password_field: bool = False  # likely a login form
+    hidden: int = 0  # elements left out to keep the prompt small
+
+    def element(self, element_id: int) -> Element | None:
+        return next((e for e in self.elements if e.id == element_id), None)
+
+
+def kind_of(raw: dict[str, Any]) -> str:
+    tag = str(raw.get("tag") or "")
+    role = str(raw.get("role") or "")
+    typ = str(raw.get("type") or "")
+    if role:
+        return role
+    if tag == "a":
+        return "link"
+    if tag == "input":
+        return f"input:{typ or 'text'}"
+    if tag in ("textarea", "select", "summary", "button"):
+        return tag
+    return "clickable"
+
+
+def block_reason(
+    raw: dict[str, Any], target_host: str, robots: RobotsPolicy | None = None
+) -> str | None:
+    """Why the agent must not use this element, or None if it's allowed."""
+    label = str(raw.get("label") or "")
+    href = str(raw.get("href") or "")
+    kind = kind_of(raw)
+    if raw.get("download"):
+        return "download"
+    if kind.startswith("input:") and kind.split(":", 1)[1] in FORBIDDEN_INPUT_TYPES:
+        return "sensitive input"
+    if DESTRUCTIVE_RE.search(label) or (href and DESTRUCTIVE_RE.search(urlsplit(href).path)):
+        return "destructive"
+    if href:
+        scheme = urlsplit(href).scheme
+        if scheme in ("mailto", "tel", "sms", "data", "blob"):
+            return f"{scheme} link"
+        if scheme in ("http", "https"):
+            if not is_same_site(href, target_host):
+                return "off-site"
+            if robots is not None and not robots.allows(href):
+                return "robots.txt"
+    return None
+
+
+def build_snapshot(
+    raw: dict[str, Any], target_host: str, robots: RobotsPolicy | None = None
+) -> PageSnapshot:
+    """Turn the page script's output into a safe, size-capped snapshot."""
+    kept: list[Element] = []
+    seen: set[tuple[str, str, str]] = set()
+    blocked = 0
+    password = False
+    for item in raw.get("elements", []):
+        password = password or kind_of(item) == "input:password"
+        if block_reason(item, target_host, robots):
+            blocked += 1
+            continue
+        el = Element(
+            id=int(item["id"]),
+            kind=kind_of(item),
+            label=str(item.get("label") or ""),
+            href=str(item.get("href") or ""),
+            in_viewport=bool(item.get("inViewport")),
+        )
+        key = (el.kind, el.label, el.href)
+        if key in seen:  # e.g. twenty identical "Read more" links
+            continue
+        seen.add(key)
+        kept.append(el)
+    # Elements on screen first; the rest in page order.
+    kept.sort(key=lambda e: not e.in_viewport)
+    shown = kept[:MAX_ELEMENTS]
+    return PageSnapshot(
+        url=str(raw.get("url", "")),
+        title=str(raw.get("title", ""))[:120],
+        scroll=str(raw.get("scroll", "")),
+        text=str(raw.get("text", "")),
+        elements=shown,
+        blocked=blocked,
+        hidden=len(kept) - len(shown),
+        has_password_field=password,
+    )
+
+
+def _short_href(href: str) -> str:
+    parts = urlsplit(href)
+    if parts.scheme == "javascript":
+        return "(script)"
+    path = parts.path or "/"
+    text = (
+        path
+        + (f"?{parts.query}" if parts.query else "")
+        + (f"#{parts.fragment}" if parts.fragment else "")
+    )
+    return text if len(text) <= 70 else text[:67] + "..."

ghost_api/capture/filters.py

@@ -0,0 +1,255 @@
+"""Decide which captured requests are API traffic, and strip secrets out of them.
+
+Everything here is plain, deterministic code. Redaction runs before anything is stored,
+logged or sent to an LLM (FR-16, NFR-13).
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from typing import Any
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+REDACTED = "[REDACTED]"
+
+# Resource types Playwright reports for script-initiated requests.
+API_RESOURCE_TYPES = {"xhr", "fetch"}
+
+# Third-party analytics / tracking / monitoring hosts. Matched as a domain suffix.
+NOISE_DOMAINS = (
+    "google-analytics.com",
+    "analytics.google.com",
+    "googletagmanager.com",
+    "doubleclick.net",
+    "googlesyndication.com",
+    "googleadservices.com",
+    "facebook.net",
+    "facebook.com",
+    "connect.facebook.net",
+    "segment.io",
+    "segment.com",
+    "mixpanel.com",
+    "amplitude.com",
+    "hotjar.com",
+    "hotjar.io",
+    "clarity.ms",
+    "fullstory.com",
+    "sentry.io",
+    "nr-data.net",
+    "newrelic.com",
+    "datadoghq.com",
+    "browser-intake-datadoghq.com",
+    "intercom.io",
+    "launchdarkly.com",
+    "optimizely.com",
+    "bing.com",
+    "linkedin.com",
+    "tiktok.com",
+    "twitter.com",
+    "ads-twitter.com",
+)
+
+# Third-party infrastructure that returns JSON but is never the site's data API:
+# CDNs, captchas, consent banners, payment widgets, video players, push services.
+INFRA_DOMAINS = (
+    "gstatic.com", "fonts.googleapis.com", "recaptcha.net", "hcaptcha.com",
+    "challenges.cloudflare.com",
+    "cloudflareinsights.com", "jsdelivr.net", "unpkg.com", "cdnjs.cloudflare.com",
+    "stripe.com", "stripe.network", "paypal.com", "braintreegateway.com", "youtube.com",
+    "ytimg.com", "vimeo.com", "onesignal.com", "pusher.com", "cookielaw.org", "onetrust.com",
+    "cookiebot.com", "usercentrics.eu", "trustarc.com", "quantcast.com", "gravatar.com",
+    "wp.com", "zendesk.com", "hubspot.com", "hs-scripts.com", "typekit.net", "fontawesome.com",
+)  # fmt: skip
+
+# Subdomains that carry tracking or monitoring, even on the target's own domain.
+NOISE_SUBDOMAINS = {
+    "telemetry", "analytics", "metrics", "stats", "tracking", "track", "log", "logs", "rum",
+    "beacon", "pixel", "collector", "sentry",
+}  # fmt: skip
+
+# Second-level public suffixes, so "shop.example.co.uk" maps to "example.co.uk".
+_TWO_PART_SUFFIXES = {
+    "co.uk", "org.uk", "ac.uk", "gov.uk", "com.au", "net.au", "org.au", "co.in", "net.in",
+    "org.in", "co.jp", "ne.jp", "com.br", "com.cn", "com.mx", "co.nz", "co.za", "com.sg",
+}  # fmt: skip
+
+_SENSITIVE_WORDS = {
+    "token", "secret", "password", "passwd", "pwd", "apikey", "csrf", "xsrf", "session",
+    "sessionid", "sessid", "sid", "auth", "authorization", "signature", "sig", "cookie",
+    "jwt", "otp", "bearer", "credential", "credentials",
+}  # fmt: skip
+# Run-together names like PHPSESSID, csrftoken, authtoken.
+_SENSITIVE_SUFFIXES = ("sessid", "sessionid", "token", "secret", "password", "apikey")
+# "key" alone is often a legitimate ID field, so only these combinations count.
+_SENSITIVE_KEY_PREFIXES = {"api", "access", "private", "secret", "client", "auth", "signing"}
+
+_IP_RE = re.compile(r"\d{1,3}(\.\d{1,3}){3}")
+
+# Beacon-style endpoints that sites host on their own domain.
+_FIRST_PARTY_NOISE_RE = re.compile(r"/(collect|beacon|telemetry|pixel|__track|rum)(/|$)", re.I)
+# Health checks and liveness probes (first- or third-party): never a data API.
+_HEALTH_RE = re.compile(
+    r"/(isalive|is_alive|health|healthz|healthcheck|ping|heartbeat|livez|readyz)/?$", re.I
+)
+
+_JWT_RE = re.compile(r"eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}")
+_BEARER_RE = re.compile(r"(?i)\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]{8,}")
+_CAMEL_RE = re.compile(r"(?<=[a-z0-9])(?=[A-Z])")
+
+
+def site_of(host: str) -> str:
+    """Approximate the registrable domain: 'api.shop.example.com' -> 'example.com'."""
+    host = host.lower().rstrip(".").split(":")[0]
+    if _IP_RE.fullmatch(host):
+        return host
+    parts = host.split(".")
+    if len(parts) <= 2:
+        return host
+    last_two = ".".join(parts[-2:])
+    if last_two in _TWO_PART_SUFFIXES:
+        return ".".join(parts[-3:])
+    return last_two
+
+
+def is_same_site(url: str, target_host: str) -> bool:
+    """True if the URL is on the target's domain or one of its subdomains (FR-10)."""
+    host = (urlsplit(url).hostname or "").lower()
+    return bool(host) and site_of(host) == site_of(target_host)
+
+
+def is_noise_domain(url: str) -> bool:
+    host = (urlsplit(url).hostname or "").lower()
+    if host.split(".", 1)[0] in NOISE_SUBDOMAINS:
+        return True
+    return any(host == d or host.endswith("." + d) for d in NOISE_DOMAINS)
+
+
+def is_infra_domain(url: str) -> bool:
+    host = (urlsplit(url).hostname or "").lower()
+    return any(host == d or host.endswith("." + d) for d in INFRA_DOMAINS)
+
+
+def is_third_party(url: str, target_host: str) -> bool:
+    return not is_same_site(url, target_host)
+
+
+def is_json_content_type(content_type: str | None) -> bool:
+    return bool(content_type) and "json" in content_type.lower()  # type: ignore[union-attr]
+
+
+# Some sites put a guard before their JSON so another site's <script> tag can't read it
+# (Google: ")]}'"; older sites: "while(1);" or "for(;;);"). Browsers' own code strips it.
+XSSI_PREFIXES = (")]}'", "while(1);", "for(;;);")
+
+
+def strip_xssi(text: str) -> str:
+    """Remove an anti-hijacking prefix from a JSON response, if there is one."""
+    stripped = text.lstrip()
+    for prefix in XSSI_PREFIXES:
+        if stripped.startswith(prefix):
+            return stripped[len(prefix) :].lstrip(",").lstrip()
+    return text
+
+
+def parse_response_json(text: str) -> Any:
+    """Parse a JSON response body, allowing an anti-hijacking prefix. Raises ValueError."""
+    return json.loads(strip_xssi(text))
+
+
+def classify(resource_type: str, url: str, target_host: str) -> str | None:
+    """Return None if this request should be kept, otherwise the reason it was dropped.
+
+    Requests to other domains are kept (as third-party APIs, e.g. a site's search running
+    on Algolia) unless they are analytics or known infrastructure.
+    """
+    if resource_type not in API_RESOURCE_TYPES:
+        return f"resource:{resource_type}"
+    if is_noise_domain(url):
+        return "analytics"
+    if is_third_party(url, target_host) and is_infra_domain(url):
+        return "infrastructure"
+    if _HEALTH_RE.search(urlsplit(url).path):
+        return "health-check"
+    if _FIRST_PARTY_NOISE_RE.search(urlsplit(url).path):
+        return "analytics"
+    return None
+
+
+# --- Redaction -----------------------------------------------------------------------------
+
+
+def _name_tokens(name: str) -> list[str]:
+    name = _CAMEL_RE.sub("_", name)
+    return [t for t in re.split(r"[^a-z0-9]+", name.lower()) if t]
+
+
+def is_sensitive_name(name: str) -> bool:
+    """True for header / parameter / field names that usually hold secrets.
+
+    Splits on separators and camelCase so 'accessToken' and 'X-Auth-Token' match,
+    but 'author' and 'key' do not.
+    """
+    tokens = _name_tokens(name)
+    joined = "".join(tokens)
+    if joined in _SENSITIVE_WORDS:
+        return True
+    for i, tok in enumerate(tokens):
+        if tok in _SENSITIVE_WORDS or tok.endswith(_SENSITIVE_SUFFIXES):
+            return True
+        if tok == "key" and i > 0 and tokens[i - 1] in _SENSITIVE_KEY_PREFIXES:
+            return True
+    return False
+
+
+def redact_text(text: str) -> str:
+    """Mask secret-looking values (JWTs, bearer/basic credentials) inside free text."""
+    text = _JWT_RE.sub(REDACTED, text)
+    return _BEARER_RE.sub(lambda m: f"{m.group(1)} {REDACTED}", text)
+
+
+def redact_headers(headers: dict[str, str]) -> dict[str, str]:
+    return {k: (REDACTED if is_sensitive_name(k) else redact_text(v)) for k, v in headers.items()}
+
+
+def redact_url(url: str) -> str:
+    parts = urlsplit(url)
+    if not parts.query:
+        return url
+    pairs = parse_qsl(parts.query, keep_blank_values=True)
+    safe = [(k, REDACTED if is_sensitive_name(k) else redact_text(v)) for k, v in pairs]
+    return urlunsplit(parts._replace(query=urlencode(safe, safe="[]{}")))
+
+
+def redact_json(value: Any) -> Any:
+    if isinstance(value, dict):
+        return {
+            k: (
+                REDACTED
+                if is_sensitive_name(k) and not isinstance(v, dict | list)
+                else redact_json(v)
+            )
+            for k, v in value.items()
+        }
+    if isinstance(value, list):
+        return [redact_json(v) for v in value]
+    if isinstance(value, str):
+        return redact_text(value)
+    return value
+
+
+def redact_body(body: str | None, content_type: str | None) -> str | None:
+    """Redact a request or response body, keeping its format."""
+    if not body:
+        return body
+    ctype = (content_type or "").lower()
+    stripped = body.lstrip()
+    if "json" in ctype or stripped[:1] in ("{", "["):
+        try:
+            return json.dumps(redact_json(json.loads(body)), ensure_ascii=False)
+        except ValueError:
+            pass
+    if "x-www-form-urlencoded" in ctype:
+        pairs = parse_qsl(body, keep_blank_values=True)
+        return urlencode([(k, REDACTED if is_sensitive_name(k) else v) for k, v in pairs])
+    return redact_text(body)

ghost_api/capture/har.py

@@ -0,0 +1,148 @@
+"""Read a HAR file (the browser's "Save all as HAR") into captured requests.
+
+For sites Ghost API can't explore by itself: behind a login, a bot check you passed by
+hand, or a flow too long for the agent. You browse; your browser records; Ghost API maps
+the API from the recording, with no browser and no LLM of its own.
+
+Every entry goes through the same filters as live capture (`recorder.build_capture`):
+only JSON API calls are kept, cookies, tokens and keys are redacted, and personal data is
+hidden before anything is stored. The unredacted originals stay in memory only, for
+testing endpoints with direct calls. A HAR file itself holds your cookies and tokens in
+plain text: delete it when you are done.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import json
+from collections import Counter
+from collections.abc import Iterator
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlsplit
+
+from ghost_api.capture import filters
+from ghost_api.capture.models import CapturedRequest, RawRequest
+from ghost_api.capture.recorder import build_capture
+
+MAX_HAR_BYTES = 500_000_000
+
+
+class HarError(Exception):
+    """The file is not a HAR file Ghost API can read."""
+
+
+@dataclass
+class HarImport:
+    start_url: str
+    target_host: str
+    captured: list[tuple[CapturedRequest, RawRequest]] = field(default_factory=list)
+    dropped: Counter[str] = field(default_factory=Counter)
+    entries: int = 0
+
+
+def read_har(path: Path, start_url: str | None = None) -> HarImport:
+    if path.stat().st_size > MAX_HAR_BYTES:
+        raise HarError(f"{path.name} is larger than {MAX_HAR_BYTES // 1_000_000} MB")
+    try:
+        data = json.loads(path.read_text(encoding="utf-8-sig"))
+        entries = data["log"]["entries"]
+    except (OSError, ValueError, KeyError, TypeError) as exc:
+        raise HarError(f"{path.name} is not a HAR file ({exc})") from exc
+    if not isinstance(entries, list):
+        raise HarError(f"{path.name} has no entries")
+    start = start_url or _start_url(entries)
+    if not start:
+        raise HarError(f"{path.name} has no http(s) requests")
+    result = HarImport(start, (urlsplit(start).hostname or "").lower(), entries=len(entries))
+    for entry in entries:
+        try:
+            item = _entry(entry, result.target_host)
+        except (KeyError, TypeError, ValueError):
+            result.dropped["unreadable"] += 1
+            continue
+        if isinstance(item, str):
+            result.dropped[item] += 1
+        else:
+            result.captured.append(item)
+    return result
+
+
+def _start_url(entries: list[Any]) -> str:
+    """The first page the browser loaded, or the origin of the first request."""
+    for entry in entries:
+        url = str(entry.get("request", {}).get("url", ""))
+        if entry.get("_resourceType") == "document" and url.startswith("http"):
+            return url
+    for entry in entries:
+        url = str(entry.get("request", {}).get("url", ""))
+        if url.startswith("http"):
+            parts = urlsplit(url)
+            return f"{parts.scheme}://{parts.netloc}/"
+    return ""
+
+
+def _headers(items: list[dict[str, Any]]) -> dict[str, str]:
+    # HTTP/2 pseudo-headers (":authority") are not real headers.
+    return {str(h["name"]).lower(): str(h["value"]) for h in items if not h["name"].startswith(":")}
+
+
+def _resource_type(entry: dict[str, Any], resp_headers: dict[str, str]) -> str:
+    """Chrome and Edge write `_resourceType`; Firefox doesn't, so JSON replies count as fetch."""
+    kind = entry.get("_resourceType")
+    if isinstance(kind, str) and kind:
+        return kind.lower()
+    if filters.is_json_content_type(resp_headers.get("content-type")):
+        return "fetch"
+    return "other"
+
+
+def _entry(entry: dict[str, Any], target_host: str) -> tuple[CapturedRequest, RawRequest] | str:
+    request, response = entry["request"], entry["response"]
+    url = str(request["url"])
+    if not url.startswith(("http://", "https://")):
+        return "not-http"
+    req_headers = _headers(request.get("headers", []))
+    resp_headers = _headers(response.get("headers", []))
+    kind = _resource_type(entry, resp_headers)
+    reason = filters.classify(kind, url, target_host)
+    if reason:
+        return reason
+    status = int(response.get("status") or 0)
+    if 300 <= status < 400:
+        return "redirect"
+    if status == 0:
+        return "no-reply"  # blocked, cancelled or failed in the browser
+    content = response.get("content", {})
+    text = content.get("text")
+    if text is None:
+        return "no-body"  # saved without content ("Save all as HAR (sanitized)")
+    body = _decode(str(text), content.get("encoding"))
+    if body is None:
+        return "binary"
+    post = request.get("postData", {}).get("text")
+    post_bytes = str(post).encode("utf-8") if post is not None else None
+    if "content-type" not in resp_headers and content.get("mimeType"):
+        resp_headers["content-type"] = str(content["mimeType"])
+    return build_capture(
+        str(request["method"]).upper(), url, kind, status, req_headers, post_bytes,
+        resp_headers, body, target_host,
+    )  # fmt: skip
+
+
+def _decode(text: str, encoding: Any) -> bytes | None:
+    if encoding == "base64":
+        try:
+            return base64.b64decode(text, validate=True)
+        except (binascii.Error, ValueError):
+            return None
+    return text.encode("utf-8")
+
+
+def summary(result: HarImport) -> Iterator[str]:
+    yield f"{result.entries} entries, {len(result.captured)} API calls kept"
+    if result.dropped:
+        top = ", ".join(f"{k}: {v}" for k, v in result.dropped.most_common(6))
+        yield f"left out: {top}"

ghost_api/capture/models.py

@@ -0,0 +1,41 @@
+"""Data captured from the browser's network traffic."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from typing import Any
+
+
+@dataclass
+class CapturedRequest:
+    """One API request/response pair. All fields are already redacted."""
+
+    method: str
+    url: str
+    status: int
+    resource_type: str
+    req_headers: dict[str, str]
+    req_body: str | None
+    req_content_type: str | None
+    resp_content_type: str | None
+    resp_body: str | None
+    truncated: bool = False
+    third_party: bool = False  # served from a different domain than the target site
+    timestamp: str = field(default_factory=lambda: datetime.now(UTC).isoformat())
+    # Personal data hidden from resp_body before storing (capture/personal.py):
+    # [{"path", "kind", "severity", "example" (masked), "count"}].
+    personal: list[dict[str, Any]] = field(default_factory=list)
+
+
+@dataclass
+class RawRequest:
+    """The original, UNREDACTED request, kept in memory only so endpoints can be replayed.
+
+    Never stored, logged or sent to an LLM (NFR-13). repr() hides headers and body.
+    """
+
+    method: str
+    url: str = field(repr=False)
+    headers: dict[str, str] = field(repr=False)
+    body: str | None = field(default=None, repr=False)