ghost-api-cli 0.2.0__py3-none-any.whl

ghost_api/capture/personal.py

@@ -0,0 +1,188 @@
+"""Find personal and sensitive data in API replies, and hide it before anything is kept.
+
+Found on turftown.in: a public venue-details call returned owners' PAN and GST numbers,
+bank UPI IDs and staff emails to anyone. Ghost API now spots such values when a reply is
+captured, replaces them with a marker like "[hidden email: a***@g***.com]" (strings) or
+0 (numbers, so the inferred type stays the same), and records what it hid. The report
+then shows which endpoints expose what, without ever storing the values themselves.
+
+Detection uses the value's shape and the field's name together, to keep false alarms
+low: a 10-digit number is only a phone number in a field called phone/mobile/contact.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import asdict, dataclass
+from typing import Any, Literal
+
+Severity = Literal["high", "medium"]
+
+_EMAIL_RE = re.compile(r"^[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}$")
+_UPI_RE = re.compile(r"^[A-Za-z0-9._-]{2,}@[A-Za-z][A-Za-z0-9]{1,}$")  # name@okaxis (no dot)
+_PAN_RE = re.compile(r"^[A-Z]{5}[0-9]{4}[A-Z]$")
+_GSTIN_RE = re.compile(r"^[0-9]{2}[A-Z]{5}[0-9]{4}[A-Z][1-9A-Z]Z[0-9A-Z]$")
+_PHONE_RE = re.compile(r"^\+?[0-9][0-9 ()-]{7,16}[0-9]$")
+_INTL_PHONE_RE = re.compile(r"^\+[1-9][0-9 ()-]{7,16}[0-9]$")
+_AADHAAR_RE = re.compile(r"^[2-9][0-9]{3}[ -]?[0-9]{4}[ -]?[0-9]{4}$")
+_DIGITS_RE = re.compile(r"^[0-9]{9,19}$")
+_MARKER_RE = re.compile(r"^\[hidden ([a-z_]+): (.*)\]$")
+
+_PHONE_NAMES = ("phone", "mobile", "contact", "whatsapp", "msisdn", "tel")
+_ACCOUNT_NAMES = ("account_number", "accountnumber", "account_no", "acc_no", "bank_account")
+_CARD_NAMES = ("card", "cc_number", "ccnumber")
+_AADHAAR_NAMES = ("aadhaar", "aadhar", "uid_number")
+_UPI_NAMES = ("upi", "vpa")
+# Common UPI handle providers (the part after "@"), for values in fields not named upi/vpa.
+_UPI_PROVIDERS = {
+    "okaxis", "okhdfcbank", "okicici", "oksbi", "ybl", "ibl", "axl", "paytm", "upi", "apl",
+    "yapl", "rapl", "airtel", "freecharge", "jio", "kotak", "hdfcbank", "icici", "sbi",
+    "axisbank", "pthdfc", "ptsbi", "ptyes", "ptaxis", "waaxis", "wahdfcbank", "wasbi",
+    "waicici", "abfspay", "ikwik", "idfcbank", "indus", "barodampay", "fbl", "federal",
+}  # fmt: skip
+_DOB_NAMES = ("dob", "date_of_birth", "dateofbirth", "birth_date", "birthdate", "birthday")
+# A person's name, when the field says so (names can't be told from their value alone).
+_HOLDER_NAMES = (
+    "accountname", "account_name", "accountholder", "account_holder", "holder_name",
+    "holdername", "beneficiary", "owner_name", "ownername", "card_holder", "cardholder",
+)  # fmt: skip
+
+SEVERITY: dict[str, Severity] = {
+    "pan": "high", "gstin": "high", "aadhaar": "high", "bank_account": "high",
+    "card": "high", "upi": "high", "email": "medium", "phone": "medium",
+    "date_of_birth": "medium", "account_holder": "medium",
+}  # fmt: skip
+LABEL = {
+    "pan": "PAN number", "gstin": "GST number", "aadhaar": "Aadhaar number",
+    "bank_account": "bank account number", "card": "card number", "upi": "UPI ID",
+    "email": "email address", "phone": "phone number", "date_of_birth": "date of birth",
+    "account_holder": "account holder's name",
+}  # fmt: skip
+
+
+@dataclass(frozen=True)
+class Finding:
+    path: str  # field path, list positions as []: "data.rating[].email"
+    kind: str
+    severity: Severity
+    example: str  # masked: "a***@g***.com"
+
+    def to_dict(self) -> dict[str, str]:
+        return asdict(self)
+
+
+def classify(key: str, value: Any) -> str | None:
+    """The kind of personal data `value` (in field `key`) is, or None."""
+    if isinstance(value, bool) or value is None:
+        return None
+    name = key.lower()
+    text = str(value).strip()
+    if not text or _MARKER_RE.match(text):
+        return None
+    if isinstance(value, str):
+        if _EMAIL_RE.match(text):
+            return "email"
+        if _PAN_RE.match(text):
+            return "pan"
+        if _GSTIN_RE.match(text):
+            return "gstin"
+        if _UPI_RE.match(text) and (
+            any(n in name for n in _UPI_NAMES) or text.rpartition("@")[2].lower() in _UPI_PROVIDERS
+        ):
+            return "upi"
+        if _INTL_PHONE_RE.match(text) and _digit_count(text) >= 10:
+            return "phone"
+    if (
+        isinstance(value, str)
+        and any(n in name.replace("-", "_") for n in _HOLDER_NAMES)
+        and re.search(r"[A-Za-z]{2}", text)
+    ):
+        return "account_holder"
+    if any(n in name for n in _DOB_NAMES) and isinstance(value, str) and re.search(r"\d", text):
+        return "date_of_birth"
+    if any(n in name for n in _AADHAAR_NAMES) and _AADHAAR_RE.match(text):
+        return "aadhaar"
+    if any(n in name for n in _ACCOUNT_NAMES) and _DIGITS_RE.match(text):
+        return "bank_account"
+    if any(n in name for n in _CARD_NAMES) and _DIGITS_RE.match(text) and _luhn(text):
+        return "card"
+    if (
+        any(n in name for n in _PHONE_NAMES)
+        and _PHONE_RE.match(text)
+        and 10 <= _digit_count(text) <= 15
+    ):
+        return "phone"
+    return None
+
+
+def mask(text: str, kind: str) -> str:
+    """Enough to recognise the kind, nothing to recover the value: 'a***@g***.com'."""
+    if kind in ("email", "upi") and "@" in text:
+        user, _, domain = text.partition("@")
+        head, dot, tld = domain.rpartition(".")
+        domain_masked = f"{head[:1]}***{dot}{tld}" if dot else f"{domain[:1]}***"
+        return f"{user[:1]}***@{domain_masked}"
+    keep = 2 if len(text) > 6 else 1
+    return text[:keep] + "*" * max(3, len(text) - keep - 1) + text[-1:]
+
+
+def hide(data: Any, path: str = "") -> tuple[Any, list[Finding]]:
+    """A copy of `data` with personal values replaced, and what was replaced."""
+    findings: list[Finding] = []
+
+    def walk(value: Any, key: str, where: str) -> Any:
+        if isinstance(value, dict):
+            return {
+                k: walk(v, str(k), f"{where}.{k}" if where else str(k)) for k, v in value.items()
+            }
+        if isinstance(value, list):
+            return [walk(v, key, f"{where}[]") for v in value]
+        kind = classify(key, value)
+        if kind is None:
+            return value
+        example = mask(str(value).strip(), kind)
+        findings.append(Finding(where or key or "value", kind, SEVERITY[kind], example))
+        if isinstance(value, str):
+            return f"[hidden {kind}: {example}]"
+        return 0  # numbers stay numbers, so the inferred schema keeps its type
+
+    return walk(data, "", path), findings
+
+
+def hide_body(body: str | None) -> tuple[str | None, list[dict[str, str]]]:
+    """For a JSON body (text): the body with personal values hidden, and what was hidden
+    (one entry per field path). Non-JSON bodies are returned as they are."""
+    if not body:
+        return body, []
+    try:
+        data = json.loads(body)
+    except ValueError:
+        return body, []
+    hidden, findings = hide(data)
+    if not findings:
+        return body, []
+    return json.dumps(hidden, ensure_ascii=False), summarize(findings)
+
+
+def summarize(findings: list[Finding]) -> list[dict[str, str]]:
+    """One entry per (path, kind), with how many values were hidden."""
+    seen: dict[tuple[str, str], dict[str, Any]] = {}
+    for f in findings:
+        entry = seen.setdefault((f.path, f.kind), {**f.to_dict(), "count": 0})
+        entry["count"] += 1
+    return list(seen.values())
+
+
+def _digit_count(text: str) -> int:
+    return sum(c.isdigit() for c in text)
+
+
+def _luhn(digits: str) -> bool:
+    total = 0
+    for i, ch in enumerate(reversed(digits)):
+        n = int(ch)
+        if i % 2:
+            n = n * 2 - 9 if n > 4 else n * 2
+        total += n
+    return total % 10 == 0

ghost_api/capture/recorder.py

@@ -0,0 +1,181 @@
+"""Record API traffic from a Playwright page (FR-13, FR-14).
+
+The 'response' event handler only queues responses; bodies are read later in `flush()`,
+which the browser driver calls after every action. Reading bodies outside the event
+handler keeps the sync Playwright API safe from re-entrancy problems.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from collections.abc import Callable
+
+from playwright.sync_api import Error as PlaywrightError
+from playwright.sync_api import Page, Response
+
+from ghost_api.capture import filters, personal
+from ghost_api.capture.models import CapturedRequest, RawRequest
+
+MAX_BODY_CHARS = 500_000
+MAX_CAPTURE_BYTES = 5_000_000  # one reply bigger than this is not read (response-too-large)
+RUN_CAPTURE_BUDGET = 200_000_000  # bodies read in one run; beyond it nothing more is read
+
+
+class TrafficRecorder:
+    def __init__(
+        self,
+        target_host: str,
+        on_capture: Callable[[CapturedRequest], None] | None = None,
+    ) -> None:
+        self.target_host = target_host
+        self.on_capture = on_capture
+        self.captured: list[CapturedRequest] = []
+        # raw[i] is the unredacted original of captured[i] (memory only, for replaying).
+        self.raw: list[RawRequest | None] = []
+        self.dropped: Counter[str] = Counter()
+        self._pending: list[Response] = []
+        self.bytes_read = 0
+
+    def attach(self, page: Page) -> None:
+        page.on("response", self._on_response)
+
+    def _on_response(self, response: Response) -> None:
+        request = response.request
+        reason = filters.classify(request.resource_type, request.url, self.target_host)
+        if reason:
+            self.dropped[reason] += 1
+            return
+        if 300 <= response.status < 400:
+            self.dropped["redirect"] += 1
+            return
+        self._pending.append(response)
+
+    def flush(self) -> list[CapturedRequest]:
+        """Read bodies of queued responses, keep JSON ones, redact, and store them."""
+        new: list[CapturedRequest] = []
+        pending, self._pending = self._pending, []
+        for response in pending:
+            try:
+                built = self._build(response)
+            except Exception:  # noqa: BLE001 - one odd response must not stop the run
+                self.dropped["unreadable"] += 1
+                continue
+            if built is None:
+                continue
+            new.append(self.add(*built))
+        return new
+
+    def add(self, item: CapturedRequest, raw: RawRequest | None = None) -> CapturedRequest:
+        """Record a request (from the browser, or a replay made by the prober)."""
+        self.captured.append(item)
+        self.raw.append(raw)
+        if self.on_capture:
+            self.on_capture(item)
+        return item
+
+    def _build(self, response: Response) -> tuple[CapturedRequest, RawRequest] | None:
+        request = response.request
+        try:
+            resp_headers = response.all_headers()
+            req_headers = request.all_headers()
+            declared = resp_headers.get("content-length", "")
+            if declared.isdigit() and int(declared) > MAX_CAPTURE_BYTES:
+                self.dropped["response-too-large"] += 1  # never read into memory
+                return None
+            if self.bytes_read > RUN_CAPTURE_BUDGET:
+                self.dropped["capture-budget-used"] += 1
+                return None
+            body_bytes = response.body()
+            post_bytes = request.post_data_buffer
+        except PlaywrightError:
+            # Body is gone (page navigated away, request aborted, ...).
+            self.dropped["no-body"] += 1
+            return None
+
+        self.bytes_read += len(body_bytes)
+        built = build_capture(
+            request.method, request.url, request.resource_type, response.status,
+            req_headers, post_bytes, resp_headers, body_bytes, self.target_host,
+        )  # fmt: skip
+        if isinstance(built, str):
+            self.dropped[built] += 1
+            return None
+        return built
+
+
+def build_capture(
+    method: str,
+    url: str,
+    resource_type: str,
+    status: int,
+    req_headers: dict[str, str],
+    post_bytes: bytes | None,
+    resp_headers: dict[str, str],
+    body_bytes: bytes,
+    target_host: str,
+) -> tuple[CapturedRequest, RawRequest] | str:
+    """One request and reply, filtered and made safe to store: secrets redacted, personal
+    data hidden. Returns why it was dropped (a short reason) when it is not a JSON API call.
+    Used for the browser's traffic and for imported HAR files alike."""
+    req_headers = {k.lower(): v for k, v in req_headers.items()}
+    resp_headers = {k.lower(): v for k, v in resp_headers.items()}
+    if len(body_bytes) > MAX_CAPTURE_BYTES:
+        # The body is not decoded, parsed or stored, so a partial reply never becomes a schema.
+        return "response-too-large"
+    # Binary bodies (protobuf, images sent over fetch...) are not JSON APIs.
+    body = _utf8(body_bytes)
+    post_data = _utf8(post_bytes) if post_bytes else None
+    if body is None or (post_bytes and post_data is None):
+        return "binary"
+    resp_ctype = resp_headers.get("content-type")
+    if not filters.is_json_content_type(resp_ctype) and not _looks_like_json(body):
+        return "non-json"
+    body = filters.strip_xssi(body)
+    if body.strip() and not _looks_like_json(body):
+        # Says JSON but isn't (e.g. Google's length-prefixed batchexecute envelope).
+        return "invalid-json"
+
+    # Redact the full body first: truncated JSON can't be parsed for key-based redaction.
+    safe_body = filters.redact_body(body, resp_ctype) or ""
+    hidden_body, personal_data = personal.hide_body(safe_body)
+    safe_body = hidden_body or ""
+    truncated = len(safe_body) > MAX_BODY_CHARS
+    if truncated:
+        safe_body = safe_body[:MAX_BODY_CHARS]
+
+    req_ctype = req_headers.get("content-type")
+    raw = RawRequest(method, url, dict(req_headers), post_data)
+    captured = CapturedRequest(
+        method=method,
+        url=filters.redact_url(url),
+        status=status,
+        resource_type=resource_type,
+        req_headers=filters.redact_headers(req_headers),
+        req_body=filters.redact_body(post_data, req_ctype),
+        req_content_type=req_ctype,
+        resp_content_type=resp_ctype,
+        resp_body=safe_body,
+        truncated=truncated,
+        third_party=filters.is_third_party(url, target_host),
+        personal=personal_data,
+    )
+    return captured, raw
+
+
+def _utf8(data: bytes) -> str | None:
+    try:
+        return data.decode("utf-8")
+    except UnicodeDecodeError:
+        return None
+
+
+def _looks_like_json(body: str) -> bool:
+    stripped = body.lstrip()
+    if stripped[:1] not in ("{", "["):
+        return False
+    try:
+        json.loads(stripped)
+    except ValueError:
+        return False
+    return True

ghost_api/cli/__init__.py

@@ -0,0 +1,44 @@
+"""The `ghost-api` command line.
+
+_common.py   the app, global options (--json, --plain, project defaults) and helpers
+explore.py   run, import, refresh, login, logout
+client.py    generate, verify, report, diff, mock
+data.py      fetch, watch, serve, mcp, ui
+manage.py    init, doctor, inspect, forget
+"""
+
+# isort: off
+# Importing the command modules registers their commands, in this order in --help.
+from ghost_api.cli import explore  # noqa: F401
+from ghost_api.cli import client  # noqa: F401
+from ghost_api.cli import data  # noqa: F401
+from ghost_api.cli import manage  # noqa: F401
+
+# isort: on
+from ghost_api.cli._common import (
+    GLOBAL_FLAGS,
+    _invoke,
+    _json_first,
+    _options,
+    _RunResult,
+    app,
+    console,
+    entry,
+    json_out,
+    safe_output,
+)
+from ghost_api.cli.data import schedule_command
+
+__all__ = [
+    "GLOBAL_FLAGS",
+    "_RunResult",
+    "_invoke",
+    "_json_first",
+    "_options",
+    "app",
+    "console",
+    "entry",
+    "json_out",
+    "safe_output",
+    "schedule_command",
+]

ghost_api/cli/_common.py

@@ -0,0 +1,219 @@
+"""Shared parts of the command line: the app, global options and helpers."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Annotated
+from urllib.parse import urlsplit
+
+import typer
+from rich.console import Console
+
+from ghost_api import __version__, exitcodes, workflows
+from ghost_api.config import (
+    check_language,
+    default_db_path,
+    load_env,
+    state_dir,
+)
+from ghost_api.llm.providers import default_providers
+from ghost_api.memory.store import Store, StoreError
+from ghost_api.project import ProjectError, default_map, find_project
+from ghost_api.site import SiteAmbiguous, resolve_site
+from ghost_api.ui import keys
+from ghost_api.ui.terminal import (
+    TerminalRenderer,
+)
+
+app = typer.Typer(
+    help="Ghost API: turn any website into a typed, tested API client.",
+    add_completion=True,
+)
+console = Console()
+_options = {"plain": False, "json": False}
+
+
+def _version(value: bool) -> None:
+    if value:
+        console.print(f"ghost-api {__version__}")
+        raise typer.Exit()
+
+
+@app.callback(invoke_without_command=True)
+def main(
+    ctx: typer.Context,
+    version: Annotated[
+        bool | None,
+        typer.Option(
+            "--version", callback=_version, is_eager=True, expose_value=False, help="Show version."
+        ),
+    ] = None,
+    plain: Annotated[
+        bool, typer.Option("--plain", help="Simple line-by-line output (for logs and CI).")
+    ] = False,
+    json_output: Annotated[
+        bool,
+        typer.Option(
+            "--json",
+            help="Print the result as JSON on stdout (progress goes to stderr). "
+            "Works anywhere on the command line.",
+        ),
+    ] = False,
+) -> None:
+    load_env()
+    for stream in (sys.stdout, sys.stderr):
+        safe_output(stream)
+    _options["plain"] = plain or json_output
+    _options["json"] = json_output
+    if json_output:  # stdout is only for the JSON result; everything else goes to stderr
+        saved = console._file  # usually None: Rich then follows sys.stdout as it changes
+        console.file = sys.stderr
+        ctx.call_on_close(lambda: setattr(console, "_file", saved))
+    try:
+        project = find_project()
+    except ProjectError as exc:
+        console.print(f"[red]{exc}[/]")
+        raise typer.Exit(code=exitcodes.USAGE) from exc
+    if project is not None:
+        ctx.default_map = default_map(project, ctx.command)
+    if ctx.invoked_subcommand is None:
+        if not sys.stdin.isatty():
+            console.print(ctx.get_help())
+            raise typer.Exit()
+        _interactive()
+
+
+GLOBAL_FLAGS = ("--json", "--plain")
+
+
+def _json_first(args: list[str]) -> list[str]:
+    """`--json` and `--plain` may be typed anywhere (`ghost-api inspect --json`); they are
+    global options, so they are moved in front of the command."""
+    flags = [f for f in GLOBAL_FLAGS if f in args]
+    return [*flags, *(a for a in args if a not in GLOBAL_FLAGS)] if flags else args
+
+
+def json_out(data: object) -> None:
+    """The command's result, as JSON on stdout (progress and messages are on stderr)."""
+    sys.stdout.write(json.dumps(data, indent=2, ensure_ascii=False, default=str) + "\n")
+    sys.stdout.flush()
+
+
+def entry() -> None:
+    """The `ghost-api` command: the app, with a plain message for a damaged database."""
+    try:
+        app(args=_json_first(sys.argv[1:]), prog_name="ghost-api")
+    except StoreError as exc:
+        console.print(f"[red]{exc}[/]")
+        raise SystemExit(1) from None
+
+
+def safe_output(stream: object) -> None:
+    """Never crash on a character the output can't show (e.g. '◆' when output is
+    redirected to a file on Windows, which uses cp1252): print a '?' instead."""
+    if stream is None:  # pythonw / no console: nothing to protect
+        return
+    encoding = (getattr(stream, "encoding", None) or "").lower().replace("-", "")
+    reconfigure = getattr(stream, "reconfigure", None)
+    if encoding != "utf8" and reconfigure is not None:
+        reconfigure(errors="replace")
+
+
+def _invoke(args: list[str]) -> None:
+    """Run `ghost-api ARGS...` inside this process (the interactive session's commands)."""
+    command = typer.main.get_command(app)
+    try:
+        command.main(args=_json_first(args), prog_name="ghost-api", standalone_mode=False)
+    except typer.Abort:
+        console.print("[yellow]Stopped.[/]")
+    except typer.Exit:
+        pass
+    except StoreError as exc:
+        console.print(f"[red]{exc}[/]")
+    except typer.TyperException as exc:  # usage errors: missing argument, unknown option...
+        format_message = getattr(exc, "format_message", None)
+        message = format_message() if callable(format_message) else str(exc)
+        console.print(f"[red]{message}[/]  [dim](see /help)[/]")
+
+
+def _interactive() -> None:
+    """`ghost-api` on its own: a session that stays open until /exit or Ctrl+C twice."""
+    from ghost_api.ui.repl import Repl, run_session
+
+    def ask(question: str) -> str:
+        with keys.input_paused():
+            return console.input(f"[bold]{question}[/]")
+
+    def has_llm() -> bool:
+        load_env()  # a key added to .env while the session is open counts at once
+        return any(p.api_key for p in default_providers())
+
+    repl = Repl(console, _invoke, ask)
+    run_session(console, repl, has_llm, history_file=state_dir() / "history.txt")
+
+
+class _RunResult(workflows.RunCollector):
+    """A run's outcome for `--json`, and its exit code."""
+
+    def finish(self, db: Path) -> None:
+        """Print the JSON (with --json) and exit with the code for the run's status."""
+        status = str(self.data["status"])
+        if _options["json"]:
+            json_out(self.data | {"database": str(db), "exit_code": _run_exit(status)})
+        if _run_exit(status):
+            raise typer.Exit(code=_run_exit(status))
+
+
+def _run_exit(status: str) -> int:
+    return exitcodes.for_run_status(status)
+
+
+def _renderer() -> TerminalRenderer:
+    return TerminalRenderer(console, plain=True if _options["plain"] else None)
+
+
+def _normalize_url(url: str) -> str:
+    if "://" not in url:
+        url = "https://" + url
+    parts = urlsplit(url)
+    if parts.scheme not in ("http", "https") or not parts.hostname:
+        raise typer.BadParameter(f"Not a valid http(s) URL: {url}")
+    return url
+
+
+def _check_lang(lang: str) -> str:
+    try:
+        return check_language(lang)
+    except ValueError as exc:
+        raise typer.BadParameter(str(exc)) from exc
+
+
+def _not_allowed(url: str, reason: str = "disallowed by robots.txt") -> None:
+    if _options["json"]:
+        json_out({"status": "not-allowed", "url": url, "reason": reason,
+                  "exit_code": exitcodes.NOT_ALLOWED})  # fmt: skip
+    elif not reason.startswith("robots.txt"):  # robots.txt was already reported
+        console.print(f"[yellow]{reason}[/]")
+    raise typer.Exit(code=exitcodes.NOT_ALLOWED)
+
+
+def _need_db(db: Path | None) -> Path:
+    path = db or default_db_path()
+    if not path.exists():
+        console.print(f"No database at {path}. Run `ghost-api run` first.")
+        raise typer.Exit(code=1)
+    return path
+
+
+SITE_HELP = "Site explored before, e.g. quotes.toscrape.com or www.google.com/maps"
+
+
+def _site(store: Store, name: str) -> str:
+    """The stored site for what was typed: a site, a URL, or just the domain."""
+    try:
+        return resolve_site(name, {r.domain for r in store.list_runs()})
+    except SiteAmbiguous as exc:
+        console.print(str(exc))
+        raise typer.Exit(code=1) from exc