ghost-api-cli 0.2.0__py3-none-any.whl

ghost_api/probe/prober.py

@@ -0,0 +1,311 @@
+"""Confirm or reject guesses about each endpoint by calling it (Stage 5).
+
+For every endpoint with a replayable sample:
+  1. Replay it as captured. If that fails, the endpoint can't be called outside the
+     browser (signed URLs, one-time tokens...), and testing stops there.
+  2. Pagination: request the next page / offset / cursor and compare the records.
+  3. Required headers: drop all non-browser headers at once; only if that breaks the call,
+     drop them one at a time to find the ones that matter.
+  4. Secret-looking query parameters: drop each to see if it's required (API keys).
+  5. Name the auth style from what turned out to be required.
+
+Every test is recorded as a hypothesis with its evidence (FR-24).
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any, Literal
+from urllib.parse import parse_qsl, urlsplit
+
+from ghost_api.analysis import detectors
+from ghost_api.analysis.models import Endpoint
+from ghost_api.capture.filters import is_sensitive_name
+from ghost_api.capture.models import CapturedRequest, RawRequest
+from ghost_api.probe.replay import (
+    Replayer,
+    ReplayRefused,
+    ReplayResult,
+    is_replayable,
+    with_overrides,
+)
+
+Outcome = Literal["confirmed", "rejected", "inconclusive"]
+MAX_HEADER_TESTS = 10
+
+
+@dataclass
+class Hypothesis:
+    endpoint: str  # Endpoint.key
+    text: str
+    result: Outcome
+    evidence: str
+
+
+@dataclass
+class ProbeReport:
+    endpoint: str  # Endpoint.key
+    replayable: bool = False
+    pagination: dict[str, Any] | None = None
+    required_headers: list[str] = field(default_factory=list)
+    required_query: list[str] = field(default_factory=list)
+    auth_type: str | None = None
+    hypotheses: list[Hypothesis] = field(default_factory=list)
+    note: str = ""
+
+
+class Prober:
+    def __init__(
+        self,
+        replayer: Replayer,
+        on_sample: Callable[[CapturedRequest, RawRequest], object] | None = None,
+    ) -> None:
+        self.replayer = replayer
+        self.on_sample = on_sample  # new samples (e.g. page 2) enrich the schemas
+
+    def probe(self, endpoint: Endpoint, samples: list[RawRequest]) -> ProbeReport:
+        report = ProbeReport(endpoint.key)
+        sample = next((s for s in reversed(samples) if is_replayable(s)), None)
+        if sample is None:
+            report.note = "no sample that is safe to replay (only read-only requests are sent)"
+            return report
+        try:
+            baseline = self._baseline(endpoint, sample, report)
+            if baseline is None:
+                return report
+            self._pagination(endpoint, sample, baseline, report, samples)
+            self._headers(endpoint, sample, baseline, report)
+            self._secret_params(endpoint, sample, baseline, report)
+        except ReplayRefused as exc:
+            report.note = f"testing stopped: {exc}"
+        report.auth_type = detectors.auth_style(report.required_headers, report.required_query)
+        if report.auth_type == "bearer":
+            auth_value = next(
+                (v for k, v in sample.headers.items() if k.lower() == "authorization"), ""
+            )
+            if auth_value.lower().startswith("basic "):
+                report.auth_type = "basic"
+        return report
+
+    # --- tests ---
+
+    def _baseline(
+        self, ep: Endpoint, sample: RawRequest, report: ProbeReport
+    ) -> ReplayResult | None:
+        result = self.replayer.send(sample)
+        ok = result.ok
+        report.replayable = ok
+        report.hypotheses.append(
+            Hypothesis(
+                ep.key,
+                "can be called directly, outside the browser",
+                "confirmed" if ok else "rejected",
+                _evidence(result),
+            )
+        )
+        return result if ok else None
+
+    def _pagination(
+        self,
+        ep: Endpoint,
+        sample: RawRequest,
+        baseline: ReplayResult,
+        report: ProbeReport,
+        samples: list[RawRequest] | None = None,
+    ) -> None:
+        query = dict(parse_qsl(urlsplit(sample.url).query, keep_blank_values=True))
+        body = _json_body(sample)
+        guess = detectors.guess_pagination(query, body, baseline.body)
+        if guess is None:
+            return
+        source = query if guess.location == "query" else (body or {})
+        current = source.get(guess.param)
+        text = f"paginates with {guess.style} parameter '{guess.param}' ({guess.evidence})"
+
+        next_value: str | int | None
+        if guess.style == "cursor":
+            next_value = detectors.next_cursor(baseline.body)
+            if next_value is None:
+                # The newest sample is often the last page (no next cursor): an earlier one
+                # that still had a next page shows the paging just as well.
+                earlier = self._page_with_cursor(sample, samples or [])
+                if earlier is not None:
+                    sample, baseline = earlier
+                    query = dict(parse_qsl(urlsplit(sample.url).query, keep_blank_values=True))
+                    body = _json_body(sample)
+                    source = query if guess.location == "query" else (body or {})
+                    current = source.get(guess.param)
+                    next_value = detectors.next_cursor(baseline.body)
+            if next_value is None:
+                report.hypotheses.append(
+                    Hypothesis(ep.key, text, "inconclusive", "no next cursor in the response")
+                )
+                return
+        elif guess.style == "page":
+            next_value = _as_int(current, default=1) + 1
+        else:  # offset
+            items = detectors.find_items(baseline.body) or []
+            step = _as_int(source.get(guess.size_param) if guess.size_param else None, len(items))
+            next_value = _as_int(current, default=0) + max(step, 1)
+
+        if guess.location == "query":
+            probe = with_overrides(sample, query={guess.param: str(next_value)})
+        else:
+            typed = next_value if isinstance(current, int) else str(next_value)
+            probe = with_overrides(sample, body={guess.param: typed})
+        result = self.replayer.send(probe)
+        outcome, evidence = _compare_pages(baseline, result)
+        report.hypotheses.append(Hypothesis(ep.key, text, outcome, evidence))
+        if outcome != "rejected":
+            report.pagination = {
+                "style": guess.style,
+                "param": guess.param,
+                "location": guess.location,
+                "size_param": guess.size_param,
+                "confirmed": outcome == "confirmed",
+            }
+        if result.ok and result.captured is not None and self.on_sample:
+            self.on_sample(result.captured, probe)
+
+    def _page_with_cursor(
+        self, newest: RawRequest, samples: list[RawRequest]
+    ) -> tuple[RawRequest, ReplayResult] | None:
+        """Replay at most two earlier samples to find a page that has a next cursor."""
+        tried = 0
+        for sample in reversed(samples):
+            if sample.url == newest.url or not is_replayable(sample) or tried >= 2:
+                continue
+            tried += 1
+            result = self.replayer.send(sample)
+            if result.ok and detectors.next_cursor(result.body) is not None:
+                return sample, result
+        return None
+
+    def _headers(
+        self, ep: Endpoint, sample: RawRequest, baseline: ReplayResult, report: ProbeReport
+    ) -> None:
+        candidates = detectors.candidate_headers(sample.headers)[:MAX_HEADER_TESTS]
+        if not candidates:
+            return
+        stripped = self.replayer.send(with_overrides(sample, drop_headers=set(candidates)))
+        if _same(baseline, stripped):
+            report.hypotheses.append(
+                Hypothesis(
+                    ep.key,
+                    "needs none of its custom headers or cookies",
+                    "confirmed",
+                    f"still works without {', '.join(candidates)}",
+                )  # fmt: skip
+            )
+            return
+        for header in candidates:
+            result = self.replayer.send(with_overrides(sample, drop_headers={header}))
+            required = not _same(baseline, result)
+            if required:
+                report.required_headers.append(header)
+            report.hypotheses.append(
+                Hypothesis(
+                    ep.key,
+                    f"header '{header}' is required",
+                    "confirmed" if required else "rejected",
+                    f"without it: {_against(baseline, result)}",
+                )
+            )
+
+    def _secret_params(
+        self, ep: Endpoint, sample: RawRequest, baseline: ReplayResult, report: ProbeReport
+    ) -> None:
+        query = dict(parse_qsl(urlsplit(sample.url).query, keep_blank_values=True))
+        for name in [k for k in query if is_sensitive_name(k)]:
+            result = self.replayer.send(with_overrides(sample, query={name: None}))
+            required = not _same(baseline, result)
+            if required:
+                report.required_query.append(name)
+            report.hypotheses.append(
+                Hypothesis(
+                    ep.key,
+                    f"query parameter '{name}' (a key or token) is required",
+                    "confirmed" if required else "rejected",
+                    f"without it: {_against(baseline, result)}",
+                )
+            )
+
+
+# --- helpers ---------------------------------------------------------------------------------
+
+
+def _json_body(raw: RawRequest) -> dict[str, Any] | None:
+    if not raw.body:
+        return None
+    try:
+        data = json.loads(raw.body)
+    except ValueError:
+        return None
+    return data if isinstance(data, dict) else None
+
+
+def _as_int(value: Any, default: int) -> int:
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def _same(baseline: ReplayResult, other: ReplayResult) -> bool:
+    return other.ok and detectors.shape(other.body) == detectors.shape(baseline.body)
+
+
+def _evidence(result: ReplayResult) -> str:
+    if result.error and result.status == 0:
+        return result.error
+    text = f"HTTP {result.status}"
+    if result.error:
+        text += f", {result.error}"
+    elif result.ok:
+        items = detectors.find_items(result.body)
+        if items is not None:
+            text += f", {len(items)} record{'' if len(items) == 1 else 's'}"
+    return text
+
+
+def _against(baseline: ReplayResult, result: ReplayResult) -> str:
+    """Evidence for "X is required": what came back without X. An HTTP 200 alone would read
+    as "not required", so a different kind of response says how it differs."""
+    text = _evidence(result)
+    if result.ok and not _same(baseline, result):
+        text += (
+            f", but a different response: {_describe(result.body)} "
+            f"instead of {_describe(baseline.body)}"
+        )
+    return text
+
+
+def _describe(body: Any) -> str:
+    if isinstance(body, dict):
+        keys = list(body)
+        if not keys:
+            return "an empty object"
+        more = f" (+{len(keys) - 4} more)" if len(keys) > 4 else ""
+        return "fields " + ", ".join(keys[:4]) + more
+    if isinstance(body, list):
+        return f"a list of {len(body)}"
+    return f"a {type(body).__name__}"
+
+
+def _compare_pages(first: ReplayResult, second: ReplayResult) -> tuple[Outcome, str]:
+    if not second.ok:
+        return "inconclusive", f"next page failed: {_evidence(second)}"
+    if detectors.shape(first.body) != detectors.shape(second.body):
+        return "inconclusive", "next page returned a different kind of response"
+    a = detectors.item_fingerprints(detectors.find_items(first.body) or [])
+    b = detectors.item_fingerprints(detectors.find_items(second.body) or [])
+    if not a or not b:
+        return "inconclusive", f"pages had {len(a)} and {len(b)} records"
+    overlap = len(set(a) & set(b)) / max(len(set(b)), 1)
+    if overlap == 1.0:
+        return "rejected", "the next page returned the same records"
+    if overlap < 0.5:
+        return "confirmed", f"the next page returned {len(b)} different records"
+    return "inconclusive", f"{overlap:.0%} of records repeated on the next page"

ghost_api/probe/replay.py

@@ -0,0 +1,313 @@
+"""Call endpoints directly, outside the browser, to test hypotheses (FR-23).
+
+Safety rules:
+  - Only read-only requests are replayed: GET/HEAD, and POSTs whose URL says they are
+    searches or queries (e.g. Algolia, GraphQL queries). Never PUT, PATCH or DELETE.
+  - Every request goes through the shared rate limiter (NFR-18).
+  - robots.txt is checked per host (NFR-17).
+  - Original headers (including cookies) are reused from memory and never stored.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import json
+import re
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import parse_qsl, quote, urlencode, urlsplit, urlunsplit
+
+import httpx
+
+from ghost_api.capture import filters, personal
+from ghost_api.capture.models import CapturedRequest, RawRequest
+from ghost_api.net import RateLimiter, RobotsPolicy, fetch_robots
+from ghost_api.netguard import (
+    NetworkPolicy,
+    PolicyDenied,
+    ResponseTooLarge,
+    guarded_client,
+    read_limited,
+)
+
+# Whole words in the URL (path segments split on punctuation and camelCase). A POST is
+# read-only only if a READ word is present and no WRITE word is. Substring matching was a
+# bug: "/api/lists/7/delete" and "/api/wishlist/add" passed because they contain "list".
+READ_WORDS = {
+    "search", "query", "queries", "graphql", "filter", "filters", "find", "lookup",
+    "autocomplete", "suggest", "suggestions", "typeahead", "list", "browse", "count", "facets",
+}  # fmt: skip
+WRITE_WORDS = {
+    "delete", "remove", "destroy", "erase", "add", "create", "insert", "update", "edit",
+    "modify", "save", "submit", "send", "cancel", "buy", "purchase", "pay", "checkout", "cart",
+    "order", "subscribe", "unsubscribe", "follow", "unfollow", "like", "unlike", "vote",
+    "upvote", "downvote", "logout", "signout", "login", "signin", "register", "signup",
+    "reset", "upload", "import", "publish", "archive", "restore", "move", "rename", "share",
+    "invite", "approve", "reject", "transfer", "withdraw", "deposit", "block", "report",
+    "flag", "hide", "mark", "set", "toggle", "enable", "disable", "apply", "claim", "redeem",
+    "book", "reserve", "rate", "review", "comment", "reply", "post",
+}  # fmt: skip
+_CAMEL_RE = re.compile(r"(?<=[a-z0-9])(?=[A-Z])")
+# GraphQL mutations change data even though they go to the same URL as queries.
+_MUTATION_RE = re.compile(r"^\s*mutation\b", re.I | re.M)
+DROP_HEADERS = {"host", "content-length", "connection", "accept-encoding"}
+TIMEOUT_S = 20.0
+
+
+class ReplayRefused(Exception):
+    """The request is not safe or not allowed to replay."""
+
+
+@dataclass
+class ReplayResult:
+    status: int
+    body: Any | None  # parsed JSON, or None
+    error: str | None = None
+    captured: CapturedRequest | None = None  # redacted record of this call
+    hidden: list[dict[str, Any]] = field(default_factory=list)  # personal data hidden
+
+    @property
+    def ok(self) -> bool:
+        return self.error is None and 200 <= self.status < 300 and self.body is not None
+
+
+def url_words(url: str) -> set[str]:
+    """Lower-case words in a URL's path and query values: '/api/wishList/add?x=a' ->
+    {'api', 'wish', 'list', 'add', 'a'}."""
+    parts = urlsplit(url)
+    text = parts.path + " " + " ".join(v for _, v in parse_qsl(parts.query))
+    text = _CAMEL_RE.sub(" ", text)
+    return {w.lower() for w in re.split(r"[^A-Za-z0-9]+", text) if w and not w.isdigit()}
+
+
+def is_replayable(raw: RawRequest) -> bool:
+    """Only requests that read data. Anything with a write word in its URL is refused,
+    even a GET, because some sites change data with GET (`/logout`, `/item/7/delete`)."""
+    method = raw.method.upper()
+    words = url_words(raw.url)
+    if words & WRITE_WORDS:
+        return False
+    if method in ("GET", "HEAD"):
+        return True
+    if method == "POST" and words & READ_WORDS:
+        return not raw.body or _graphql_reads_only(raw.body)
+    return False
+
+
+GRAPHQL_READ_NAME = re.compile(
+    r"^(get|list|search|fetch|find|load|query|read|lookup|browse|view|show|count)", re.I
+)
+
+
+def _graphql_reads_only(body: str) -> bool:
+    """True when every GraphQL operation in the body only reads. A batch (a list) is
+    checked item by item. A persisted query has no text to check: it counts as reading only
+    when its name says so (getUser, ListOrders...)."""
+    try:
+        data = json.loads(body)
+    except ValueError:
+        return not _MUTATION_RE.search(body)
+    operations = data if isinstance(data, list) else [data]
+    for op in operations:
+        if not isinstance(op, dict):
+            return False
+        query = str(op.get("query") or "")
+        if query:
+            if _MUTATION_RE.search(query):
+                return False
+            continue
+        name = str(op.get("operationName") or "")
+        persisted = "extensions" in op or name  # only a hash and a name, no query text
+        if persisted and not GRAPHQL_READ_NAME.match(name):
+            return False
+    return True
+
+
+def with_overrides(
+    raw: RawRequest,
+    query: dict[str, str | None] | None = None,
+    body: dict[str, Any] | None = None,
+    drop_headers: set[str] | None = None,
+) -> RawRequest:
+    """A copy of `raw` with query params / top-level JSON body keys changed, and headers
+    removed. A value of None removes a query parameter."""
+    url = raw.url
+    if query:
+        parts = urlsplit(url)
+        pairs = dict(parse_qsl(parts.query, keep_blank_values=True))
+        for k, v in query.items():
+            if v is None:
+                pairs.pop(k, None)
+            else:
+                pairs[k] = v
+        url = urlunsplit(parts._replace(query=urlencode(pairs)))
+    new_body = raw.body
+    if body and raw.body:
+        try:
+            data = json.loads(raw.body)
+        except ValueError:
+            data = None
+        if isinstance(data, dict):
+            data.update(body)
+            new_body = json.dumps(data)
+    drop = {h.lower() for h in (drop_headers or set())}
+    headers = {k: v for k, v in raw.headers.items() if k.lower() not in drop}
+    return RawRequest(raw.method, url, headers, new_body)
+
+
+def with_path_params(raw: RawRequest, template: str, values: dict[str, str]) -> RawRequest:
+    """Fill path parameters: template /indexes/{index}/query + {"index": "Comments"}."""
+    parts = urlsplit(raw.url)
+    segments = parts.path.split("/")
+    names = template.split("/")
+    if len(segments) != len(names):
+        return raw
+    for i, name in enumerate(names):
+        if name.startswith("{") and name.endswith("}") and name[1:-1] in values:
+            segments[i] = quote(values[name[1:-1]], safe="")
+    url = urlunsplit(parts._replace(path="/".join(segments)))
+    return RawRequest(raw.method, url, raw.headers, raw.body)
+
+
+def body_keys(raw: RawRequest) -> set[str]:
+    try:
+        data = json.loads(raw.body or "")
+    except ValueError:
+        return set()
+    return set(data) if isinstance(data, dict) else set()
+
+
+def typed_value(value: str) -> object:
+    """'2' -> 2 and 'true' -> True for JSON bodies; anything else stays a string."""
+    try:
+        return json.loads(value)
+    except ValueError:
+        return value
+
+
+def apply_params(
+    sample: RawRequest, path_template: str, path_names: set[str], params: dict[str, str]
+) -> RawRequest:
+    """Put each parameter where the endpoint expects it: the path, the JSON body or the
+    query string. Used by the agent's call_endpoint and the dashboard's "Try it"."""
+    in_body = body_keys(sample) - path_names
+    path = {k: v for k, v in params.items() if k in path_names}
+    body = {k: typed_value(v) for k, v in params.items() if k in in_body}
+    query: dict[str, str | None] = {
+        k: v for k, v in params.items() if k not in path_names and k not in in_body
+    }
+    request = with_path_params(sample, path_template, path)
+    return with_overrides(request, query=query or None, body=body or None)
+
+
+class Replayer:
+    def __init__(
+        self,
+        target_host: str,
+        limiter: RateLimiter,
+        robots: RobotsPolicy | None = None,
+        client: httpx.Client | None = None,
+        robots_fetcher: Callable[[str], RobotsPolicy] = fetch_robots,
+        max_requests: int = 80,
+        policy: NetworkPolicy | None = None,
+    ) -> None:
+        """`policy`: where requests may go, checked on every redirect hop (default: decided
+        from the target, so a public site can't send replays to a private address)."""
+        self.target_host = target_host
+        self.limiter = limiter
+        self._robots: dict[str, RobotsPolicy] = {}
+        if robots is not None:
+            self._robots[robots.host] = robots
+        self._fetch_robots = robots_fetcher
+        self.policy = policy or NetworkPolicy.for_target(f"https://{target_host}/")
+        self._client = client or guarded_client(
+            self.policy, timeout=TIMEOUT_S, follow_redirects=False
+        )
+        self.max_requests = max_requests
+        # Called for every request sent or refused: (method, url, status, decision). The
+        # audit log (audit.py) uses it.
+        self.on_sent: Callable[[str, str, int | None, str], None] | None = None
+        self.requests_made = 0
+
+    def close(self) -> None:
+        self._client.close()
+
+    def _note(self, raw: RawRequest, status: int | None, decision: str) -> None:
+        if self.on_sent is not None:
+            # The audit log must never stop a request from being reported.
+            with contextlib.suppress(OSError):
+                self.on_sent(raw.method, raw.url, status, decision)
+
+    def robots_allows(self, url: str) -> bool:
+        host = (urlsplit(url).hostname or "").lower()
+        if host not in self._robots:
+            self._robots[host] = self._fetch_robots(url)
+        return self._robots[host].allows(url)
+
+    def send(self, raw: RawRequest, trusted_read: bool = False) -> ReplayResult:
+        """Send a read-only request. `trusted_read`: the user checked this endpoint only
+        reads (e.g. a POST search, see `generate --allow`); otherwise the URL must say so."""
+        if not trusted_read and not is_replayable(raw):
+            self._note(raw, None, "refused: may change data")
+            raise ReplayRefused(f"{raw.method} requests that may change data are never replayed")
+        if self.requests_made >= self.max_requests:
+            self._note(raw, None, "refused: request budget used up")
+            raise ReplayRefused(f"probe budget of {self.max_requests} requests used up")
+        if not self.robots_allows(raw.url):
+            self._note(raw, None, "refused: robots.txt or project rules")
+            raise ReplayRefused("disallowed by robots.txt")
+
+        self.limiter.wait()
+        self.requests_made += 1
+        # Browsers report HTTP/2 pseudo-headers (":authority", ":path"...); they can't be
+        # sent as normal headers, and httpx derives them from the URL anyway.
+        headers = {
+            k: v
+            for k, v in raw.headers.items()
+            if k.lower() not in DROP_HEADERS and not k.startswith(":")
+        }
+        try:  # the default client checks the destination of every hop (netguard.py)
+            with self._client.stream(
+                raw.method,
+                raw.url,
+                headers=headers,
+                content=raw.body.encode() if raw.body else None,
+            ) as resp:
+                text = read_limited(resp).text
+        except PolicyDenied as exc:
+            self._note(raw, None, "refused: private network")
+            raise ReplayRefused(str(exc)) from exc
+        except ResponseTooLarge as exc:
+            self._note(raw, resp.status_code, "sent: reply too large, not read")
+            return ReplayResult(resp.status_code, None, str(exc))
+        except httpx.HTTPError as exc:
+            self._note(raw, 0, f"sent: network error ({type(exc).__name__})")
+            return ReplayResult(0, None, f"network error: {type(exc).__name__}")
+        self._note(raw, resp.status_code, "sent")
+
+        try:
+            body = filters.parse_response_json(text)
+        except ValueError:
+            body = None
+        ctype = resp.headers.get("content-type")
+        stored_body, personal_data = (
+            personal.hide_body(filters.redact_body(filters.strip_xssi(text), ctype))
+            if body is not None
+            else (None, [])
+        )
+        captured = CapturedRequest(
+            method=raw.method,
+            url=filters.redact_url(raw.url),
+            status=resp.status_code,
+            resource_type="probe",
+            req_headers=filters.redact_headers(headers),
+            req_body=filters.redact_body(raw.body, headers.get("content-type")),
+            req_content_type=headers.get("content-type"),
+            resp_content_type=ctype,
+            resp_body=stored_body,
+            personal=personal_data,
+            third_party=filters.is_third_party(raw.url, self.target_host),
+        )
+        error = None if body is not None else "response is not JSON"
+        return ReplayResult(resp.status_code, body, error, captured)