sonde 0.1.0__py3-none-any.whl

sonde/__init__.py

@@ -0,0 +1,25 @@
+"""sonde — probe any HTTP API for its rate limits, burst ceiling, and full-scrape time."""
+
+__version__ = "0.1.0"
+
+# Re-exported after __version__ so core.py's `from . import __version__` resolves.
+from .endpoint import (  # noqa: E402
+    Endpoint,
+    PageResult,
+    RequestSpec,
+    add_pagination_args,
+    pagination_from_args,
+    register,
+)
+from .provider import Provider  # noqa: E402
+
+__all__ = [
+    "__version__",
+    "Endpoint",
+    "RequestSpec",
+    "PageResult",
+    "register",
+    "Provider",
+    "add_pagination_args",
+    "pagination_from_args",
+]

sonde/__main__.py

@@ -0,0 +1,5 @@
+"""Entry point: `python -m sonde <endpoint> [options]`."""
+
+from .cli import main
+
+main()

sonde/cli.py

@@ -0,0 +1,343 @@
+"""
+cli.py — argument parsing, endpoint selection, and run orchestration.
+
+Usage:
+    python -m sonde <endpoint> [common options] [endpoint options]
+
+The common rate-limit options are shared across every endpoint; each registered
+endpoint contributes its own options as a subcommand.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+from collections.abc import Iterable
+from typing import Any
+
+from . import (
+    core,
+    endpoint,
+    endpoints,  # noqa: F401  (import registers all endpoints)
+    phases,
+)
+from .logconfig import register_log_secrets, setup_logging
+
+logger = logging.getLogger(__name__)
+
+# Header names whose values are credentials and must be kept out of logs.
+_SECRET_HEADER_KEYS = frozenset({"authorization", "cookie", "proxy-authorization", "x-api-key"})
+
+
+def _secret_variants(value: str) -> Iterable[str]:
+    """The full header value plus the bare credential inside it, so a target that
+    echoes just the token (no `Bearer `, no `.ROBLOSECURITY=`) is still redacted."""
+    yield value
+    # Only emit a bare variant if it's long enough to be a real credential, so a
+    # short prefix can't over-redact unrelated log text.
+    after_scheme = value.split(" ", 1)  # "Bearer <tok>" -> "<tok>"
+    if len(after_scheme) == 2 and len(after_scheme[1]) >= 8:
+        yield after_scheme[1]
+    after_eq = value.split("=", 1)  # ".ROBLOSECURITY=<cookie>" -> "<cookie>"
+    if len(after_eq) == 2 and len(after_eq[1]) >= 8:
+        yield after_eq[1]
+
+
+def _int_list(raw: str) -> list[int]:
+    """argparse type for a comma-separated list of ints (clean exit-2 on bad input)."""
+    try:
+        vals = [int(x) for x in raw.split(",") if x.strip()]
+    except ValueError as e:
+        raise argparse.ArgumentTypeError(f"comma-separated integers required: {e}")
+    if not vals:
+        raise argparse.ArgumentTypeError("at least one value required")
+    return vals
+
+
+def _float_list(raw: str) -> list[float]:
+    """argparse type for a comma-separated list of floats (clean exit-2 on bad input)."""
+    try:
+        vals = [float(x) for x in raw.split(",") if x.strip()]
+    except ValueError as e:
+        raise argparse.ArgumentTypeError(f"comma-separated numbers required: {e}")
+    if not vals:
+        raise argparse.ArgumentTypeError("at least one value required")
+    return vals
+
+
+def build_common_parser() -> argparse.ArgumentParser:
+    """All endpoint-agnostic probe options (shared by every subcommand)."""
+    c = argparse.ArgumentParser(add_help=False)
+    g = c.add_argument_group("rate-limit probe options")
+    g.add_argument(
+        "--max-requests", type=int, default=1200, help="hard global cap across all phases (safety)"
+    )
+    g.add_argument(
+        "--seq-cap", type=int, default=150, help="max sequential requests before giving up on a 429"
+    )
+    g.add_argument("--skip-burst", action="store_true")
+    g.add_argument(
+        "--burst-sizes",
+        type=_int_list,
+        default=[10, 20, 40, 80],
+        help="comma list of concurrent burst sizes (default: 10,20,40,80)",
+    )
+    g.add_argument(
+        "--burst-cooldown",
+        type=float,
+        default=60.0,
+        help="fallback seconds between bursts if the window can't be measured",
+    )
+    g.add_argument(
+        "--recovery-step",
+        type=float,
+        default=0.25,
+        help="first poll delay when measuring the throttle window (grows geometrically)",
+    )
+    g.add_argument(
+        "--recovery-max",
+        type=float,
+        default=90.0,
+        help="give up measuring the window after this many seconds",
+    )
+    g.add_argument(
+        "--recovery-polls",
+        type=int,
+        default=15,
+        help="max polls during recovery measurement (bounds request count)",
+    )
+    g.add_argument("--skip-sweep", action="store_true", help="skip the sustained-interval sweep")
+    g.add_argument(
+        "--force-sweep",
+        action="store_true",
+        help="run the sweep even when authoritative headers are present "
+        "(skipped by default in that case; it's redundant and slow)",
+    )
+    g.add_argument(
+        "--sweep-intervals",
+        type=_float_list,
+        default=[8, 5, 3, 2, 1.2, 0.6, 0.3, 0.15],
+        help="inter-request intervals (s) to test, SLOW->FAST (default: "
+        "8,5,3,2,1.2,0.6,0.3,0.15). Wide so it can bracket slow limits; only "
+        "used as a fallback when headers are missing.",
+    )
+    g.add_argument(
+        "--sweep-count", type=int, default=20, help="paced requests per interval after draining"
+    )
+    g.add_argument(
+        "--sweep-drain",
+        type=int,
+        default=500,
+        help="cap on rapid requests used to empty the bucket before each interval; "
+        "the drain runs until empty or this cap",
+    )
+    g.add_argument(
+        "--sweep-tolerance",
+        type=float,
+        default=0.1,
+        help="max fraction of 429s from empty for an interval to count as sustainable",
+    )
+    g.add_argument(
+        "--margin",
+        type=float,
+        default=0.8,
+        help="safety margin: recommended interval = floor / margin (0.8 => 25%% slower)",
+    )
+    g.add_argument(
+        "--output",
+        default="sonde_report.json",
+        help="report output file (use '-' for stdout)",
+    )
+
+    vq = c.add_mutually_exclusive_group()
+    vq.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="show per-request detail (sets log level to DEBUG)",
+    )
+    vq.add_argument(
+        "-q",
+        "--quiet",
+        action="store_true",
+        help="only show warnings and errors (sets log level to WARNING)",
+    )
+    c.add_argument(
+        "--log-format",
+        choices=["plain", "json"],
+        default="plain",
+        help="log line format: plain (message-only, default) or json (structured)",
+    )
+    return c
+
+
+def build_parser() -> argparse.ArgumentParser:
+    common = build_common_parser()
+    p = argparse.ArgumentParser(
+        prog="sonde",
+        description="Probe any HTTP API for its rate limits. Pick an endpoint subcommand.",
+    )
+    sub = p.add_subparsers(dest="endpoint", required=True, metavar="ENDPOINT")
+    for name, cls in sorted(endpoint.all_endpoints().items()):
+        sp = sub.add_parser(name, parents=[common], help=cls.help, description=cls.help)
+        cls.add_arguments(sp)
+    return p
+
+
+def run(args: argparse.Namespace) -> dict[str, Any]:
+    ep_cls = endpoint.get(args.endpoint)
+    if ep_cls is None:
+        raise SystemExit(f"unknown endpoint: {args.endpoint}")
+    _preflight_output(args.output)
+    ep = ep_cls.from_args(args)
+    provider = ep.provider()
+    burst_sizes = args.burst_sizes
+    sweep_intervals = sorted(args.sweep_intervals, reverse=True)
+    budget = core.Budget(max_requests=args.max_requests)
+    # base headers < provider auth < endpoint extras
+    headers = {**core.BASE_HEADERS, **provider.auth_headers(), **ep.extra_headers()}
+    # Keep our own credentials out of logs if the target echoes them back.
+    register_log_secrets(
+        variant
+        for k, v in headers.items()
+        if k.lower() in _SECRET_HEADER_KEYS
+        for variant in _secret_variants(v)
+    )
+    session = core.build_session(headers=headers)
+
+    logger.info("Endpoint : %s", ep.name)
+    logger.info("Provider : %s", provider.name)
+    logger.info(
+        "Auth     : %s",
+        "credentials set" if provider.auth_headers() else "none (anonymous)",
+    )
+    logger.info("Budget   : %s requests total", args.max_requests)
+
+    report = {"endpoint": ep.name, "provider": provider.name}
+
+    sanity, rl = phases.phase_sanity(session, ep, budget)
+    report["sanity"] = {
+        "status": sanity.status,
+        "rclass": sanity.rclass.value,
+        "items": sanity.count,
+        "headers": sanity.headers,
+    }
+    report["ratelimit_headers"] = rl
+    if sanity.rclass != core.RClass.OK:
+        logger.warning(
+            "\nAborting: no usable success response from the endpoint. "
+            "Fix auth / arguments and re-run."
+        )
+        _dump(args.output, report)
+        return report
+    page_count = sanity.count  # items per successful page, for the estimate
+
+    seq_summary, cursor_pool = phases.phase_seq(session, ep, budget, args.seq_cap)
+    report["sequential"] = seq_summary
+
+    burst_results, measured_window = [], None
+    if not args.skip_burst:
+        burst_results, measured_window = phases.phase_burst(
+            headers,
+            ep,
+            budget,
+            burst_sizes,
+            args.burst_cooldown,
+            cursor_pool,
+            args.recovery_step,
+            args.recovery_max,
+            args.recovery_polls,
+        )
+    report["burst"] = burst_results
+    report["measured_window_seconds"] = measured_window
+
+    swept_interval, sweep_rows = None, []
+    headers_authoritative = bool(rl.get("limit") and rl.get("window_s"))
+    run_sweep = (not args.skip_sweep) and (args.force_sweep or not headers_authoritative)
+    if run_sweep:
+        swept_interval, sweep_rows = phases.phase_sweep(
+            session,
+            ep,
+            budget,
+            cursor_pool,
+            sweep_intervals,
+            args.sweep_count,
+            args.sweep_drain,
+            args.sweep_tolerance,
+        )
+    elif headers_authoritative and not args.skip_sweep:
+        logger.info("\n== PHASE: sustained-interval sweep ==")
+        logger.info(
+            "  skipped: authoritative rate-limit headers already give the limit. "
+            "Use --force-sweep to run it anyway as an independent check."
+        )
+    report["sweep"] = sweep_rows
+    report["swept_floor_interval_s"] = swept_interval
+
+    report["estimate"] = phases.phase_estimate(
+        ep, page_count, seq_summary, burst_results, measured_window, swept_interval, args.margin, rl
+    )
+    report["requests_used"] = budget.used
+
+    _dump(args.output, report)
+    logger.info("\nRequests used: %s/%s", budget.used, args.max_requests)
+    if args.output != "-":
+        logger.info("Full report written to: %s", args.output)
+    return report
+
+
+def _preflight_output(path: str) -> None:
+    """Fail fast (exit 2) on an unwritable --output path before probing."""
+    if path == "-":
+        return
+    try:
+        # Append mode: tests writability without truncating an existing report.
+        # On a new path this creates a zero-byte file; if the probe is interrupted
+        # before _dump, that empty file remains (acceptable for fail-fast).
+        with open(path, "a"):
+            pass
+    except OSError as e:
+        logger.error("cannot write --output %r: %s", path, e)
+        raise SystemExit(2)
+
+
+def _dump(path: str, report: dict[str, Any]) -> None:
+    if path == "-":
+        json.dump(report, sys.stdout, indent=2)
+        sys.stdout.write("\n")
+    else:
+        with open(path, "w") as f:
+            json.dump(report, f, indent=2)
+
+
+def _aborted(report: dict[str, Any]) -> bool:
+    """True when the probe bailed because the endpoint returned no usable response
+    (non-OK sanity). main() maps this to a non-zero exit so CI can detect it."""
+    sanity = report.get("sanity")
+    return bool(sanity) and sanity.get("rclass") != core.RClass.OK.value
+
+
+def main(argv: list[str] | None = None) -> None:
+    """Exit codes: 0 success, 2 precondition failure (bad args / unwritable output /
+    endpoint returned no usable response), 1 unexpected crash, 130 interrupted."""
+    args = build_parser().parse_args(argv)
+    level = logging.DEBUG if args.verbose else logging.WARNING if args.quiet else logging.INFO
+    setup_logging(level=level, fmt=args.log_format)
+    try:
+        report = run(args)
+    except KeyboardInterrupt:
+        logger.warning("interrupted.")
+        sys.exit(130)
+    except Exception:
+        # Route crashes through the logger so --log-format json keeps stderr valid
+        # JSON and the traceback is escaped (PlainFormatter) rather than dumped raw.
+        logger.error("unexpected error", exc_info=True)
+        sys.exit(1)
+    if _aborted(report):
+        sys.exit(2)
+
+
+if __name__ == "__main__":
+    main()

sonde/core.py

@@ -0,0 +1,185 @@
+"""
+core.py — endpoint- and provider-agnostic HTTP plumbing.
+
+Response classification, rate-limit-header parsing, and auth are NOT here — those
+vary per API and live behind the Provider interface (provider.py). core only knows
+how to issue a request, time it, and hand the response to the endpoint's provider
+for classification and to the endpoint for item extraction.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from http.cookiejar import DefaultCookiePolicy
+from typing import TYPE_CHECKING, Any
+
+import requests
+from requests.adapters import HTTPAdapter
+
+from . import __version__
+
+if TYPE_CHECKING:
+    from .endpoint import Endpoint
+
+__all__ = [
+    "RClass",
+    "Budget",
+    "Result",
+    "build_session",
+    "fetch",
+    "default_rclass",
+    "interesting_headers",
+    "BASE_HEADERS",
+]
+
+BASE_HEADERS = {
+    "Accept": "application/json",
+    "User-Agent": f"sonde/{__version__} (one-time diagnostic)",
+}
+
+# Response headers worth surfacing (case-insensitive substring match).
+HEADER_SUBSTRINGS = ("ratelimit", "retry-after", "x-request", "server", "cf-ray")
+
+
+# --------------------------------------------------------------------------- #
+# Normalised response class — phases branch on this, never on raw status.
+# --------------------------------------------------------------------------- #
+class RClass(str, Enum):
+    OK = "ok"  # a usable success response
+    THROTTLED = "throttled"  # rate-limited (429, or provider-specific)
+    ERROR = "error"  # any other non-success (4xx/5xx/network)
+    BUDGET = "budget"  # local request budget exhausted (not a server response)
+
+
+def default_rclass(status: int) -> RClass:
+    """Fallback classification (also what the generic Provider uses)."""
+    if status == 200:
+        return RClass.OK
+    if status == 429:
+        return RClass.THROTTLED
+    if status == -1:
+        return RClass.BUDGET
+    return RClass.ERROR
+
+
+# --------------------------------------------------------------------------- #
+# Request budget: thread-safe hard ceiling.
+# --------------------------------------------------------------------------- #
+@dataclass
+class Budget:
+    max_requests: int
+    used: int = 0
+    _lock: threading.Lock = field(default_factory=threading.Lock, repr=False)
+
+    def take(self) -> bool:
+        with self._lock:
+            if self.used >= self.max_requests:
+                return False
+            self.used += 1
+            return True
+
+    def remaining(self) -> int:
+        with self._lock:
+            return max(0, self.max_requests - self.used)
+
+
+# --------------------------------------------------------------------------- #
+# Session
+# --------------------------------------------------------------------------- #
+def build_session(headers: dict[str, str] | None = None) -> requests.Session:
+    """Session for the serial phases; auth rides on headers so the no-write cookie
+    jar is never mutated. The burst phase builds its own httpx client."""
+    s = requests.Session()
+    s.headers.update(headers or dict(BASE_HEADERS))
+    s.cookies.set_policy(DefaultCookiePolicy(allowed_domains=[]))
+    adapter = HTTPAdapter(pool_connections=4, pool_maxsize=10, max_retries=0)
+    s.mount("https://", adapter)
+    s.mount("http://", adapter)
+    return s
+
+
+# --------------------------------------------------------------------------- #
+# Result + response handling
+# --------------------------------------------------------------------------- #
+@dataclass
+class Result:
+    status: int
+    elapsed: float
+    # None only on input -> derived from status in __post_init__ (never None after construction).
+    rclass: RClass | None = None
+    count: int = 0
+    next_cursor: Any = None
+    retry_after: float | None = None
+    headers: dict[str, str] = field(default_factory=dict)
+    error: str | None = None
+
+    def __post_init__(self) -> None:
+        if self.rclass is None:
+            self.rclass = default_rclass(self.status)
+
+
+def interesting_headers(resp: Any) -> dict[str, str]:
+    return {
+        k: v for k, v in resp.headers.items() if any(sub in k.lower() for sub in HEADER_SUBSTRINGS)
+    }
+
+
+_PARSE_ERRORS = (ValueError, KeyError, TypeError, AttributeError, IndexError)
+
+
+def _parse_response(resp: Any, elapsed: float, endpoint: Endpoint) -> Result:
+    """Classify via the endpoint's provider, then (on success) let the endpoint pull
+    item count + next cursor from the FULL response (so header-based pagination and
+    non-JSON bodies are possible)."""
+    provider = endpoint.provider()
+    rclass = provider.classify(resp)
+
+    ra = resp.headers.get("Retry-After")
+    retry_after = None
+    if ra is not None:
+        try:
+            retry_after = float(ra)
+        except (ValueError, TypeError):
+            retry_after = None
+
+    res = Result(
+        status=resp.status_code,
+        elapsed=elapsed,
+        rclass=rclass,
+        retry_after=retry_after,
+        headers=interesting_headers(resp),
+    )
+    if rclass == RClass.OK:
+        try:
+            page = endpoint.parse_page(resp)
+            res.count = page.count
+            res.next_cursor = page.next_cursor
+        except _PARSE_ERRORS as e:
+            res.error = f"OK response but parse_page failed: {e}"
+    elif rclass == RClass.ERROR and resp.status_code >= 400:
+        res.error = resp.text[:200]
+    return res
+
+
+def fetch(session: requests.Session, endpoint: Endpoint, cursor: Any, budget: Budget) -> Result:
+    """One probe request for `endpoint` at pagination position `cursor`."""
+    if not budget.take():
+        return Result(
+            status=-1, elapsed=0.0, rclass=RClass.BUDGET, error="request budget exhausted"
+        )
+
+    provider = endpoint.provider()
+    spec = endpoint.build_request(cursor)
+    params = {**provider.auth_params(), **(spec.params or {})}
+
+    t0 = time.perf_counter()
+    try:
+        resp = session.request(
+            spec.method, spec.url, params=params, json=spec.json_body, timeout=30
+        )
+    except requests.RequestException as e:
+        return Result(status=0, elapsed=time.perf_counter() - t0, rclass=RClass.ERROR, error=str(e))
+    return _parse_response(resp, time.perf_counter() - t0, endpoint)