responsible-request 0.4.0__py3-none-any.whl

responsible_request/config.py

@@ -0,0 +1,266 @@
+"""User-facing configuration objects."""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field, replace
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Literal
+
+from .logfiles import COMPRESSIONS, compression_of
+
+if TYPE_CHECKING:
+    from .estimator import LoadEstimator
+    from .records import RequestRecord
+
+Metric = Literal["latency_per_token", "latency", "ttfb"]
+
+
+@dataclass(frozen=True)
+class ThrottleConfig:
+    """How requests are paced and how the pace adapts to the observed load.
+
+    Requests are spaced evenly at the current RPM (no bursts). The RPM starts at ``start_rpm``,
+    ramps up towards ``max_rpm`` while latency stays close to the baseline, and drops to
+    ``min_rpm`` as soon as latency reaches ``high_ratio`` x baseline (or the server returns
+    429/5xx/timeouts).
+    """
+
+    max_rpm: float = 15
+    """Upper bound on requests per minute when the endpoint is idle."""
+    min_rpm: float = 1
+    """Requests per minute while other users are active. Also serves as probing rate."""
+    start_rpm: float = 2
+    """Initial rate while the latency baseline is being established (clamped to [min, max])."""
+    max_concurrency: int = 32
+    """Maximum number of in-flight requests per model, regardless of RPM."""
+
+    high_ratio: float = 3.0
+    """Throttle to ``min_rpm`` when current latency >= ``high_ratio`` x baseline."""
+    recover_ratio: float = 1.5
+    """Ramp up only while current latency < ``recover_ratio`` x baseline (dead band in between)."""
+    cooldown_s: float = 120
+    """Minimum time spent at ``min_rpm`` after the last high-load signal."""
+    ramp_factor: float = 1.5
+    """Multiplicative RPM increase per ramp step."""
+    ramp_interval_s: float = 30
+    """Minimum time between two ramp steps."""
+
+    metric: Metric | Callable[[RequestRecord], float | None] = "latency_per_token"
+    """Latency signal: ``latency_per_token`` (latency / max(completion tokens, ``min_tokens``);
+    uses time-to-first-byte for streamed requests), ``latency`` (raw), ``ttfb``, or a callable."""
+    min_tokens: int = 16
+    """Floor for the token count used by the ``latency_per_token`` metric."""
+    window: int = 10
+    """Number of recent observations whose median forms the current latency signal."""
+    warmup_requests: int = 20
+    """Observations required before the baseline is trusted and the RPM may ramp up."""
+    baseline: float | None = None
+    """Pin the baseline (in units of ``metric``) instead of estimating it."""
+    baseline_percentile: float = 10
+    """Baseline = this percentile of the observations inside ``baseline_window_s``."""
+    baseline_window_s: float = 1800
+    """Time span of observations used for the baseline (samples taken while throttled are
+    excluded, so the baseline freezes during high-load periods)."""
+
+    observe_paths: tuple[str, ...] = ("/chat/completions", "/embeddings")
+    """Endpoints whose latency feeds the load estimate. Other endpoints are paced and logged,
+    and react to 429/5xx, but do not adapt to latency."""
+    ignore_server_tools: bool = True
+    """Exclude requests using server-side tools (e.g. MCP, web search) from the load estimate,
+    since their latency includes external calls."""
+    adaptive: bool = True
+    """If False, requests are paced at a constant ``max_rpm``; a 429 then pauses the lane (see
+    ``backoff_initial_s``) instead of lowering the rate."""
+    backoff_initial_s: float = 1.0
+    """Fixed mode: pause after a 429 without a usable ``Retry-After`` header; doubles with every
+    further 429 until a request succeeds."""
+    backoff_max_s: float = 60.0
+    """Fixed mode: upper bound of that backoff (an explicit ``Retry-After`` is obeyed as is)."""
+    backoff_jitter_s: float = 1.0
+    """Fixed mode: random extra time in ``[0, backoff_jitter_s]`` added to every pause."""
+    estimator_factory: Callable[[ThrottleConfig], LoadEstimator] | None = None
+    """Custom load estimator (see :class:`responsible_request.estimator.LoadEstimator`)."""
+
+    def __post_init__(self) -> None:
+        if not 0 < self.min_rpm <= self.max_rpm:
+            raise ValueError("require 0 < min_rpm <= max_rpm")
+        if not 1 < self.recover_ratio < self.high_ratio:
+            raise ValueError("require 1 < recover_ratio < high_ratio")
+        if self.ramp_factor <= 1:
+            raise ValueError("ramp_factor must be > 1")
+        if self.max_concurrency < 1 or self.window < 1:
+            raise ValueError("max_concurrency and window must be >= 1")
+        if not 0 < self.baseline_percentile < 100:
+            raise ValueError("baseline_percentile must be in (0, 100)")
+        if not 0 <= self.backoff_initial_s <= self.backoff_max_s or self.backoff_jitter_s < 0:
+            raise ValueError("require 0 <= backoff_initial_s <= backoff_max_s and jitter >= 0")
+
+    @property
+    def initial_rpm(self) -> float:
+        if not self.adaptive:
+            return self.max_rpm
+        return min(self.max_rpm, max(self.min_rpm, self.start_rpm))
+
+    @classmethod
+    def fixed(cls, rpm: float, max_concurrency: int = 32, **kwargs: Any) -> ThrottleConfig:
+        """A plain rate limiter at a constant ``rpm`` (no load adaptation).
+
+        A 429 pauses the affected model for ``Retry-After`` (or an exponential backoff) plus
+        jitter. Further keyword arguments (e.g. ``backoff_max_s``) are passed to the constructor.
+        """
+        return cls(
+            max_rpm=rpm,
+            min_rpm=rpm,
+            start_rpm=rpm,
+            max_concurrency=max_concurrency,
+            adaptive=False,
+            **kwargs,
+        )
+
+
+FIELD_GROUPS = (
+    "timing",
+    "usage",
+    "throttle",
+    "response_meta",
+    "params",
+    "request_body",
+    "response_body",
+)
+"""Record field groups that can be toggled in :class:`LogConfig`. Core fields (id, timestamp,
+path, model, status, error, tags) are always logged."""
+
+
+@dataclass(frozen=True)
+class LogConfig:
+    """Where and what to log.
+
+    The console only receives throttle changes, a periodic summary, and warnings. Every request
+    is written as one structured record to the configured JSONL file and/or SQLite database.
+    """
+
+    jsonl: str | Path | None = None
+    """Append one JSON object per request to this file."""
+    sqlite: str | Path | None = None
+    """Insert one row per request into the ``requests`` table of this SQLite database."""
+    fields: Mapping[str, bool] = field(default_factory=dict)
+    """Toggle record field groups (see ``FIELD_GROUPS``); groups not mentioned are logged."""
+    summary_interval_s: float | None = 60
+    """Emit an INFO summary line per model at most this often (None disables)."""
+    console_level: str | None = None
+    """If set, replace loguru's default stderr handler with one at this level. If None, the
+    console handlers are left untouched."""
+    rotation: str | int | None = "500 MB"
+    """Passed to loguru for the JSONL file."""
+    compression: str | None = None
+    """Compress JSONL segments once they are rotated away: ``"gz"``, ``"bz2"``, ``"xz"`` or
+    ``"zst"`` (zstd: Python 3.14+ or ``pip install responsible-request[zstd]``). Without
+    rotation, the file is compressed when its sink is removed. The cache, cost budget and
+    :func:`~responsible_request.load_records` read compressed segments too."""
+
+    def __post_init__(self) -> None:
+        unknown = set(self.fields) - set(FIELD_GROUPS)
+        if unknown:
+            raise ValueError(f"unknown field groups {sorted(unknown)}; valid: {FIELD_GROUPS}")
+        if self.compression is not None and self.compression not in COMPRESSIONS:
+            raise ValueError(f"unknown compression {self.compression!r}; valid: {COMPRESSIONS}")
+        if self.jsonl is not None and compression_of(Path(self.jsonl)):
+            raise ValueError("log to a plain .jsonl file and set compression= for its segments")
+
+    def enabled(self, group: str) -> bool:
+        return self.fields.get(group, True)
+
+    @classmethod
+    def minimal(cls, **kwargs: Any) -> LogConfig:
+        """Log timing, usage and throttle state, but no parameters or message contents."""
+        off = {"params": False, "request_body": False, "response_body": False}
+        return cls(fields=off, **kwargs)
+
+    @classmethod
+    def full(cls, **kwargs: Any) -> LogConfig:
+        """Log everything (the default)."""
+        return cls(**kwargs)
+
+    def with_fields(self, **toggles: bool) -> LogConfig:
+        return replace(self, fields={**self.fields, **toggles})
+
+
+@dataclass(frozen=True)
+class CacheConfig:
+    """Answer requests from previously logged records instead of sending them again.
+
+    A request is served from the cache if a successful (HTTP 200) record with the same cache key
+    exists. The key is a hash of the URL and the complete JSON request body (model, messages and
+    all parameters, after ``default_params`` are applied), plus the values of ``key_tags``.
+    Streamed requests are never served from the cache. Cache hits are logged with
+    ``cache_hit=True`` and neither wait for nor affect the throttle.
+    """
+
+    sqlite: str | Path | None = None
+    """Read cached responses from this SQLite database (defaults to ``LogConfig.sqlite``)."""
+    jsonl: str | Path | None = None
+    """Read cached responses from this JSONL file (defaults to ``LogConfig.jsonl``)."""
+    key_tags: tuple[str, ...] = ()
+    """Tags (see :func:`responsible_request.tags`) that are part of the cache key, e.g.
+    ``("sample",)`` to draw several independent samples for the same request."""
+
+
+@dataclass(frozen=True)
+class Price:
+    """Token prices in USD per million tokens (used when the response reports no cost)."""
+
+    input: float
+    output: float
+    cached_input: float | None = None
+    """Price of cached prompt tokens; defaults to ``input``."""
+
+    def cost(
+        self, prompt_tokens: int | None, completion_tokens: int | None, cached_tokens: int | None
+    ) -> float:
+        prompt, cached = prompt_tokens or 0, min(cached_tokens or 0, prompt_tokens or 0)
+        cached_price = self.input if self.cached_input is None else self.cached_input
+        total = (prompt - cached) * self.input + cached * cached_price
+        return (total + (completion_tokens or 0) * self.output) / 1_000_000
+
+
+@dataclass(frozen=True)
+class CostConfig:
+    """Track the cost of every request and optionally stop sending requests above a budget.
+
+    The cost of a request is the one reported by the provider (OpenRouter's ``usage.cost``, the
+    LiteLLM ``x-litellm-response-cost`` header, or a custom extractor), else it is computed from
+    ``prices`` and the token usage. Cache hits cost nothing.
+    """
+
+    budget_usd: float | None = None
+    """Refuse new requests (raising :class:`responsible_request.BudgetExceeded`) once this much
+    has been spent. Requests already in flight still complete, so the budget can be overshot by
+    their cost. Cache hits are always served."""
+    prices: Mapping[str, Price] = field(default_factory=dict)
+    """Fallback prices per model name, for providers that don't report a cost."""
+    include_logged: bool = True
+    """Count the cost already recorded in the request log (``LogConfig.sqlite``/``jsonl``), so
+    that restarting a script does not reset the budget."""
+
+    def __post_init__(self) -> None:
+        if self.budget_usd is not None and self.budget_usd < 0:
+            raise ValueError("budget_usd must be >= 0")
+
+
+@dataclass(frozen=True)
+class RunConfig:
+    """Describes a run: one client (or ``http_client``) from creation to close.
+
+    Every request record carries the ``run_id`` of its run. On the first request a run manifest
+    (versions, git commit, command line, all configurations, ``default_params``, ``metadata``) is
+    written to the ``runs`` table of the SQLite log, or to ``<name>.runs.jsonl`` next to the
+    JSONL log.
+    """
+
+    name: str | None = None
+    """Human-readable name of the run, e.g. the experiment."""
+    metadata: Mapping[str, Any] = field(default_factory=dict)
+    """Anything else worth recording (dataset version, hyperparameters, notes, ...)."""
+    capture_git: bool = True
+    """Record the git commit and dirty flag of the working directory."""

responsible_request/controller.py

@@ -0,0 +1,79 @@
+"""Decide the target request rate from the load signal."""
+
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from dataclasses import dataclass
+from enum import Enum
+
+from .config import ThrottleConfig
+
+
+class State(str, Enum):
+    WARMUP = "warmup"  # establishing the baseline at start_rpm
+    NORMAL = "normal"  # endpoint looks idle: ramp up to max_rpm
+    THROTTLED = "throttled"  # other users detected: stay at min_rpm for at least the cooldown
+    RECOVERING = "recovering"  # load cleared: ramping back up
+    FIXED = "fixed"  # adaptation disabled
+
+
+@dataclass(frozen=True)
+class Transition:
+    old_state: State
+    new_state: State
+    old_rpm: float
+    new_rpm: float
+    reason: str
+
+
+class ThrottleController:
+    """State machine with hysteresis mapping load ratios to a requests-per-minute target.
+
+    * ratio >= ``high_ratio`` or a congestion error: drop to ``min_rpm`` immediately.
+    * ``recover_ratio`` <= ratio < ``high_ratio``: hold the current rate (dead band).
+    * ratio < ``recover_ratio``: multiply the rate by ``ramp_factor`` every ``ramp_interval_s``
+      up to ``max_rpm`` (after the cooldown, if throttled).
+    """
+
+    def __init__(self, config: ThrottleConfig, clock: Callable[[], float] = time.monotonic) -> None:
+        self.config = config
+        self._clock = clock
+        self.state = State.WARMUP if config.adaptive else State.FIXED
+        self.rpm = config.initial_rpm
+        self._throttled_at = -float("inf")
+        self._last_ramp = clock()
+
+    def update(self, ratio: float | None, *, congested: bool = False) -> Transition | None:
+        """Process one observation. Returns the transition if the state or the rate changed."""
+        if self.state is State.FIXED:
+            return None
+        cfg, now = self.config, self._clock()
+        old_state, old_rpm = self.state, self.rpm
+        reason = ""
+
+        if congested or (ratio is not None and ratio >= cfg.high_ratio):
+            self._throttled_at = now
+            self.state, self.rpm = State.THROTTLED, cfg.min_rpm
+            reason = "server errors" if congested else f"load {ratio:.1f}x baseline"
+        elif self.state is State.WARMUP:
+            if ratio is not None:
+                self.state, self._last_ramp = State.NORMAL, now
+                reason = "baseline established"
+        elif self.state is State.THROTTLED:
+            cooled_down = now - self._throttled_at >= cfg.cooldown_s
+            if cooled_down and ratio is not None and ratio < cfg.recover_ratio:
+                self.state, self._last_ramp = State.RECOVERING, now
+                reason = f"load {ratio:.1f}x baseline, recovering"
+        elif ratio is not None and ratio < cfg.recover_ratio:
+            if now - self._last_ramp >= cfg.ramp_interval_s and self.rpm < cfg.max_rpm:
+                self.rpm = min(cfg.max_rpm, self.rpm * cfg.ramp_factor)
+                self._last_ramp = now
+                reason = f"load {ratio:.1f}x baseline, ramping up"
+            if self.state is State.RECOVERING and self.rpm >= cfg.max_rpm:
+                self.state = State.NORMAL
+                reason = reason or "fully recovered"
+
+        if self.state is old_state and self.rpm == old_rpm:
+            return None
+        return Transition(old_state, self.state, old_rpm, self.rpm, reason)

responsible_request/cost.py

@@ -0,0 +1,96 @@
+"""Per-request cost and the budget guard."""
+
+from __future__ import annotations
+
+import openai
+from loguru import logger
+
+from .config import CostConfig, LogConfig
+from .records import RequestRecord
+from .sources import open_source
+
+
+class BudgetExceeded(openai.OpenAIError):
+    """Raised instead of sending a request once ``CostConfig.budget_usd`` has been spent."""
+
+    def __init__(self, spent_usd: float, budget_usd: float) -> None:
+        super().__init__(f"budget of ${budget_usd:.4f} exhausted (${spent_usd:.4f} spent)")
+        self.spent_usd = spent_usd
+        self.budget_usd = budget_usd
+
+
+class CostTracker:
+    """Adds up the cost of the requests of one client. Available as ``client.cost``."""
+
+    def __init__(self, config: CostConfig, log: LogConfig | None = None) -> None:
+        self.config = config
+        self.logged_usd = 0.0  # spent before this client was created, according to the log
+        if config.include_logged:
+            source = open_source(log)
+            if source is not None:
+                try:
+                    self.logged_usd = source.total_cost()
+                finally:
+                    source.close()
+        self.session_usd = 0.0
+        self.priced = 0
+        self.unpriced = 0
+        self._warned: set[str | None] = set()
+        self._exhausted_logged = False
+
+    @property
+    def spent_usd(self) -> float:
+        return self.logged_usd + self.session_usd
+
+    @property
+    def remaining_usd(self) -> float | None:
+        if self.config.budget_usd is None:
+            return None
+        return max(0.0, self.config.budget_usd - self.spent_usd)
+
+    def check(self) -> None:
+        """Raise :class:`BudgetExceeded` if the budget is exhausted."""
+        budget = self.config.budget_usd
+        if budget is None or self.spent_usd < budget:
+            return
+        if not self._exhausted_logged:
+            self._exhausted_logged = True
+            logger.warning(
+                "budget of ${:.4f} exhausted (${:.4f} spent): refusing new requests",
+                budget,
+                self.spent_usd,
+            )
+        raise BudgetExceeded(self.spent_usd, budget)
+
+    def add(self, record: RequestRecord) -> None:
+        """Fill in ``record.cost_usd`` if the provider did not report it, and count it."""
+        if record.cost_usd is None:
+            price = self.config.prices.get(record.model or "")
+            if price is None and record.response_model:
+                price = self.config.prices.get(record.response_model)
+            if price is not None and record.prompt_tokens is not None:
+                record.cost_usd = price.cost(
+                    record.prompt_tokens, record.completion_tokens, record.cached_tokens
+                )
+        if record.cost_usd is not None:
+            self.priced += 1
+            self.session_usd += record.cost_usd
+        elif record.status_code == 200:
+            self.unpriced += 1
+            if self.config.budget_usd is not None and record.model not in self._warned:
+                self._warned.add(record.model)
+                logger.warning(
+                    "{}: no cost reported and no price configured; the budget does not see "
+                    "these requests (set CostConfig(prices=...))",
+                    record.model or record.path,
+                )
+
+    def stats(self) -> dict[str, float | int | None]:
+        return {
+            "spent_usd": round(self.spent_usd, 6),
+            "session_usd": round(self.session_usd, 6),
+            "budget_usd": self.config.budget_usd,
+            "remaining_usd": self.remaining_usd,
+            "priced": self.priced,
+            "unpriced": self.unpriced,
+        }

responsible_request/estimator.py

@@ -0,0 +1,113 @@
+"""Load estimation: turn observed request latencies into a load ratio (current / baseline)."""
+
+from __future__ import annotations
+
+import math
+import statistics
+import time
+from collections import deque
+from collections.abc import Callable
+from typing import Protocol, runtime_checkable
+
+from .config import ThrottleConfig
+from .records import RequestRecord
+
+
+@runtime_checkable
+class LoadEstimator(Protocol):
+    """Anything that can judge the endpoint's load from completed requests.
+
+    Implement this to use a different signal (e.g. queue depth from a Prometheus endpoint) and
+    pass it via ``ThrottleConfig(estimator_factory=...)``.
+    """
+
+    def observe(self, record: RequestRecord, *, throttled: bool) -> None:
+        """Feed one successful, eligible request. ``throttled`` is True while at min RPM."""
+
+    def load_ratio(self) -> float | None:
+        """Current load relative to an idle endpoint (1.0 = idle); None while unknown."""
+
+    @property
+    def baseline(self) -> float | None: ...
+
+    def reset(self) -> None:
+        """Forget the learned baseline."""
+
+
+def metric_value(record: RequestRecord, config: ThrottleConfig) -> float | None:
+    """Compute the latency signal of one request according to ``config.metric``."""
+    metric = config.metric
+    if callable(metric):
+        return metric(record)
+    if metric == "latency":
+        return record.latency_s
+    if metric == "ttfb":
+        return record.ttfb_s
+    # latency_per_token: streamed requests expose the queueing delay directly as TTFB.
+    if record.stream and record.ttfb_s is not None:
+        return record.ttfb_s
+    if record.latency_s is None:
+        return None
+    tokens = record.completion_tokens or 0
+    return record.latency_s / max(tokens, config.min_tokens)
+
+
+def _percentile(values: list[float], pct: float) -> float:
+    ordered = sorted(values)
+    k = (len(ordered) - 1) * pct / 100
+    lo, hi = math.floor(k), math.ceil(k)
+    return ordered[lo] + (ordered[hi] - ordered[lo]) * (k - lo)
+
+
+class LatencyEstimator:
+    """Compare the median of the last ``window`` observations to a low-percentile baseline.
+
+    The baseline is the ``baseline_percentile``-th percentile of observations made within
+    ``baseline_window_s``. Observations made while throttled are not added to the baseline, so a
+    long busy period cannot turn into the new "normal". The window is measured relative to the
+    newest baseline sample, which means the baseline freezes (rather than expires) meanwhile.
+    """
+
+    def __init__(self, config: ThrottleConfig, clock: Callable[[], float] = time.monotonic) -> None:
+        self.config = config
+        self._clock = clock
+        self._recent: deque[float] = deque(maxlen=config.window)
+        self._samples: deque[tuple[float, float]] = deque(maxlen=5000)
+        self._pinned: float | None = config.baseline
+
+    def observe(self, record: RequestRecord, *, throttled: bool) -> None:
+        value = metric_value(record, self.config)
+        if value is None or value <= 0:
+            return
+        self._recent.append(value)
+        if not throttled:
+            now = self._clock()
+            self._samples.append((now, value))
+            while self._samples and self._samples[0][0] < now - self.config.baseline_window_s:
+                self._samples.popleft()
+
+    @property
+    def baseline(self) -> float | None:
+        if self._pinned is not None:
+            return self._pinned
+        if len(self._samples) < self.config.warmup_requests:
+            return None
+        return _percentile([v for _, v in self._samples], self.config.baseline_percentile)
+
+    def current(self) -> float | None:
+        return statistics.median(self._recent) if self._recent else None
+
+    def load_ratio(self) -> float | None:
+        baseline, current = self.baseline, self.current()
+        if baseline is None or current is None or baseline <= 0:
+            return None
+        return current / baseline
+
+    def pin_baseline(self, value: float | None) -> None:
+        """Use ``value`` as the baseline (None returns to estimating it)."""
+        self._pinned = value
+
+    def reset(self) -> None:
+        self._pinned = self.config.baseline
+        self._samples.clear()
+        self._recent.clear()

responsible_request/helpers/__init__.py

@@ -0,0 +1,21 @@
+from .batch import calibrate, run_batch, run_batch_sync
+from .messages import cached_system_message
+from .params import reproducible
+from .structured import (
+    StructuredOutputError,
+    extract_json,
+    response_format_from_model,
+    structured,
+)
+
+__all__ = [
+    "StructuredOutputError",
+    "cached_system_message",
+    "calibrate",
+    "extract_json",
+    "reproducible",
+    "response_format_from_model",
+    "run_batch",
+    "run_batch_sync",
+    "structured",
+]