responsible-request 0.4.0__py3-none-any.whl

responsible_request/__init__.py

@@ -0,0 +1,94 @@
+"""Responsible LLM requests for research: polite (load-aware throttling), reproducible (request
+logs, run manifests, decoding presets) and cost-safe (response cache, cost tracking, budgets).
+Works with any OpenAI-compatible endpoint (LiteLLM, OpenRouter, vLLM, Ollama, OpenAI, ...).
+
+Typical use::
+
+    import responsible_request as rr
+
+    client = rr.AsyncOpenAI(base_url=..., api_key=..., log=rr.LogConfig(sqlite="requests.db"))
+    response = await client.chat.completions.create(model=..., messages=..., **rr.reproducible())
+"""
+
+from importlib.metadata import PackageNotFoundError, version
+
+from loguru import logger
+
+from . import providers
+from .analysis import load_records, load_runs
+from .cache import ResponseCache
+from .client import AsyncOpenAI, get_throttle, http_client
+from .config import (
+    FIELD_GROUPS,
+    CacheConfig,
+    CostConfig,
+    LogConfig,
+    Price,
+    RunConfig,
+    ThrottleConfig,
+)
+from .controller import State, ThrottleController
+from .cost import BudgetExceeded, CostTracker
+from .estimator import LatencyEstimator, LoadEstimator
+from .helpers import (
+    StructuredOutputError,
+    cached_system_message,
+    calibrate,
+    extract_json,
+    reproducible,
+    response_format_from_model,
+    run_batch,
+    run_batch_sync,
+    structured,
+)
+from .logging import remove_sinks, setup_logging
+from .records import RequestRecord, tags
+from .throttle import Throttle
+from .transport import ThrottledTransport
+
+try:
+    __version__ = version("responsible-request")
+except PackageNotFoundError:  # pragma: no cover
+    __version__ = "0.0.0"
+
+# Libraries should stay silent unless the application opts in (loguru convention).
+logger.disable(__name__)
+
+__all__ = [
+    "FIELD_GROUPS",
+    "AsyncOpenAI",
+    "BudgetExceeded",
+    "CacheConfig",
+    "CostConfig",
+    "CostTracker",
+    "LatencyEstimator",
+    "LoadEstimator",
+    "LogConfig",
+    "Price",
+    "RequestRecord",
+    "ResponseCache",
+    "RunConfig",
+    "State",
+    "StructuredOutputError",
+    "Throttle",
+    "ThrottleConfig",
+    "ThrottleController",
+    "ThrottledTransport",
+    "__version__",
+    "cached_system_message",
+    "calibrate",
+    "extract_json",
+    "get_throttle",
+    "http_client",
+    "load_records",
+    "load_runs",
+    "providers",
+    "remove_sinks",
+    "reproducible",
+    "response_format_from_model",
+    "run_batch",
+    "run_batch_sync",
+    "setup_logging",
+    "structured",
+    "tags",
+]

responsible_request/_http.py

@@ -0,0 +1,31 @@
+"""Resolve the HTTP library used by the installed ``openai`` SDK.
+
+``openai<3`` is built on ``httpx``; ``openai>=3`` is built on its fork ``httpx2``. Both expose the
+same transport API, so we pick whichever one ``openai.DefaultAsyncHttpxClient`` derives from.
+
+The library's per-request ``INFO`` line ("HTTP Request: POST ...") is silenced, since every
+request is already recorded by our transport. Lower the level again to get it back.
+"""
+
+from __future__ import annotations
+
+import importlib
+import logging
+from types import ModuleType
+
+import openai
+
+
+def _resolve() -> ModuleType:
+    for base in openai.DefaultAsyncHttpxClient.__mro__:
+        root = base.__module__.split(".")[0]
+        if root in ("httpx", "httpx2"):
+            return importlib.import_module(root)
+    return importlib.import_module("httpx")  # pragma: no cover
+
+
+httpx = _resolve()
+
+logging.getLogger(httpx.__name__).setLevel(logging.WARNING)
+
+__all__ = ["httpx"]

responsible_request/analysis.py

@@ -0,0 +1,81 @@
+"""Read logged request records back for analysis."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+from .logfiles import log_segments, open_binary, strip_compression
+from .logging import runs_path
+from .records import JSON_FIELDS
+from .runs import RUN_JSON_FIELDS
+
+_JSONL_SUFFIXES = (".jsonl", ".json", ".log")
+
+
+def _is_jsonl(path: Path) -> bool:
+    return strip_compression(path).suffix in _JSONL_SUFFIXES
+
+
+def load_records(path: str | Path, *, as_dataframe: bool | None = None) -> Any:
+    """Load request records from a JSONL file or SQLite database written by this package.
+
+    For a JSONL log, the segments rotated away from it are read too (oldest first, compressed or
+    not). A compressed file (e.g. ``requests.jsonl.zst``) is read on its own.
+
+    Returns a pandas DataFrame if pandas is installed (or ``as_dataframe=True``), otherwise a
+    list of dicts.
+    """
+    path = Path(path)
+    if _is_jsonl(path):
+        rows = [row for segment in log_segments(path) for row in _read_jsonl(segment)]
+    else:
+        rows = _read_table(path, "requests", JSON_FIELDS)
+    return _output(rows, as_dataframe)
+
+
+def load_runs(path: str | Path, *, as_dataframe: bool | None = None) -> Any:
+    """Load run manifests (see :class:`responsible_request.RunConfig`) from the SQLite database,
+    or for a JSONL log (``requests.jsonl``) from ``requests.runs.jsonl`` next to it."""
+    path = Path(path)
+    if _is_jsonl(path):
+        runs = path if strip_compression(path).stem.endswith(".runs") else runs_path(path)
+        rows = _read_jsonl(runs) if runs.exists() else []
+    else:
+        rows = _read_table(path, "runs", RUN_JSON_FIELDS)
+    return _output(rows, as_dataframe)
+
+
+def _read_jsonl(path: Path) -> list[dict[str, Any]]:
+    with open_binary(path) as fh:
+        return [json.loads(line) for line in fh if line.strip()]
+
+
+def _read_table(path: Path, table: str, json_fields: tuple[str, ...]) -> list[dict[str, Any]]:
+    conn = sqlite3.connect(path)
+    conn.row_factory = sqlite3.Row
+    try:
+        rows = [dict(r) for r in conn.execute(f"SELECT * FROM {table}")]
+    except sqlite3.OperationalError:  # table does not exist (e.g. no runs in an old database)
+        rows = []
+    finally:
+        conn.close()
+    for row in rows:
+        for key in json_fields:
+            if isinstance(row.get(key), str):
+                row[key] = json.loads(row[key])
+    return rows
+
+
+def _output(rows: list[dict[str, Any]], as_dataframe: bool | None) -> Any:
+    if as_dataframe is False:
+        return rows
+    try:
+        import pandas as pd
+    except ImportError:
+        if as_dataframe:
+            raise ImportError("install pandas: pip install responsible-request[pandas]") from None
+        return rows
+    return pd.DataFrame(rows)

responsible_request/cache.py

@@ -0,0 +1,59 @@
+"""Serve requests from previously logged request records."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+from .config import CacheConfig, LogConfig
+from .sources import RecordSource, open_source
+
+
+def cache_key(url: str, body: Any, salt: dict[str, Any] | None = None) -> str:
+    """Hash of the request URL, its canonical JSON body and the salt (selected tag values)."""
+    payload = {"url": url, "body": body}
+    if salt:
+        payload["salt"] = salt
+    canonical = json.dumps(
+        payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False, default=str
+    )
+    return hashlib.sha256(canonical.encode()).hexdigest()
+
+
+class ResponseCache:
+    """Response lookup by cache key, backed by the records of a SQLite database or JSONL file."""
+
+    def __init__(self, config: CacheConfig, log: LogConfig | None = None) -> None:
+        self.config = config
+        target: CacheConfig | LogConfig | None = config
+        if config.sqlite is None and config.jsonl is None:  # read the records we write
+            target = log
+            if log is not None and not log.enabled("response_body"):
+                raise ValueError("the cache reads responses from the log: enable response_body")
+        source = open_source(target)
+        if source is None:
+            raise ValueError(
+                "the cache needs records to read from: set LogConfig(sqlite=...) or "
+                "LogConfig(jsonl=...), or CacheConfig(sqlite=...) or CacheConfig(jsonl=...)"
+            )
+        self._source: RecordSource = source
+        self.hits = 0
+        self.misses = 0
+
+    def salt(self, tags: dict[str, Any]) -> dict[str, Any]:
+        return {name: tags[name] for name in self.config.key_tags if name in tags}
+
+    def get(self, key: str) -> dict[str, Any] | None:
+        body = self._source.get(key)
+        if body is None:
+            self.misses += 1
+        else:
+            self.hits += 1
+        return body
+
+    def stats(self) -> dict[str, int]:
+        return {"hits": self.hits, "misses": self.misses}
+
+    def close(self) -> None:
+        self._source.close()

responsible_request/client.py

@@ -0,0 +1,169 @@
+"""Drop-in OpenAI client and an ``http_client`` factory for existing code."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+import openai
+from loguru import logger
+
+from .cache import ResponseCache
+from .config import CacheConfig, CostConfig, LogConfig, RunConfig, ThrottleConfig
+from .cost import BudgetExceeded, CostTracker
+from .logging import setup_logging
+from .providers import DEFAULT_EXTRACTORS, Extractor
+from .runs import Run
+from .throttle import Throttle
+from .transport import ThrottledTransport
+
+
+def http_client(
+    throttle: ThrottleConfig | Throttle | None = None,
+    log: LogConfig | bool | None = True,
+    *,
+    default_params: Mapping[str, Any] | None = None,
+    inject_stream_usage: bool = True,
+    cache: CacheConfig | bool | None = None,
+    cost: CostConfig | bool | None = None,
+    run: RunConfig | str | None = None,
+    extractors: Iterable[Extractor] = (),
+    transport: Any = None,
+    **client_kwargs: Any,
+) -> Any:
+    """Build an async HTTP client for the OpenAI SDK that paces, measures and logs requests.
+
+    Use it to add throttling to existing code::
+
+        client = openai.AsyncOpenAI(http_client=rr.http_client(rr.ThrottleConfig(max_rpm=120)))
+
+    :param throttle: Throttle configuration (or an existing :class:`Throttle` to share state).
+    :param log: ``True`` for console output only, a :class:`LogConfig` to also write request
+        records to JSONL/SQLite, ``False``/``None`` to leave logging disabled.
+    :param default_params: Chat completion parameters added to every request that does not set
+        them itself, e.g. ``rr.reproducible()``.
+    :param inject_stream_usage: Request token usage for streamed chat completions.
+    :param cache: ``True`` or a :class:`CacheConfig` to answer requests that were already
+        answered successfully from the logged records instead of sending them again.
+    :param cost: ``True`` or a :class:`CostConfig` to track the cost of every request and
+        optionally enforce a budget.
+    :param run: A :class:`RunConfig` (or just a run name) for the run manifest written on the
+        first request. Every record carries the ``run_id``.
+    :param extractors: Additional response metadata extractors (see
+        :mod:`responsible_request.providers`), run after the built-in ones.
+    :param transport: Underlying transport (defaults to a plain async HTTP transport).
+    :param client_kwargs: Passed to ``openai.DefaultAsyncHttpxClient`` (e.g. ``timeout``).
+    """
+    state = throttle if isinstance(throttle, Throttle) else Throttle(throttle)
+    log_config = LogConfig() if log is True else (log or None)
+    cache_config = CacheConfig() if cache is True else (cache or None)
+    response_cache = ResponseCache(cache_config, log_config) if cache_config is not None else None
+    cost_config = CostConfig() if cost is True else (cost or None)
+    tracker = CostTracker(cost_config, log_config) if cost_config is not None else None
+    run_config = RunConfig(name=run) if isinstance(run, str) else run
+    rr_run = Run(
+        run_config,
+        throttle=state.config,
+        log=log_config,
+        cache=cache_config,
+        cost=cost_config,
+        default_params=default_params,
+    )
+    if log_config is not None:
+        setup_logging(log_config)
+    rr_transport = ThrottledTransport(
+        state,
+        log=log_config,
+        default_params=default_params,
+        inject_stream_usage=inject_stream_usage,
+        cache=response_cache,
+        cost=tracker,
+        run=rr_run,
+        extractors=(*DEFAULT_EXTRACTORS, *extractors),
+        transport=transport,
+    )
+    client = openai.DefaultAsyncHttpxClient(transport=rr_transport, **client_kwargs)
+    client.rr_throttle = state  # type: ignore[attr-defined]
+    client.rr_cache = response_cache  # type: ignore[attr-defined]
+    client.rr_cost = tracker  # type: ignore[attr-defined]
+    client.rr_run_id = rr_run.run_id  # type: ignore[attr-defined]
+    return client
+
+
+def get_throttle(client: Any) -> Throttle:
+    """Return the :class:`Throttle` behind an OpenAI client created with :func:`http_client`."""
+    throttle = getattr(client, "throttle", None)
+    if isinstance(throttle, Throttle):
+        return throttle
+    inner = getattr(client, "_client", client)  # the SDK keeps its http client in ``_client``
+    throttle = getattr(inner, "rr_throttle", None)
+    if not isinstance(throttle, Throttle):
+        raise TypeError("client was not created with responsible_request")
+    return throttle
+
+
+class AsyncOpenAI(openai.AsyncOpenAI):
+    """``openai.AsyncOpenAI`` with load-aware throttling and request logging.
+
+    Accepts every argument of ``openai.AsyncOpenAI`` plus ``throttle``, ``log``,
+    ``default_params``, ``cache``, ``cost``, ``run`` and ``extractors`` (see :func:`http_client`).
+    The throttle state is available as ``client.throttle``; ``client.throttle.stats()`` shows the
+    current rate and load per model. ``client.cache.stats()`` counts cache hits and misses and
+    ``client.cost.stats()`` shows the spent cost (both are None unless enabled).
+    ``client.run_id`` identifies this client's records and run manifest.
+    """
+
+    throttle: Throttle
+    cache: ResponseCache | None
+    cost: CostTracker | None
+    run_id: str
+
+    def __init__(
+        self,
+        *,
+        throttle: ThrottleConfig | Throttle | None = None,
+        log: LogConfig | bool | None = True,
+        default_params: Mapping[str, Any] | None = None,
+        cache: CacheConfig | bool | None = None,
+        cost: CostConfig | bool | None = None,
+        run: RunConfig | str | None = None,
+        extractors: Iterable[Extractor] = (),
+        http_client: Any = None,
+        **kwargs: Any,
+    ) -> None:
+        if http_client is None:
+            http_client = _http_client(
+                throttle,
+                log,
+                default_params=default_params,
+                cache=cache,
+                cost=cost,
+                run=run,
+                extractors=extractors,
+            )
+        elif getattr(http_client, "rr_throttle", None) is None:
+            raise TypeError(
+                "pass throttle/log options instead of http_client, or build the http client "
+                "with responsible_request.http_client()"
+            )
+        super().__init__(http_client=http_client, **kwargs)
+        self.throttle = http_client.rr_throttle
+        self.cache = http_client.rr_cache
+        self.cost = http_client.rr_cost
+        self.run_id = http_client.rr_run_id
+
+    async def request(self, *args: Any, **kwargs: Any) -> Any:
+        try:
+            return await super().request(*args, **kwargs)
+        except openai.APIConnectionError as exc:
+            # older SDKs wrap exceptions raised by the transport
+            if isinstance(exc.__cause__, BudgetExceeded):
+                raise exc.__cause__ from None
+            raise
+
+    async def close(self) -> None:
+        await super().close()
+        await logger.complete()
+
+
+_http_client = http_client