tokenbiryani 0.2.0__py3-none-any.whl

tokenbiryani/observability/events.py

@@ -0,0 +1,171 @@
+"""Request events: the one stream the console, the CLI and the logs all read from."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import time
+from collections import deque
+from dataclasses import asdict, dataclass, field
+from typing import Any, Deque, Dict, List, Optional, Set
+
+logger = logging.getLogger("tokenbiryani")
+
+
+@dataclass
+class Attempt:
+    account_id: str
+    status: Optional[int] = None
+    kind: str = ""
+    latency: float = 0.0
+    detail: str = ""
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass
+class RequestEvent:
+    request_id: str
+    started_at: float
+    model: str = ""
+    key_name: str = ""
+    session_key: str = ""
+    streamed: bool = False
+    priority: str = "interactive"
+    #: The model the caller asked for, when pacing substituted a different one.
+    #: Empty when nothing was substituted, which is the normal case.
+    model_requested: str = ""
+    #: "messages" (the normal path) or "batch" (spilled to the Batches API)
+    via: str = "messages"
+    batch_id: Optional[str] = None
+    attempts: List[Attempt] = field(default_factory=list)
+    account_id: Optional[str] = None
+    status: Optional[int] = None
+    latency: float = 0.0
+    ttft: Optional[float] = None
+    queued_for: float = 0.0
+    #: Seconds this request was deliberately held back to protect a quota window.
+    #: Batch priority only, and only when pacing is enforcing.
+    paced_for: float = 0.0
+    input_tokens: int = 0
+    output_tokens: int = 0
+    cache_read_tokens: int = 0
+    cache_creation_tokens: int = 0
+    cost_usd: Optional[float] = None
+    saved_usd: Optional[float] = None
+    affinity_broken: bool = False
+    affinity_honored: bool = False
+    #: Whether the caller marked anything with `cache_control`. None when the
+    #: request never got far enough to be inspected.
+    cache_breakpoint: Optional[bool] = None
+    #: Rough size of the stable head — the part a breakpoint would cover.
+    prefix_tokens: int = 0
+    decision: Optional[Dict[str, Any]] = None
+    error: Optional[str] = None
+
+    @property
+    def cache_hit(self) -> bool:
+        return self.cache_read_tokens > 0
+
+    def to_dict(self) -> Dict[str, Any]:
+        data = asdict(self)
+        data["attempts"] = [a.to_dict() for a in self.attempts]
+        data["cache_hit"] = self.cache_hit
+        return data
+
+
+class EventLog:
+    """A bounded ring buffer plus fan-out to live subscribers (the console's SSE feed)."""
+
+    def __init__(self, capacity: int = 500) -> None:
+        self._events: Deque[RequestEvent] = deque(maxlen=capacity)
+        self._subscribers: Set[asyncio.Queue[Dict[str, Any]]] = set()
+
+    def record(self, event: RequestEvent) -> None:
+        self._events.append(event)
+        payload = event.to_dict()
+        for queue in list(self._subscribers):
+            try:
+                queue.put_nowait(payload)
+            except asyncio.QueueFull:
+                # A slow console must never apply backpressure to a request.
+                pass
+        logger.info("request %s", json.dumps(_log_view(payload), separators=(",", ":")))
+
+    def recent(self, limit: int = 50) -> List[Dict[str, Any]]:
+        events = list(self._events)[-limit:]
+        return [e.to_dict() for e in reversed(events)]
+
+    def recent_for(self, account_id: str, limit: int = 50) -> List[Dict[str, Any]]:
+        """Requests this account took part in, including attempts it failed."""
+        matched = [
+            event
+            for event in self._events
+            if event.account_id == account_id
+            or any(attempt.account_id == account_id for attempt in event.attempts)
+        ]
+        return [e.to_dict() for e in reversed(matched[-limit:])]
+
+    def get(self, request_id: str) -> Optional[Dict[str, Any]]:
+        for event in reversed(self._events):
+            if event.request_id == request_id:
+                return event.to_dict()
+        return None
+
+    def stats(self, window_seconds: float = 3600.0) -> Dict[str, Any]:
+        cutoff = time.time() - window_seconds
+        recent = [e for e in self._events if e.started_at >= cutoff]
+        failovers = sum(1 for e in recent if len(e.attempts) > 1)
+        breaks = sum(1 for e in recent if e.affinity_broken)
+        errors = sum(1 for e in recent if e.status is None or e.status >= 400)
+        billed = sum(
+            e.input_tokens + e.cache_read_tokens + e.cache_creation_tokens for e in recent
+        )
+        cached = sum(e.cache_read_tokens for e in recent)
+        spend = sum(e.cost_usd or 0.0 for e in recent)
+        return {
+            "window_seconds": window_seconds,
+            "requests": len(recent),
+            "failovers": failovers,
+            "cache_breaks": breaks,
+            "errors": errors,
+            "cache_hit_rate": round(cached / billed, 4) if billed else None,
+            "spend_usd": round(spend, 4),
+        }
+
+    def subscribe(self) -> asyncio.Queue[Dict[str, Any]]:
+        queue: asyncio.Queue[Dict[str, Any]] = asyncio.Queue(maxsize=64)
+        self._subscribers.add(queue)
+        return queue
+
+    def unsubscribe(self, queue: asyncio.Queue[Dict[str, Any]]) -> None:
+        self._subscribers.discard(queue)
+
+
+def _log_view(payload: Dict[str, Any]) -> Dict[str, Any]:
+    """Structured log line. Prompts are never in here — only accounting."""
+    return {
+        "request_id": payload["request_id"],
+        "model": payload["model"],
+        "key": payload["key_name"],
+        "account": payload["account_id"],
+        "status": payload["status"],
+        "latency": round(payload["latency"], 3),
+        "attempts": [
+            {"account": a["account_id"], "status": a["status"], "kind": a["kind"]}
+            for a in payload["attempts"]
+        ],
+        "tokens": {
+            "in": payload["input_tokens"],
+            "out": payload["output_tokens"],
+            "cache_read": payload["cache_read_tokens"],
+        },
+        "cache_hit": payload["cache_hit"],
+        "priority": payload["priority"],
+        "via": payload["via"],
+        "queued_for": payload["queued_for"],
+        "affinity_broken": payload["affinity_broken"],
+        "cost_usd": payload["cost_usd"],
+    }

tokenbiryani/observability/usage.py

@@ -0,0 +1,226 @@
+"""Persisted usage history, and the bucketing that turns it into charts.
+
+The event log (`events.py`) is a 500-entry ring in memory: perfect for "what is
+happening right now", useless for "what did last Tuesday cost". Charts need the
+second question answered, and answering it means writing a row per request and
+keeping it.
+
+Two decisions worth stating.
+
+**One row per request, like the spend ledger.** Not a pre-aggregated rollup. A
+rollup has to choose its buckets at write time, and every question you did not
+anticipate — cost by model, cache rate for one account, errors in a five minute
+window — becomes unanswerable. Rows are cheap and the retention window is bounded.
+
+**Aggregation lives here, not in the backends.** Each store only has to append a
+row and hand back the rows in a window; the bucketing, the grouping and every
+derived rate are computed once, in `aggregate`. Three backends implementing the
+same arithmetic three times is three chances to disagree, and a chart that differs
+by backend is worse than no chart.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Iterable, List, Optional
+
+#: What a stored row holds. Deliberately flat and JSON-safe: SQLite gets columns,
+#: Redis and memory get the dict verbatim.
+USAGE_FIELDS = (
+    "at",
+    "account_id",
+    "key_name",
+    "model",
+    "status",
+    "input_tokens",
+    "output_tokens",
+    "cache_read_tokens",
+    "cache_creation_tokens",
+    "cost_usd",
+    "latency",
+    "affinity_broken",
+    "via",
+    "error",
+)
+
+#: How a `group_by` name maps onto a row field.
+GROUP_FIELDS = {
+    "account": "account_id",
+    "model": "model",
+    "key": "key_name",
+}
+
+#: Windows the console offers, in seconds.
+WINDOWS = {"1h": 3600, "24h": 86400, "7d": 604800, "30d": 2592000}
+
+#: A readable bucket for each, chosen so a chart lands between 12 and 60 marks.
+DEFAULT_BUCKETS = {3600: 300, 86400: 3600, 604800: 21600, 2592000: 86400}
+
+
+def sample_from_event(event: Any) -> Dict[str, Any]:
+    """Flatten a `RequestEvent` into a storable row.
+
+    Failed requests are recorded too — a chart that only shows successes hides the
+    outage, which is the thing you opened the chart to see.
+    """
+    return {
+        "at": float(event.started_at),
+        "account_id": event.account_id or "",
+        "key_name": event.key_name or "",
+        "model": event.model or "",
+        "status": int(event.status) if event.status is not None else 0,
+        "input_tokens": int(event.input_tokens or 0),
+        "output_tokens": int(event.output_tokens or 0),
+        "cache_read_tokens": int(event.cache_read_tokens or 0),
+        "cache_creation_tokens": int(event.cache_creation_tokens or 0),
+        "cost_usd": float(event.cost_usd) if event.cost_usd is not None else 0.0,
+        "latency": float(event.latency or 0.0),
+        "affinity_broken": 1 if event.affinity_broken else 0,
+        "via": event.via or "messages",
+        "error": event.error or "",
+    }
+
+
+def resolve_window(raw: Optional[str], fallback: int = 86400) -> int:
+    """Accept either a label the console sends (`24h`) or a raw second count."""
+    if raw is None:
+        return fallback
+    text = str(raw).strip().lower()
+    if text in WINDOWS:
+        return WINDOWS[text]
+    try:
+        seconds = int(float(text))
+    except ValueError:
+        return fallback
+    # A window of a year of per-request rows is a denial of service on yourself.
+    return max(60, min(seconds, WINDOWS["30d"]))
+
+
+def resolve_bucket(window_seconds: int, raw: Optional[str]) -> int:
+    if raw is not None:
+        try:
+            requested = int(float(raw))
+        except ValueError:
+            requested = 0
+        if requested > 0:
+            # Never fewer than 2 or more than 240 marks: past that a line chart is
+            # either a single dot or an unreadable comb.
+            return max(window_seconds // 240, min(requested, window_seconds // 2))
+    for size, bucket in sorted(DEFAULT_BUCKETS.items()):
+        if window_seconds <= size:
+            return bucket
+    return DEFAULT_BUCKETS[2592000]
+
+
+def _blank() -> Dict[str, Any]:
+    return {
+        "requests": 0,
+        "errors": 0,
+        "input_tokens": 0,
+        "output_tokens": 0,
+        "cache_read_tokens": 0,
+        "cache_creation_tokens": 0,
+        "cost_usd": 0.0,
+        "cache_breaks": 0,
+        "_latency_sum": 0.0,
+    }
+
+
+def _add(cell: Dict[str, Any], row: Dict[str, Any]) -> None:
+    cell["requests"] += 1
+    status = int(row.get("status") or 0)
+    if row.get("error") or status == 0 or status >= 400:
+        cell["errors"] += 1
+    cell["input_tokens"] += int(row.get("input_tokens") or 0)
+    cell["output_tokens"] += int(row.get("output_tokens") or 0)
+    cell["cache_read_tokens"] += int(row.get("cache_read_tokens") or 0)
+    cell["cache_creation_tokens"] += int(row.get("cache_creation_tokens") or 0)
+    cell["cost_usd"] += float(row.get("cost_usd") or 0.0)
+    cell["cache_breaks"] += 1 if row.get("affinity_broken") else 0
+    cell["_latency_sum"] += float(row.get("latency") or 0.0)
+
+
+def _finalise(cell: Dict[str, Any]) -> Dict[str, Any]:
+    """Turn running sums into the derived rates a chart actually plots."""
+    billed = (
+        cell["input_tokens"] + cell["cache_read_tokens"] + cell["cache_creation_tokens"]
+    )
+    requests = cell["requests"]
+    out = {k: v for k, v in cell.items() if not k.startswith("_")}
+    out["billed_input_tokens"] = billed
+    out["total_tokens"] = billed + cell["output_tokens"]
+    out["cost_usd"] = round(cell["cost_usd"], 6)
+    # None, not 0.0: "no traffic" and "every request missed the cache" are different
+    # answers, and a chart that draws them the same is lying about the quiet hours.
+    out["cache_hit_rate"] = (
+        round(cell["cache_read_tokens"] / billed, 4) if billed else None
+    )
+    out["latency_avg"] = round(cell["_latency_sum"] / requests, 4) if requests else None
+    return out
+
+
+def aggregate(
+    rows: Iterable[Dict[str, Any]],
+    now: float,
+    window_seconds: int,
+    bucket_seconds: int,
+    group_by: str = "account",
+) -> Dict[str, Any]:
+    """Bucket raw usage rows into a time series plus totals.
+
+    Buckets are emitted for the whole window even when empty, so a gap in traffic
+    draws as a gap rather than silently compressing the time axis.
+    """
+    field = GROUP_FIELDS.get(group_by, "account_id")
+    count = max(1, int(window_seconds // bucket_seconds))
+    # Anchor to the bucket grid so the last bucket is the live, partial one and
+    # every earlier bucket covers a full period.
+    end = (int(now) // bucket_seconds) * bucket_seconds + bucket_seconds
+    start = end - count * bucket_seconds
+
+    buckets: List[Dict[str, Any]] = [
+        {"at": start + index * bucket_seconds, "_all": _blank(), "_by": {}}
+        for index in range(count)
+    ]
+    totals: Dict[str, Dict[str, Any]] = {}
+    overall = _blank()
+
+    for row in rows:
+        at = float(row.get("at") or 0.0)
+        if at < start or at >= end:
+            continue
+        index = int((at - start) // bucket_seconds)
+        if index < 0 or index >= count:
+            continue
+        name = str(row.get(field) or "—")
+        bucket = buckets[index]
+        _add(bucket["_all"], row)
+        if name not in bucket["_by"]:
+            bucket["_by"][name] = _blank()
+        _add(bucket["_by"][name], row)
+        if name not in totals:
+            totals[name] = _blank()
+        _add(totals[name], row)
+        _add(overall, row)
+
+    # Largest first: the legend order and the stacking order should agree, and the
+    # series worth reading should be the one nearest the axis.
+    order = sorted(totals, key=lambda n: -totals[n]["input_tokens"] - totals[n]["output_tokens"])
+
+    return {
+        "window_seconds": window_seconds,
+        "bucket_seconds": bucket_seconds,
+        "group_by": group_by,
+        "start": start,
+        "end": end,
+        "groups": order,
+        "buckets": [
+            {
+                "at": bucket["at"],
+                "all": _finalise(bucket["_all"]),
+                "by": {name: _finalise(cell) for name, cell in bucket["_by"].items()},
+            }
+            for bucket in buckets
+        ],
+        "totals": {name: _finalise(cell) for name, cell in totals.items()},
+        "overall": _finalise(overall),
+    }

tokenbiryani/prices.yaml

@@ -0,0 +1,77 @@
+# Bundled price table — USD per million tokens.
+#
+# This file is DATA, not code, and it is DATED. That distinction is the whole
+# argument for shipping it: the project's standing rule is that the gateway must
+# never bill against a number nobody can attribute, and a dated table that the
+# console displays the date of satisfies that rule. A hard-coded dict in
+# config.py would not.
+#
+# Opt in with `pricing: builtin` in tokenbiryani.yaml. Name a model under
+# `pricing:` to override any line here — your entry always wins.
+#
+# Check the current numbers at https://anthropic.com/pricing and either update
+# this file or override in your own config. The console shows `as_of` beside
+# every cost so a stale table is visible rather than silent.
+#
+# cache_read and cache_write are the standard multiples of the input rate:
+# a cache read bills at ~0.1x input, a 5-minute cache write at ~1.25x input.
+# Claude Fable 5.1 is the exception — its cache reads are priced at a flat
+# $0.25/MTok rather than 0.1x — so it is written out rather than derived.
+
+as_of: 2026-06-24
+source: https://anthropic.com/pricing
+
+models:
+  claude-fable-5-1*:
+    input: 10.0
+    output: 50.0
+    cache_read: 0.25
+    cache_write: 12.5
+
+  claude-fable-5*:
+    input: 10.0
+    output: 50.0
+    cache_read: 1.0
+    cache_write: 12.5
+
+  claude-opus-5*:
+    input: 5.0
+    output: 25.0
+    cache_read: 0.5
+    cache_write: 6.25
+
+  claude-opus-4-8*:
+    input: 5.0
+    output: 25.0
+    cache_read: 0.5
+    cache_write: 6.25
+
+  claude-opus-4-7*:
+    input: 5.0
+    output: 25.0
+    cache_read: 0.5
+    cache_write: 6.25
+
+  claude-opus-4-6*:
+    input: 5.0
+    output: 25.0
+    cache_read: 0.5
+    cache_write: 6.25
+
+  claude-sonnet-5*:
+    input: 2.0
+    output: 10.0
+    cache_read: 0.2
+    cache_write: 2.5
+
+  claude-sonnet-4-6*:
+    input: 3.0
+    output: 15.0
+    cache_read: 0.3
+    cache_write: 3.75
+
+  claude-haiku-4-5*:
+    input: 1.0
+    output: 5.0
+    cache_read: 0.1
+    cache_write: 1.25

tokenbiryani/providers/anthropic_api.py

@@ -0,0 +1,118 @@
+"""Anthropic API key upstream — the supported path."""
+
+from __future__ import annotations
+
+import logging
+from typing import Callable, Dict, List, Optional
+
+from ..config import AccountConfig
+from .base import Upstream
+
+logger = logging.getLogger("tokenbiryani")
+
+_plugins: Dict[str, Callable[[AccountConfig], Upstream]] = {}
+_plugins_loaded = False
+
+
+def register_upstream(name: str, factory: Callable[[AccountConfig], Upstream]) -> None:
+    """Register an account type at runtime. Built-in names cannot be shadowed."""
+    if name in BUILTIN_TYPES:
+        raise ValueError(f"{name!r} is a built-in account type and cannot be replaced")
+    _plugins[name] = factory
+
+
+def plugin_upstreams() -> Dict[str, Callable[[AccountConfig], Upstream]]:
+    global _plugins_loaded
+    if _plugins_loaded:
+        return _plugins
+    _plugins_loaded = True
+    for entry in _entry_points(PLUGIN_GROUP):
+        try:
+            register_upstream(entry.name, entry.load())
+        except Exception as exc:  # noqa: BLE001 - a bad plugin must not be fatal
+            logger.warning("could not load account type %r: %s", entry.name, exc)
+    return _plugins
+
+
+def _entry_points(group: str):
+    """importlib.metadata changed shape in 3.10; support both."""
+    from importlib.metadata import entry_points
+
+    found = entry_points()
+    selector = getattr(found, "select", None)
+    if selector is not None:
+        return list(selector(group=group))
+    return list(found.get(group, []))
+
+
+def available_types() -> List[str]:
+    return sorted(set(BUILTIN_TYPES) | set(plugin_upstreams()))
+
+#: Entry point group third-party packages publish account types under.
+PLUGIN_GROUP = "tokenbiryani.providers"
+
+BUILTIN_TYPES = ("anthropic_api", "bedrock", "vertex", "oauth")
+
+ANTHROPIC_VERSION = "2023-06-01"
+DEFAULT_BASE_URL = "https://api.anthropic.com"
+
+
+class AnthropicUpstream(Upstream):
+    def __init__(self, config: AccountConfig) -> None:
+        super().__init__(config.id)
+        self.config = config
+        self.base_url = (config.base_url or DEFAULT_BASE_URL).rstrip("/")
+
+    def url(self, path: str) -> str:
+        return self.base_url + "/" + path.lstrip("/")
+
+    def auth_headers(self) -> Dict[str, str]:
+        headers = {
+            "x-api-key": self.config.api_key,
+            "content-type": "application/json",
+        }
+        headers.update(self.config.headers)
+        headers.setdefault("anthropic-version", ANTHROPIC_VERSION)
+        return headers
+
+    def request_headers(self, client_headers):
+        headers = super().request_headers(client_headers)
+        # The client's own anthropic-version wins if it sent one; ours is only a default.
+        for name in list(headers):
+            if name.lower() == "anthropic-version" and name != "anthropic-version":
+                headers["anthropic-version"] = headers.pop(name)
+        return headers
+
+
+def build_upstream(
+    config: AccountConfig, token_provider: Optional[Callable[[], str]] = None
+) -> Upstream:
+    """Construct the upstream for an account type, built-in or installed.
+
+    `token_provider` is only meaningful for `oauth` accounts, whose credential is a
+    session the gateway refreshes rather than a value sitting in the config.
+    """
+    if config.type in ("anthropic_api", "anthropic"):
+        return AnthropicUpstream(config)
+    if config.type == "oauth":
+        from .oauth import OAuthUpstream
+
+        return OAuthUpstream(config, token_provider)
+    if config.type == "bedrock":
+        from .bedrock import BedrockUpstream
+
+        return BedrockUpstream(config)
+    if config.type == "vertex":
+        from .vertex import VertexUpstream
+
+        return VertexUpstream(config)
+
+    factory = plugin_upstreams().get(config.type)
+    if factory is not None:
+        return factory(config)
+
+    raise ValueError(
+        f"unsupported account type {config.type!r}; known: "
+        f"{', '.join(available_types())}. Implement providers.base.Upstream and "
+        f"publish it under the {PLUGIN_GROUP!r} entry point to add another."
+    )