dispatch-relay 0.0.1__py3-none-any.whl

dispatch_relay/__init__.py

@@ -0,0 +1,47 @@
+"""dispatch-relay — the swarph's canonical provider-agnostic LLM layer.
+
+Pure core + 3 injected seams. The T1 contract, AI²-converged
+with the peer 2026-06-08 (pending peer co-review).
+
+Exports the three injected-interface seams (each a ``runtime_checkable`` Protocol +
+a dependency-light default impl), the shared value types, the core-owned provider
+facts, and the pure cost model:
+
+  - ConfigSource / DefaultConfigSource       — resolve(key, role, default) → model_id
+  - UsageSink / NoOpUsageSink                — record(...) usage (separate cache fields)
+  - DispatchBackend / DefaultDispatchBackend — supports(...) + dispatch(...) → LLMResponse
+  - LLMResponse / UsageRecord                — shared value types
+  - DEFAULTS / extract_usage / resolve_usage — core-owned provider facts
+  - estimate_cost                            — pure pre-call cost estimator
+"""
+from __future__ import annotations
+
+from .cost import estimate_cost
+from .core import DEFAULTS, extract_usage, resolve_usage
+from .interfaces import (
+    ConfigSource,
+    DefaultConfigSource,
+    UsageSink,
+    NoOpUsageSink,
+    DispatchBackend,
+    DefaultDispatchBackend,
+    LLMResponse,
+    UsageRecord,
+)
+
+__all__ = [
+    "ConfigSource",
+    "DefaultConfigSource",
+    "UsageSink",
+    "NoOpUsageSink",
+    "DispatchBackend",
+    "DefaultDispatchBackend",
+    "LLMResponse",
+    "UsageRecord",
+    "DEFAULTS",
+    "extract_usage",
+    "resolve_usage",
+    "estimate_cost",
+]
+
+__version__ = "0.0.1"

dispatch_relay/analytics.py

@@ -0,0 +1,191 @@
+"""Token-usage analytics — pure aggregation over in-memory usage records.
+
+This is the INVERTED form of the host's old fetch-then-aggregate helper: the
+library owns the AGGREGATION LOGIC and takes the usage records AS INPUT; it does
+NOT fetch. The host keeps a thin wrapper that reads its store (a usage-table
+hypertable, etc.) and delegates the rows here. Inverting the dependency keeps
+this module pure-stdlib + zero-dep so it lives in the dependency-light core
+without re-opening the locked T1 seam contract (no 4th read-seam).
+
+Each record is a mapping with the keys this module reads:
+``provider`` / ``role`` / ``caller`` / ``model`` / ``day`` and the integer token
+columns ``input`` / ``output`` / ``cached`` / ``thought`` plus a float ``cost``.
+
+Schema-asymmetry note (load-bearing for the per-provider arithmetic in
+``_row_total_tokens`` below):
+
+  - Anthropic: ``input`` is the FRESH remainder (``lc_input - cache_read -
+    cache_create``) and ``cached`` is SEPARATE. The two columns are disjoint.
+    True prompt size = ``input + cached``.
+  - Gemini: ``input`` = ``prompt_token_count`` (the FULL prompt) with
+    ``cached`` as a SUBSET. True prompt size = ``input`` (``cached`` is
+    informational, already counted).
+  - OpenAI: ``input`` = ``prompt_tokens`` (FULL) with ``cached`` a subset.
+    Same convention as Gemini.
+  - ``thought`` carries reasoning tokens (Gemini Pro thinking mode, o1-style
+    models) — additive for total prompt cost regardless of provider, currently
+    0 for non-reasoning calls.
+
+Summing ``input + output`` everywhere undercounts Anthropic by the cache_read
+amount; summing ``input + output + cached`` everywhere double-counts
+Gemini/OpenAI by the same. The fix is per-provider arithmetic.
+"""
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Optional
+
+
+# Providers where the ``input`` column ALREADY includes cached tokens
+# (cached is a subset, not a separate bucket). For these, summing
+# ``input + cached`` would double-count.
+_INPUT_INCLUDES_CACHED = {"gemini", "openai"}
+
+
+def _row_total_tokens(provider: str, in_tok: int, out_tok: int,
+                       cached: int, thought: int) -> int:
+    """True total prompt+output+thinking tokens for a row, accounting for the
+    Anthropic-vs-Gemini/OpenAI schema asymmetry documented at module top."""
+    if provider in _INPUT_INCLUDES_CACHED:
+        return in_tok + out_tok + thought
+    # Anthropic (and any provider that stores cached disjoint from input)
+    return in_tok + out_tok + cached + thought
+
+
+def summarize_usage(records, *, days: int = 7, role: Optional[str] = None,
+                    provider: Optional[str] = None,
+                    caller: Optional[str] = None) -> dict:
+    """Aggregate already-fetched usage ``records`` into rollup buckets.
+
+    The ``days`` / ``role`` / ``provider`` / ``caller`` args are echoed into the
+    result as metadata (the host applied them as fetch filters); this function
+    aggregates whatever rows it is given.
+
+    caller bucketing: a ``by_caller`` bucket is always produced so callers can
+    rank workers by spend; NULL caller buckets as ``"<unattributed>"`` so the
+    JSON key is stable for dashboards (no null-handling on the consumer side).
+
+    Returns:
+        {
+          "days": int,
+          "role": str | None,
+          "provider": str | None,
+          "caller": str | None,
+          "total_cost_usd": float,
+          "total_tokens": int,        # provider-aware sum, see module docstring
+          "by_provider": {provider: {input, output, cached, thought, cost, n_rows}},
+          "by_model": [{model, provider, role, caller, ...}, …],
+          "by_role": {role: {cost, tokens}},
+          "by_caller": {caller_or_"<unattributed>": {cost, tokens, n_rows}},
+        }
+
+    All values default to 0 on empty input so dashboards / callers don't need to
+    guard against None.
+    """
+    by_provider: dict[str, dict] = defaultdict(
+        lambda: {"input": 0, "output": 0, "cached": 0, "thought": 0,
+                 "cost": 0.0, "n_rows": 0}
+    )
+    by_role: dict[str, dict] = defaultdict(lambda: {"cost": 0.0, "tokens": 0})
+    # NULL caller → bucket as "<unattributed>" so the JSON key is stable
+    # for dashboards (avoids null-handling on the consumer side).
+    by_caller: dict[str, dict] = defaultdict(
+        lambda: {"cost": 0.0, "tokens": 0, "n_rows": 0}
+    )
+    by_model: list[dict] = []
+    total_cost = 0.0
+    total_tokens = 0
+
+    for r in records:
+        prov = r.get("provider") or "unknown"
+        rrole = r.get("role") or "agents"
+        rcaller = r.get("caller") or "<unattributed>"
+        cost = float(r.get("cost") or 0.0)
+        in_tok = int(r.get("input") or 0)
+        out_tok = int(r.get("output") or 0)
+        cached = int(r.get("cached") or 0)
+        thought = int(r.get("thought") or 0)
+        row_total = _row_total_tokens(prov, in_tok, out_tok, cached, thought)
+
+        by_provider[prov]["input"] += in_tok
+        by_provider[prov]["output"] += out_tok
+        by_provider[prov]["cached"] += cached
+        by_provider[prov]["thought"] += thought
+        by_provider[prov]["cost"] += cost
+        by_provider[prov]["n_rows"] += 1
+
+        by_role[rrole]["cost"] += cost
+        by_role[rrole]["tokens"] += row_total
+
+        by_caller[rcaller]["cost"] += cost
+        by_caller[rcaller]["tokens"] += row_total
+        by_caller[rcaller]["n_rows"] += 1
+
+        by_model.append({
+            "model": r.get("model"),
+            "provider": prov,
+            "role": rrole,
+            "caller": r.get("caller"),  # raw NULL preserved on per-row records
+            "day": r.get("day"),
+            "input": in_tok,
+            "output": out_tok,
+            "cached": cached,
+            "thought": thought,
+            "cost": cost,
+        })
+
+        total_cost += cost
+        total_tokens += row_total
+
+    by_model.sort(key=lambda m: -m["cost"])
+
+    return {
+        "days": days,
+        "role": role,
+        "provider": provider,
+        "caller": caller,
+        "total_cost_usd": round(total_cost, 6),
+        "total_tokens": total_tokens,
+        "by_provider": {k: dict(v) for k, v in by_provider.items()},
+        "by_model": by_model,
+        "by_role": {k: dict(v) for k, v in by_role.items()},
+        "by_caller": {k: dict(v) for k, v in by_caller.items()},
+    }
+
+
+def detect_anomalies(records, *, spike_factor: float = 2.0) -> list[dict]:
+    """Flag (model, day) cells whose cost > spike_factor × baseline avg.
+
+    Baseline is the mean per-day cost for that model over the supplied records
+    EXCLUDING the day being checked. Returns rows in descending cost order.
+    Empty list when ``records`` is empty.
+    """
+    if not records:
+        return []
+
+    # Group by model -> [(day, cost), …]
+    series: dict[str, list[tuple]] = defaultdict(list)
+    for r in records:
+        model = r.get("model") or "unknown"
+        cost = float(r.get("cost") or 0.0)
+        series[model].append((r.get("day"), cost))
+
+    spikes = []
+    for model, points in series.items():
+        if len(points) < 3:
+            continue
+        for day, cost in points:
+            others = [c for d, c in points if d != day]
+            if not others:
+                continue
+            baseline = sum(others) / len(others)
+            if baseline > 0 and cost >= spike_factor * baseline:
+                spikes.append({
+                    "model": model,
+                    "day": day,
+                    "cost": round(cost, 6),
+                    "baseline_avg": round(baseline, 6),
+                    "factor": round(cost / baseline, 2),
+                })
+    spikes.sort(key=lambda s: -s["cost"])
+    return spikes

dispatch_relay/caching.py

@@ -0,0 +1,168 @@
+"""Anthropic prompt-caching helper — wrap any LangChain-compatible LLM.
+
+Wraps any LangChain-compatible LLM (typically a :class:`dispatch_relay.facade._BoundLLM`)
+so every ``.invoke(messages)`` call prepends a SystemMessage carrying
+``cache_control: {"type": "ephemeral", "ttl": ttl}`` in the correct
+list-of-blocks shape.
+
+Why list-of-blocks: ``langchain_anthropic`` SILENTLY DROPS the
+``additional_kwargs={"cache_control": ...}`` shape. The only shape that
+propagates to the wire is::
+
+    SystemMessage(content=[{"type": "text", "text": ...,
+                            "cache_control": {"type": "ephemeral", "ttl": "1h"}}])
+
+For non-Anthropic LLMs the SystemMessage is sent without cache_control (Gemini's
+implicit caching handles long stable prefixes automatically; OpenAI auto-caches
+prompt prefixes ≥1024 tokens at 50% off input price).
+
+This module lives in the ``[facade]`` extra (it needs ``langchain_core``), but the
+langchain import is LAZY (inside :func:`build_cached_system_message`) so importing
+the module is cheap and the zero-dep core stays importable without it.
+"""
+from __future__ import annotations
+
+from typing import Any, Optional
+
+
+def build_cached_system_message(text: str, ttl: str, is_anthropic: bool):
+    """Build a SystemMessage with the right shape for the provider.
+
+    For Anthropic: returns the list-of-blocks shape that langchain_anthropic
+    actually propagates to the wire. For non-Anthropic providers: plain string
+    content (cache_control would be ignored anyway).
+    """
+    from langchain_core.messages import SystemMessage
+    if is_anthropic:
+        return SystemMessage(content=[{
+            "type": "text",
+            "text": text,
+            "cache_control": {"type": "ephemeral", "ttl": ttl},
+        }])
+    return SystemMessage(content=text)
+
+
+# Backward-compat alias for the private name (callers should use the public one).
+_build_cached_system_message = build_cached_system_message
+
+
+def _detect_anthropic(inner: Any) -> bool:
+    """True if the given LLM is an Anthropic model.
+
+    A wrapper exposing ``_provider`` is trusted; raw ``ChatAnthropic`` is detected
+    by class-name sniff. Unknown wrappers default to False — safer to silently
+    disable cache_control than to inject it on a non-Anthropic wire.
+    """
+    provider = getattr(inner, "_provider", None)
+    if provider is None:
+        provider = "anthropic" if "Anthropic" in type(inner).__name__ else ""
+    return provider == "anthropic"
+
+
+class _CacheableLLM:
+    """Proxy over a LangChain LLM that prepends a cached SystemMessage on invoke.
+
+    Delegates ``bind_tools`` / ``with_structured_output`` to the underlying LLM and
+    re-wraps the result so the cached SystemMessage is preserved across chained
+    calls (matches LangChain's chainable contract).
+
+    The provider flag is captured ONCE at the outermost wrap and threaded through
+    every chained re-wrap. Re-sniffing on a chained inner is unsafe — a structured-
+    output proxy may have no ``_provider`` attribute and a class name without
+    "Anthropic", so a re-sniff would silently flip ``_is_anthropic`` to False and
+    drop cache_control from the wire.
+    """
+
+    def __init__(self, inner: Any, cached_text: str, ttl: str = "1h",
+                 *, is_anthropic: Optional[bool] = None):
+        self._inner = inner
+        self._cached_text = cached_text
+        self._ttl = ttl
+        self._is_anthropic = (
+            _detect_anthropic(inner) if is_anthropic is None else is_anthropic
+        )
+
+    def _prepend(self, messages):
+        sysmsg = build_cached_system_message(
+            self._cached_text, self._ttl, self._is_anthropic,
+        )
+        if isinstance(messages, list):
+            return [sysmsg, *messages]
+        return [sysmsg, messages]
+
+    def invoke(self, messages, *args, **kwargs):
+        return self._inner.invoke(self._prepend(messages), *args, **kwargs)
+
+    async def ainvoke(self, messages, *args, **kwargs):
+        return await self._inner.ainvoke(self._prepend(messages), *args, **kwargs)
+
+    def stream(self, messages, *args, **kwargs):
+        return self._inner.stream(self._prepend(messages), *args, **kwargs)
+
+    def bind_tools(self, *args, **kwargs):
+        bound = self._inner.bind_tools(*args, **kwargs)
+        return _CacheableLLM(bound, self._cached_text, self._ttl,
+                             is_anthropic=self._is_anthropic)
+
+    def with_structured_output(self, *args, **kwargs):
+        so = self._inner.with_structured_output(*args, **kwargs)
+        return _CacheableLLM(so, self._cached_text, self._ttl,
+                             is_anthropic=self._is_anthropic)
+
+    def __getattr__(self, name):
+        return getattr(self._inner, name)
+
+
+def with_cache(llm: Any, cached_text: str, ttl: str = "1h") -> _CacheableLLM:
+    """Wrap an LLM so every ``.invoke()`` prepends a cached SystemMessage.
+
+    Args:
+        llm: any LangChain-compatible LLM (typically a ``_BoundLLM``).
+        cached_text: the long prefix to cache (e.g. a playbook / master prompt).
+        ttl: Anthropic ephemeral cache TTL — ``"5m"`` or ``"1h"``. Ignored for
+            non-Anthropic providers (the SystemMessage is still prepended, just
+            without the cache_control marker).
+
+    Returns:
+        A :class:`_CacheableLLM` proxy. Chainable via ``.bind_tools()`` and
+        ``.with_structured_output()``.
+    """
+    return _CacheableLLM(llm, cached_text, ttl=ttl)
+
+
+# Module-level default TTL — settable by the Relay façade so
+# `relay(cache_ttl_default="5m").claude().with_cache(text)` honors the TTL (the
+# attached `.with_cache` method has no reference back to the Relay that made it).
+DEFAULT_CACHE_TTL = "1h"
+
+
+def set_default_cache_ttl(ttl: str) -> None:
+    """Set the module-wide default TTL used by the attached ``.with_cache`` when
+    no explicit ``ttl=`` is passed. Called by ``Relay.__post_init__`` so
+    per-façade defaults stay in sync."""
+    global DEFAULT_CACHE_TTL
+    DEFAULT_CACHE_TTL = ttl
+
+
+def _attach_with_cache_method():
+    """Attach ``with_cache`` as a bound method on ``_BoundLLM``.
+
+    Idempotent — safe to call multiple times. Attached at import so any LLM
+    produced via ``relay(...).gemini()`` etc. exposes ``.with_cache(text)`` as if
+    native, without forcing callers to import :func:`with_cache`.
+    """
+    try:
+        from dispatch_relay.facade import _BoundLLM
+    except ImportError:
+        return
+
+    if "with_cache" in _BoundLLM.__dict__:
+        return
+
+    def _method(self, cached_text: str, ttl: Optional[str] = None):
+        return with_cache(self, cached_text, ttl=ttl or DEFAULT_CACHE_TTL)
+
+    _BoundLLM.with_cache = _method  # type: ignore[attr-defined]
+
+
+_attach_with_cache_method()

dispatch_relay/core.py

@@ -0,0 +1,138 @@
+"""Core-owned provider facts — the DEFAULTS table + usage extraction.
+
+These are provider-facts that must live in exactly ONE place (never duplicated
+per backend or per config source):
+
+- :data:`DEFAULTS` — the 7-key abstract-key → model-id table. The core passes
+  ``default=DEFAULTS[key]`` into :meth:`ConfigSource.resolve`.
+- :func:`extract_usage` — the single place that knows each provider's
+  usage-from-raw shape, including the Anthropic dual-path (the Session-19 surface).
+- :func:`resolve_usage` — the locked reconciliation rule between a backend's
+  optional pre-populated ``LLMResponse.usage`` and core extraction.
+"""
+from __future__ import annotations
+
+import dataclasses
+from typing import Any
+
+from .interfaces import LLMResponse, UsageRecord
+
+# =====================================================================
+# DEFAULTS — abstract model key → concrete model id (the 7-key table)
+# =====================================================================
+# Moved here verbatim from the old DefaultConfigSource.DEFAULTS. The core owns
+# this provider-fact table and passes default=DEFAULTS[key] into resolve().
+DEFAULTS: dict[str, str] = {
+    "gemini_flash": "gemini-2.5-flash",
+    "gemini_flash_lite": "gemini-2.5-flash-lite",
+    "gemini_pro": "gemini-2.5-pro",
+    "gemini_deep_research": "deep-research-pro-preview-12-2025",
+    "claude_sonnet": "claude-sonnet-4-6",
+    "claude_opus": "claude-opus-4-6",
+    "claude_haiku": "claude-haiku-4-5-20251001",
+}
+
+
+def _get(obj: Any, key: str, default: Any = None) -> Any:
+    """Read ``key`` from ``obj`` attribute-style OR dict-style.
+
+    Works on LangChain ``AIMessage``-like objects (attributes) AND plain dicts
+    (canned-dict test fixtures) uniformly.
+    """
+    if obj is None:
+        return default
+    if isinstance(obj, dict):
+        return obj.get(key, default)
+    return getattr(obj, key, default)
+
+
+def extract_usage(provider: str, raw: Any) -> UsageRecord | None:
+    """Extract a :class:`UsageRecord` from a provider's raw response.
+
+    The single place that knows each provider's usage-from-raw shape (provider-fact
+    → core, never duplicated per backend).
+
+    Anthropic DUAL-PATH (the Session-19 surface):
+      * PREFER ``raw.response_metadata["usage"]`` — the UNCACHED remainder, with
+        ``input_tokens`` / ``output_tokens`` / ``cache_read_input_tokens`` /
+        ``cache_creation_input_tokens``.
+      * FALL BACK to ``raw.usage_metadata`` (the LangChain shape:
+        ``input_tokens`` / ``output_tokens`` + ``input_token_details.cache_read`` /
+        ``cache_creation``) only if ``response_metadata.usage`` is absent.
+      Using the wrong one double-counts.
+
+    Non-Anthropic (gemini / openai): read ``raw.usage_metadata``
+    (``input_tokens`` / ``output_tokens``, ``cache_read`` from
+    ``input_token_details`` if present).
+
+    The model name is read from ``raw.response_metadata["model_name"]`` (both
+    Anthropic and Gemini surface it there — a real LangChain ``AIMessage`` has NO
+    top-level ``.model`` attribute), falling back to ``""`` if absent.
+
+    Returns ``None`` if no usage metadata is present (e.g. a subscription raw that
+    is a bare string).
+    """
+    rmd = _get(raw, "response_metadata")
+    model = (_get(rmd, "model_name", "") if rmd is not None else "") or ""
+
+    if provider == "anthropic":
+        rmd_usage = _get(rmd, "usage") if rmd is not None else None
+        if rmd_usage is not None:
+            return UsageRecord(
+                input_tokens=int(_get(rmd_usage, "input_tokens", 0) or 0),
+                output_tokens=int(_get(rmd_usage, "output_tokens", 0) or 0),
+                cache_read=int(_get(rmd_usage, "cache_read_input_tokens", 0) or 0),
+                cache_creation=int(
+                    _get(rmd_usage, "cache_creation_input_tokens", 0) or 0
+                ),
+                model=model,
+            )
+        # Fall back to the LangChain usage_metadata shape.
+        um = _get(raw, "usage_metadata")
+        if um is not None:
+            details = _get(um, "input_token_details") or {}
+            return UsageRecord(
+                input_tokens=int(_get(um, "input_tokens", 0) or 0),
+                output_tokens=int(_get(um, "output_tokens", 0) or 0),
+                cache_read=int(_get(details, "cache_read", 0) or 0),
+                cache_creation=int(_get(details, "cache_creation", 0) or 0),
+                model=model,
+            )
+        return None
+
+    # gemini / openai (and any other non-anthropic provider)
+    um = _get(raw, "usage_metadata")
+    if um is not None:
+        details = _get(um, "input_token_details") or {}
+        return UsageRecord(
+            input_tokens=int(_get(um, "input_tokens", 0) or 0),
+            output_tokens=int(_get(um, "output_tokens", 0) or 0),
+            cache_read=int(_get(details, "cache_read", 0) or 0),
+            cache_creation=int(_get(details, "cache_creation", 0) or 0),
+            model=model,
+        )
+    return None
+
+
+def resolve_usage(
+    response: LLMResponse, provider: str, model: str
+) -> UsageRecord | None:
+    """The LOCKED reconciliation rule.
+
+    A backend MAY pre-populate ``response.usage`` (a real escape hatch); otherwise
+    the core extracts it from ``response.raw``.
+
+    The dispatch call KNOWS the configured model (it's the ``model`` argument), so
+    the dispatch-arg ``model`` is authoritative: once a record is resolved, its
+    ``model`` field is stamped with this argument — the configured model always
+    wins over whatever the raw echoed (or didn't). Returns ``None`` unchanged when
+    there is no usage (the subscription lane).
+    """
+    record = (
+        response.usage
+        if response.usage is not None
+        else extract_usage(provider, response.raw)
+    )
+    if record is None:
+        return None
+    return dataclasses.replace(record, model=model)