hypermind 0.11.0__py3-none-any.whl

hypermind/observability/cost.py

@@ -0,0 +1,277 @@
+"""LLM cost tracking — token accounting + USD estimation.
+
+Every OpenAI/Anthropic/OpenRouter call carries a ``usage`` payload with
+``prompt_tokens`` and ``completion_tokens``. The :class:`TokenCostTracker`
+captures these, converts to USD via :data:`PRICE_TABLE`, and emits
+Prometheus counters and recorder events so dashboards can watch spend in
+real time.
+
+Patterns:
+  * One :class:`TokenCostTracker` per :class:`_NamespaceWorld`.
+  * Each LLM responder (OpenAIResponder, AnthropicResponder,
+    ToolCallingResponder) accepts an optional ``cost_tracker``.
+  * The tracker emits ``llm_token_cost`` events with full per-call
+    detail (model, agent_kid, tokens, cost) and increments
+    ``hypermind_llm_tokens_total{model,direction}`` counters.
+
+Pricing notes:
+  * Prices are USD per 1 million tokens, sourced from public model pages
+    on 2026-05-01. Prices update regularly — operators can override via
+    ``TokenCostTracker(price_table_override={...})``.
+  * Prices for OpenRouter pass-through models match the underlying
+    provider's quoted price. OpenRouter's small markup (~5%) is not
+    accounted for; use this as an estimate, not as billing input.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from typing import Any
+
+from hypermind.responders.structured import TokenUsage
+
+# ---------------------------------------------------------------------------
+# Price table — USD per 1M tokens (input, output)
+# ---------------------------------------------------------------------------
+
+#: Update version stamp — bump when prices change so deployments can
+#: detect stale tables.
+PRICE_TABLE_VERSION = "2026-05-01"
+
+#: ``model_id -> (input_per_1m_usd, output_per_1m_usd)``.
+#:
+#: The dict is keyed by *canonical* model ID. OpenRouter prefixes
+#: (``openai/``, ``anthropic/``, ``meta-llama/``, …) are accepted in
+#: addition to bare names so callers don't have to normalise.
+PRICE_TABLE: dict[str, tuple[float, float]] = {
+    # OpenAI
+    "gpt-4o": (2.50, 10.00),
+    "gpt-4o-mini": (0.15, 0.60),
+    "gpt-4-turbo": (10.00, 30.00),
+    "openai/gpt-4o": (2.50, 10.00),
+    "openai/gpt-4o-mini": (0.15, 0.60),
+    "openai/gpt-4-turbo": (10.00, 30.00),
+    # Anthropic
+    "claude-opus-4-7": (15.00, 75.00),
+    "claude-sonnet-4-6": (3.00, 15.00),
+    "claude-sonnet-4": (3.00, 15.00),
+    "claude-haiku-4-5": (0.80, 4.00),
+    "claude-haiku-3-5": (0.80, 4.00),
+    "anthropic/claude-opus-4-7": (15.00, 75.00),
+    "anthropic/claude-sonnet-4-6": (3.00, 15.00),
+    "anthropic/claude-sonnet-4": (3.00, 15.00),
+    "anthropic/claude-haiku-4-5": (0.80, 4.00),
+    # Google
+    "google/gemini-2.0-flash": (0.075, 0.30),
+    "google/gemini-1.5-pro": (1.25, 5.00),
+    # Open-weights via OpenRouter
+    "meta-llama/llama-3.3-70b-instruct": (0.18, 0.18),
+    "meta-llama/llama-3.1-70b-instruct": (0.18, 0.18),
+    "deepseek/deepseek-chat": (0.27, 1.10),
+    "qwen/qwen-2.5-72b-instruct": (0.20, 0.20),
+}
+
+
+def cost_usd(
+    model: str,
+    prompt_tokens: int,
+    completion_tokens: int,
+    *,
+    price_table: dict[str, tuple[float, float]] | None = None,
+) -> float | None:
+    """Compute USD cost for one LLM call.
+
+    Returns None if the model isn't in the price table — operators see
+    the warning in the recorder rather than a wrong cost figure.
+    """
+    table = price_table if price_table is not None else PRICE_TABLE
+    prices = table.get(model)
+    if prices is None:
+        return None
+    input_per_1m, output_per_1m = prices
+    return (prompt_tokens / 1_000_000) * input_per_1m + (
+        completion_tokens / 1_000_000
+    ) * output_per_1m
+
+
+# ---------------------------------------------------------------------------
+# TokenCostTracker
+# ---------------------------------------------------------------------------
+
+
+class TokenCostTracker:
+    """Aggregates LLM cost events across all responders attached to a world.
+
+    Thread-safe (used from coroutines under a single event loop, but
+    OpenAI / Anthropic SDKs may schedule retries on threads).
+    """
+
+    def __init__(
+        self,
+        *,
+        recorder: Any = None,
+        price_table_override: dict[str, tuple[float, float]] | None = None,
+    ) -> None:
+        self._records: list[dict[str, Any]] = []
+        self._lock = threading.Lock()
+        self.recorder = recorder
+        self.price_table = (
+            dict(price_table_override) if price_table_override is not None else PRICE_TABLE
+        )
+
+    # ------------------------------------------------------------------
+    # Write path
+    # ------------------------------------------------------------------
+
+    def record(
+        self,
+        *,
+        model: str,
+        prompt_tokens: int,
+        completion_tokens: int,
+        agent_kid: bytes | None = None,
+        topic: str | None = None,
+        ts_wall_ms: int | None = None,
+    ) -> TokenUsage:
+        """Record one LLM call and return its :class:`TokenUsage`."""
+        ts = int(time.time() * 1000) if ts_wall_ms is None else int(ts_wall_ms)
+        c = cost_usd(model, prompt_tokens, completion_tokens, price_table=self.price_table)
+        usage = TokenUsage(
+            model=model,
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            cost_usd=c,
+        )
+        rec = {
+            "ts_wall_ms": ts,
+            "model": model,
+            "prompt_tokens": prompt_tokens,
+            "completion_tokens": completion_tokens,
+            "total_tokens": prompt_tokens + completion_tokens,
+            "cost_usd": c,
+            "agent_kid_hex": agent_kid.hex() if agent_kid else None,
+            "topic": topic,
+        }
+        with self._lock:
+            self._records.append(rec)
+
+        # Fan-out to recorder (best-effort)
+        if self.recorder is not None:
+            try:
+                self.recorder.emit(
+                    event_type="llm_token_cost",
+                    severity="info",
+                    fields=rec,
+                )
+                self.recorder.incr(
+                    "hypermind_llm_tokens_total",
+                    by=prompt_tokens + completion_tokens,
+                    model=model,
+                )
+                if c is not None:
+                    self.recorder.incr(
+                        "hypermind_llm_cost_usd_total",
+                        by=int(c * 100_000),  # store milli-cents to keep integer counter
+                        model=model,
+                    )
+                else:
+                    self.recorder.emit(
+                        event_type="llm_unknown_model_pricing",
+                        severity="warn",
+                        fields={"model": model, "tokens": prompt_tokens + completion_tokens},
+                    )
+            except Exception:
+                pass
+
+        return usage
+
+    # ------------------------------------------------------------------
+    # Read path
+    # ------------------------------------------------------------------
+
+    def total_usd(
+        self,
+        *,
+        agent_kid: bytes | None = None,
+        model: str | None = None,
+    ) -> float:
+        """Sum of cost_usd across matching records (None costs ignored)."""
+        with self._lock:
+            rows = list(self._records)
+        out = 0.0
+        for r in rows:
+            if r["cost_usd"] is None:
+                continue
+            if agent_kid is not None and r.get("agent_kid_hex") != agent_kid.hex():
+                continue
+            if model is not None and r.get("model") != model:
+                continue
+            out += r["cost_usd"]
+        return out
+
+    def total_tokens(
+        self,
+        *,
+        agent_kid: bytes | None = None,
+        model: str | None = None,
+    ) -> int:
+        with self._lock:
+            rows = list(self._records)
+        out = 0
+        for r in rows:
+            if agent_kid is not None and r.get("agent_kid_hex") != agent_kid.hex():
+                continue
+            if model is not None and r.get("model") != model:
+                continue
+            out += r["total_tokens"]
+        return out
+
+    def snapshot(self) -> list[dict[str, Any]]:
+        """Return a defensive copy of every recorded call."""
+        with self._lock:
+            return [dict(r) for r in self._records]
+
+    def to_dict(self) -> dict[str, Any]:
+        """Aggregate summary suitable for the /v1/cost endpoint."""
+        snap = self.snapshot()
+        # Per-model breakdown
+        by_model: dict[str, dict[str, Any]] = {}
+        total_usd = 0.0
+        total_tok = 0
+        for r in snap:
+            m = r["model"]
+            slot = by_model.setdefault(
+                m,
+                {
+                    "calls": 0,
+                    "total_tokens": 0,
+                    "cost_usd": 0.0,
+                },
+            )
+            slot["calls"] += 1
+            slot["total_tokens"] += r["total_tokens"]
+            if r["cost_usd"] is not None:
+                slot["cost_usd"] += r["cost_usd"]
+                total_usd += r["cost_usd"]
+            total_tok += r["total_tokens"]
+        return {
+            "n_calls": len(snap),
+            "total_tokens": total_tok,
+            "total_usd": total_usd,
+            "by_model": by_model,
+            "price_table_version": PRICE_TABLE_VERSION,
+        }
+
+    def reset(self) -> None:
+        """Clear all records — for tests or per-session resets."""
+        with self._lock:
+            self._records.clear()
+
+
+__all__ = [
+    "PRICE_TABLE",
+    "PRICE_TABLE_VERSION",
+    "TokenCostTracker",
+    "cost_usd",
+]

hypermind/observability/otel_export.py

@@ -0,0 +1,200 @@
+"""OpenTelemetry exporter — real-time OTLP push of consult/dispute spans.
+
+The :class:`OtelExporter` listens for recorder events and emits OTel
+spans for the verbs that matter to operators:
+
+  * ``publish_claim``        → span "hypermind.publish"
+  * ``dispute``              → span "hypermind.dispute"
+  * ``consult_panel``        → parent span "hypermind.consult"
+  * ``llm_token_cost``       → child span "hypermind.llm_call"
+  * ``oracle_resolve``       → span "hypermind.oracle"
+  * ``tool_call``            → child span "hypermind.tool_call"
+
+Soft-depends on ``opentelemetry-api`` + ``opentelemetry-sdk``. When
+those packages aren't installed the exporter degrades gracefully — it
+records nothing but never raises.
+
+The exporter is *attached*, not constructed inline: callers build an
+OTel ``TracerProvider`` (with whatever exporter they prefer — OTLP
+gRPC, Jaeger, etc.) and pass it. We don't pin a specific transport
+because operators have wildly different deployment shapes.
+"""
+
+from __future__ import annotations
+
+from typing import Any, ClassVar
+
+
+class OtelExporter:
+    """Bridges :class:`Recorder` events to OpenTelemetry spans.
+
+    Usage::
+
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import (
+            BatchSpanProcessor,
+            ConsoleSpanExporter,
+        )
+        from hypermind.observability.otel_export import OtelExporter
+
+        provider = TracerProvider()
+        provider.add_span_processor(BatchSpanProcessor(ConsoleSpanExporter()))
+
+        exporter = OtelExporter(tracer_provider=provider, service_name="hypermind")
+        exporter.attach(agent._world.recorder)
+
+    For an OTLP push (Jaeger/Tempo), substitute the OTLPSpanExporter::
+
+        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+        provider.add_span_processor(BatchSpanProcessor(OTLPSpanExporter(endpoint=...)))
+
+    The exporter never owns the TracerProvider — operators control its
+    lifecycle, including shutdown.
+    """
+
+    def __init__(
+        self,
+        *,
+        tracer_provider: Any = None,
+        service_name: str = "hypermind",
+    ) -> None:
+        self.service_name = service_name
+        self._tracer = None
+        self._available = False
+
+        if tracer_provider is None:
+            return
+
+        try:
+            self._tracer = tracer_provider.get_tracer(service_name)
+            self._available = True
+        except Exception:
+            self._tracer = None
+            self._available = False
+
+    # ------------------------------------------------------------------
+    # Available — operators check this to know whether OTel is hot
+    # ------------------------------------------------------------------
+
+    @property
+    def available(self) -> bool:
+        return self._available
+
+    # ------------------------------------------------------------------
+    # Attach to a recorder — listen for events and emit spans
+    # ------------------------------------------------------------------
+
+    def attach(self, recorder: Any) -> None:
+        """Subscribe to a recorder so subsequent events emit OTel spans.
+
+        We patch the recorder's ``emit()`` method to also call our
+        :meth:`_handle_event`. This keeps the exporter optional — if
+        nobody attaches one, recorder behaviour is unchanged.
+        """
+        if not self._available or recorder is None:
+            return
+
+        original_emit = recorder.emit
+
+        def emit_with_otel(*args: Any, **kwargs: Any) -> None:
+            try:
+                original_emit(*args, **kwargs)
+            finally:
+                try:
+                    self._handle_event(kwargs)
+                except Exception:
+                    pass  # never break the recorder
+
+        recorder.emit = emit_with_otel  # type: ignore[method-assign]
+
+    # ------------------------------------------------------------------
+    # Event → span mapping
+    # ------------------------------------------------------------------
+
+    _EVENT_TO_SPAN: ClassVar[dict[str, str]] = {
+        "seal": "hypermind.publish",
+        "dispute_open": "hypermind.dispute",
+        "consult_panel": "hypermind.consult",
+        "consult_deliberation": "hypermind.deliberation",
+        "consult_responder_error": "hypermind.consult.responder_error",
+        "llm_token_cost": "hypermind.llm_call",
+        "tool_call": "hypermind.tool_call",
+        "router_fallback": "hypermind.router.fallback",
+        "oracle_resolve": "hypermind.oracle",
+    }
+
+    def _handle_event(self, emit_kwargs: dict[str, Any]) -> None:
+        if self._tracer is None:
+            return
+        event_type = emit_kwargs.get("event_type", "")
+        span_name = self._EVENT_TO_SPAN.get(event_type)
+        if span_name is None:
+            return  # not an event we surface to OTel
+
+        # Span attributes — flatten the fields dict into "field.<k>=<v>"
+        # attribute keys so OTel UIs can filter on them.
+        attrs: dict[str, Any] = {
+            "service.name": self.service_name,
+            "hypermind.event_type": event_type,
+        }
+        ns = emit_kwargs.get("namespace")
+        if ns:
+            attrs["hypermind.namespace"] = ns
+        agent_id = emit_kwargs.get("agent_id")
+        if agent_id:
+            attrs["hypermind.agent_id"] = agent_id
+        fields = emit_kwargs.get("fields", {}) or {}
+        for k, v in fields.items():
+            if isinstance(v, (str, bool, int, float)):
+                attrs[f"hypermind.field.{k}"] = v
+            else:
+                attrs[f"hypermind.field.{k}"] = repr(v)[:240]
+
+        # We don't know the actual operation duration here, so we emit
+        # a zero-duration "instant" span. Production callers that want
+        # parent/child relationships should use the explicit
+        # span_consult() context manager instead.
+        with self._tracer.start_as_current_span(
+            name=span_name,
+            attributes=attrs,
+        ) as _:
+            # No-op — we just want the span recorded.
+            pass
+
+    # ------------------------------------------------------------------
+    # Explicit span context managers — for callers that want correct durations
+    # ------------------------------------------------------------------
+
+    def span_consult(self, query: str, *, panel_size: int) -> Any:
+        """Start a parent span for one consult call.
+
+        Usage::
+
+            with exporter.span_consult(query="...", panel_size=8):
+                result = await agent.consult(...)
+        """
+        if self._tracer is None:
+            return _NoopSpan()
+        return self._tracer.start_as_current_span(
+            name="hypermind.consult",
+            attributes={
+                "hypermind.consult.panel_size": int(panel_size),
+                "hypermind.consult.query_len": len(query),
+            },
+        )
+
+
+class _NoopSpan:
+    """Stand-in for an OTel span when the exporter isn't available."""
+
+    def __enter__(self) -> _NoopSpan:
+        return self
+
+    def __exit__(self, *exc: Any) -> None:
+        return None
+
+    def set_attribute(self, *args: Any, **kwargs: Any) -> None:
+        return None
+
+
+__all__ = ["OtelExporter"]