unique-toolkit 2026.32.0.dev4__tar.gz → 2026.32.0.dev6__tar.gz

{unique_toolkit-2026.32.0.dev4 → unique_toolkit-2026.32.0.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: unique-toolkit
-Version: 2026.32.0.dev4
+Version: 2026.32.0.dev6
 Summary: 
 Author: Cedric Klinkert, Martin Fadler, Sadique Sheik, Fabian Schläpfer, Pascal Hauri, Ejiroghene Oki-Peter, Rami Azouz, Ricardo Visini, Sanzio Monti
 Author-email: Cedric Klinkert <cedric.klinkert@unique.ch>, Martin Fadler <martin.fadler@unique.ch>, Sadique Sheik <sadique@unique.ch>, Fabian Schläpfer <fabian@unique.ch>, Pascal Hauri <pascal@unique.ch>, Ejiroghene Oki-Peter <oki-peter.ejiroghene@unique.ai>, Rami Azouz <rami.ext@unique.ch>, Ricardo Visini <ricardo.visini@unique.ch>, Sanzio Monti <sanzio.monti@unique.ai>
@@ -16,7 +16,7 @@ Requires-Dist: sseclient>=0.0.27,<0.0.28
 Requires-Dist: openai>=1.105.0,<3
 Requires-Dist: platformdirs>=4.0.0,<5
 Requires-Dist: pillow>=12.1.1
-Requires-Dist: unique-sdk>=2026.32.0.dev5,<2026.32.0rc0
+Requires-Dist: unique-sdk>=2026.32.0.dev8,<2026.32.0rc0
 Requires-Dist: jambo>=0.1.2,<0.2
 Requires-Dist: jinja2>=3.1.6,<4
 Requires-Dist: docxtpl>=0.20.1,<0.21
@@ -33,6 +33,7 @@ Requires-Dist: tenacity>=9.1.4
 Requires-Dist: pydantic-core>=2.41.5
 Requires-Dist: cachetools>=5.0,<6
 Requires-Dist: rapidfuzz>=3.0,<4
+Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: fastapi>=0.133.0,<1.0.0 ; extra == 'fastapi'
 Requires-Dist: starlette>=1.3.1,<2.0.0 ; extra == 'fastapi'
 Requires-Dist: uvicorn>=0.37.0,<0.38 ; extra == 'fastapi'

{unique_toolkit-2026.32.0.dev4 → unique_toolkit-2026.32.0.dev6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "unique_toolkit"
-version = "2026.32.0.dev4"
+version = "2026.32.0.dev6"
 description = ""
 readme = "README.md"
 requires-python = ">=3.12"
@@ -28,7 +28,7 @@ dependencies = [
     "openai>=1.105.0,<3",
     "platformdirs>=4.0.0,<5",
     "pillow>=12.1.1",
-    "unique-sdk>=2026.32.0.dev5,<2026.32.0rc0",
+    "unique-sdk>=2026.32.0.dev8,<2026.32.0rc0",
     "jambo>=0.1.2,<0.2",
     "jinja2>=3.1.6,<4",
     "docxtpl>=0.20.1,<0.21",
@@ -45,6 +45,7 @@ dependencies = [
     "pydantic-core>=2.41.5",
     "cachetools>=5.0,<6",
     "rapidfuzz>=3.0,<4",
+    "pyyaml>=6.0.3",
 ]
 
 [project.optional-dependencies]

{unique_toolkit-2026.32.0.dev4 → unique_toolkit-2026.32.0.dev6}/unique_toolkit/agentic/debug_info_manager/debug_info_manager.py

@@ -43,14 +43,18 @@ class AnalyticsToolName(TypedDict):
     display_name: str
 
 
-class AnalyticsTokenUsage(TypedDict):
-    model_name: str
+class AnalyticsConsumption(TypedDict):
     completion_tokens: int | None
     prompt_tokens: int | None
     total_tokens: int | None
     reasoning_tokens: int | None
     cached_tokens: int | None
     cache_write_tokens: int | None
+    cost_usd: float | None
+
+
+class AnalyticsConsumptionByLlm(AnalyticsConsumption):
+    model_name: str
 
 
 class Analytics(TypedDict):
@@ -58,6 +62,10 @@ class Analytics(TypedDict):
     artifacts_created_count: int | None
     artifacts_created_filetype: list[str] | None
     context_memory_updated: bool | None
+    # Sum of all model usages this turn. None when no invocations reported usage.
+    consumption: AnalyticsConsumption | None
+    # Per-model usage totals, sorted by model_name.
+    consumption_by_llm: list[AnalyticsConsumptionByLlm]
     language_model: AnalyticsLanguageModel
     loop_iteration_count: int
     mcp_tool_names_used: list[AnalyticsToolName]
@@ -67,7 +75,6 @@ class Analytics(TypedDict):
     references_count: int
     skills_used: list[AnalyticsSkill]
     subagent_names_used: list[AnalyticsToolName]
-    tokens: list[AnalyticsTokenUsage]
     tool_call_count: int
     tools_used: list[AnalyticsTool]
     total_time_to_answer_ms: int | None
@@ -149,6 +156,7 @@ class DebugInfoManager:
         # DisplayCodeInterpreterFilesPostProcessor) into debug_info["artifacts"].
         # Absent when no Code Interpreter ran this turn → artifact fields stay
         # None, preserving the always-present, null-when-unknown contract.
+        consumption_by_llm = _aggregate_consumption_by_llm(invocations or [])
         analytics = Analytics(
             tools_used=analytics_tools,
             tool_call_count=len(analytics_tools),
@@ -167,7 +175,8 @@ class DebugInfoManager:
             loop_iteration_count=loop_iteration_count,
             subagent_names_used=_unique_tool_names(analytics_tools, "is_sub_agent"),
             mcp_tool_names_used=_unique_tool_names(analytics_tools, "is_mcp"),
-            tokens=_aggregate_tokens_by_model(invocations or []),
+            consumption_by_llm=consumption_by_llm,
+            consumption=_aggregate_consumption(invocations or []),
             total_time_to_answer_ms=total_time_to_answer_ms,
             artifacts_created_count=artifacts["count"] if artifacts else None,
             artifacts_created_filetype=artifacts["filetypes"] if artifacts else None,
@@ -179,27 +188,66 @@ class DebugInfoManager:
         self.add("analytics", analytics)
 
 
-def _aggregate_tokens_by_model(
+def _sum_invocation_costs(
+    invocations: list[LanguageModelInvocationStats],
+) -> float | None:
+    """Sum costs only when every invocation has a known cost; else None."""
+    costs = [invocation.cost_usd for invocation in invocations]
+    if not costs or any(cost is None for cost in costs):
+        return None
+    return sum(cost for cost in costs if cost is not None)
+
+
+def _to_analytics_consumption(
+    usage: LanguageModelTokenUsage,
+    cost_usd: float | None,
+) -> AnalyticsConsumption:
+    return AnalyticsConsumption(
+        completion_tokens=usage.completion_tokens,
+        prompt_tokens=usage.prompt_tokens,
+        total_tokens=usage.total_tokens,
+        reasoning_tokens=usage.reasoning_tokens,
+        cached_tokens=usage.cached_tokens,
+        cache_write_tokens=usage.cache_write_tokens,
+        cost_usd=cost_usd,
+    )
+
+
+def _aggregate_consumption(
     invocations: list[LanguageModelInvocationStats],
-) -> list[AnalyticsTokenUsage]:
-    usages_by_model: dict[str, list[LanguageModelTokenUsage]] = defaultdict(list)
+) -> AnalyticsConsumption | None:
+    """Aggregate token usage and cost across all models for this turn."""
+    usage = LanguageModelTokenUsage.sum_usages(
+        invocation.token_usage for invocation in invocations
+    )
+    if usage is None:
+        return None
+    return _to_analytics_consumption(usage, _sum_invocation_costs(invocations))
+
+
+def _aggregate_consumption_by_llm(
+    invocations: list[LanguageModelInvocationStats],
+) -> list[AnalyticsConsumptionByLlm]:
+    invocations_by_model: dict[str, list[LanguageModelInvocationStats]] = defaultdict(
+        list
+    )
     for invocation in invocations:
-        usages_by_model[str(invocation.model_name)].append(invocation.token_usage)
+        invocations_by_model[str(invocation.model_name)].append(invocation)
 
-    totals: list[AnalyticsTokenUsage] = []
-    for model_name in sorted(usages_by_model):
-        usage = LanguageModelTokenUsage.sum_usages(usages_by_model[model_name])
+    totals: list[AnalyticsConsumptionByLlm] = []
+    for model_name in sorted(invocations_by_model):
+        model_invocations = invocations_by_model[model_name]
+        usage = LanguageModelTokenUsage.sum_usages(
+            invocation.token_usage for invocation in model_invocations
+        )
         if usage is None:
             continue
         totals.append(
-            AnalyticsTokenUsage(
+            AnalyticsConsumptionByLlm(
                 model_name=model_name,
-                completion_tokens=usage.completion_tokens,
-                prompt_tokens=usage.prompt_tokens,
-                total_tokens=usage.total_tokens,
-                reasoning_tokens=usage.reasoning_tokens,
-                cached_tokens=usage.cached_tokens,
-                cache_write_tokens=usage.cache_write_tokens,
+                **_to_analytics_consumption(
+                    usage, _sum_invocation_costs(model_invocations)
+                ),
             )
         )
     return totals

{unique_toolkit-2026.32.0.dev4 → unique_toolkit-2026.32.0.dev6}/unique_toolkit/language_model/invocation_stats.py

@@ -11,6 +11,7 @@ from humps import camelize
 from pydantic import BaseModel, BeforeValidator, ConfigDict
 
 from unique_toolkit.language_model.infos import LanguageModelName
+from unique_toolkit.language_model.model_costs import calculate_invocation_cost_usd
 from unique_toolkit.language_model.schemas import LanguageModelTokenUsage
 
 # `protected_namespaces=()` allows the `model_name` field name.
@@ -59,6 +60,7 @@ class LanguageModelInvocationStats(BaseModel):
     model_name: ModelName
     token_usage: LanguageModelTokenUsage
     source: Source  # e.g. "main_loop", tool/evaluation/postprocessor name
+    cost_usd: float | None = None
 
     @classmethod
     def from_usage(
@@ -67,4 +69,9 @@ class LanguageModelInvocationStats(BaseModel):
         token_usage: LanguageModelTokenUsage,
         source: str,
     ) -> Self:
-        return cls(model_name=model_name, token_usage=token_usage, source=source)
+        return cls(
+            model_name=model_name,
+            token_usage=token_usage,
+            source=source,
+            cost_usd=calculate_invocation_cost_usd(str(model_name), token_usage),
+        )

unique_toolkit-2026.32.0.dev6/unique_toolkit/language_model/model_costs.py

@@ -0,0 +1,118 @@
+"""Load model prices and calculate the USD cost of LLM invocations."""
+
+import logging
+import os
+import time
+from pathlib import Path
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field
+
+from unique_toolkit.language_model.schemas import LanguageModelTokenUsage
+
+logger = logging.getLogger(f"toolkit.language_model.{__name__}")
+
+MODEL_COSTS_FILE_ENV = "MODEL_COSTS_FILE"
+_LITELLM_PREFIX = "litellm:"
+_TOKENS_PER_MILLION = 1_000_000
+
+# Safety-only bound on how long a process may keep a loaded catalog. Not the
+# primary refresh mechanism — just ensures a long-lived worker eventually
+# re-reads a remounted price sheet.
+_CACHE_MAX_AGE_SECONDS = 5 * 60
+
+
+class ModelCost(BaseModel):
+    """Per-million-token prices for one language model."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    input: float
+    completion: float
+    currency: str = Field(default="USD", min_length=1)
+
+
+class ModelCostCatalog(BaseModel):
+    """Versioned model-price catalog rendered by the platform Helm charts."""
+
+    model_config = ConfigDict(extra="forbid", populate_by_name=True)
+
+    cost_schema_version: int = Field(alias="costSchemaVersion")
+    models: dict[str, ModelCost] = Field(min_length=1)
+
+
+_catalog_cache: dict[Path, tuple[ModelCostCatalog, float]] = {}
+
+
+def _parse_model_cost_catalog(path: Path) -> ModelCostCatalog:
+    payload = yaml.safe_load(path.read_text(encoding="utf-8"))
+    catalog = ModelCostCatalog.model_validate(payload)
+    if catalog.cost_schema_version != 1:
+        raise ValueError(
+            f"Unsupported model cost schema version: {catalog.cost_schema_version}"
+        )
+    return catalog
+
+
+def _load_model_cost_catalog(path: Path) -> ModelCostCatalog:
+    now = time.monotonic()
+    cached = _catalog_cache.get(path)
+    if cached is not None:
+        catalog, loaded_at = cached
+        if now - loaded_at < _CACHE_MAX_AGE_SECONDS:
+            return catalog
+
+    catalog = _parse_model_cost_catalog(path)
+    _catalog_cache[path] = (catalog, now)
+    return catalog
+
+
+def load_model_cost_catalog(path: str | Path | None = None) -> ModelCostCatalog | None:
+    """Load and cache the configured catalog, or return None when unavailable.
+
+    Returns None when the catalog path is unset, or when the file cannot be
+    read/parsed. Cost capture must never abort a successful LLM call over a
+    missing or invalid price sheet.
+    """
+
+    configured_path = path or os.getenv(MODEL_COSTS_FILE_ENV)
+    if configured_path is None or not str(configured_path).strip():
+        return None
+    try:
+        return _load_model_cost_catalog(Path(configured_path))
+    except Exception as exc:
+        logger.warning(
+            "Failed to load model cost catalog from %s: %s",
+            configured_path,
+            exc,
+        )
+        return None
+
+
+def calculate_invocation_cost_usd(
+    model_name: str,
+    token_usage: LanguageModelTokenUsage,
+    catalog: ModelCostCatalog | None = None,
+) -> float | None:
+    """Calculate invocation cost using prompt and completion token totals."""
+
+    if token_usage.prompt_tokens is None or token_usage.completion_tokens is None:
+        return None
+
+    active_catalog = catalog if catalog is not None else load_model_cost_catalog()
+    if active_catalog is None:
+        return None
+
+    normalized_model_name = str(model_name)
+    model_cost = active_catalog.models.get(normalized_model_name)
+    if model_cost is None and normalized_model_name.startswith(_LITELLM_PREFIX):
+        model_cost = active_catalog.models.get(
+            normalized_model_name.removeprefix(_LITELLM_PREFIX)
+        )
+    if model_cost is None:
+        return None
+
+    return (
+        token_usage.prompt_tokens * model_cost.input
+        + token_usage.completion_tokens * model_cost.completion
+    ) / _TOKENS_PER_MILLION

{unique_toolkit-2026.32.0.dev4 → unique_toolkit-2026.32.0.dev6}/unique_toolkit/monitoring/middleware.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import time
+from collections.abc import Sequence
 
 from prometheus_client import (  # pyright: ignore[reportMissingImports]
     Counter,
@@ -16,12 +17,6 @@ _http_requests_total = Counter(
     ["method", "path", "status_code"],
     registry=REGISTRY,
 )
-_http_request_duration_seconds = Histogram(
-    "python_http_request_duration_seconds",
-    "HTTP request duration",
-    ["method", "path"],
-    registry=REGISTRY,
-)
 _http_requests_in_progress = Gauge(
     "python_http_requests_in_progress",
     "In-flight HTTP requests",
@@ -30,6 +25,35 @@ _http_requests_in_progress = Gauge(
     multiprocess_mode="livesum",
 )
 
+# Created lazily on first MetricsMiddleware construction so services can choose the
+# bucket layout. Prometheus allows a metric name to be registered only once per
+# registry, so the buckets passed to the first middleware win for the process.
+_http_request_duration_seconds: Histogram | None = None
+
+
+def _get_or_create_duration_histogram(
+    buckets: Sequence[float] | None,
+) -> Histogram:
+    global _http_request_duration_seconds
+    if _http_request_duration_seconds is None:
+        if buckets is None:
+            _http_request_duration_seconds = Histogram(
+                "python_http_request_duration_seconds",
+                "HTTP request duration",
+                ["method", "path"],
+                registry=REGISTRY,
+            )
+        else:
+            _http_request_duration_seconds = Histogram(
+                "python_http_request_duration_seconds",
+                "HTTP request duration",
+                ["method", "path"],
+                registry=REGISTRY,
+                buckets=tuple(buckets),
+            )
+    return _http_request_duration_seconds
+
+
 _DEFAULT_EXCLUDED_PATHS = frozenset({"/health", "/metrics", "/"})
 
 
@@ -51,13 +75,29 @@ class MetricsMiddleware:
 
         # Quart
         app.asgi_app = MetricsMiddleware(app.asgi_app)
+
+    ``duration_buckets`` overrides the latency histogram buckets (seconds). Without
+    it, prometheus_client's defaults apply and their top finite bucket is 10s —
+    histogram_quantile cannot resolve past the largest finite bucket, so services
+    with slower endpoints should pass buckets covering their full latency range::
+
+        app.add_middleware(MetricsMiddleware, duration_buckets=(1, 5, 30, 60, 120))
+
+    The histogram is registered once per process; the buckets of the first
+    constructed middleware apply.
     """
 
-    def __init__(self, app, excluded_paths: set[str] | None = None):
+    def __init__(
+        self,
+        app,
+        excluded_paths: set[str] | None = None,
+        duration_buckets: Sequence[float] | None = None,
+    ):
         self.app = app
         self._excluded_paths = (
             excluded_paths if excluded_paths is not None else _DEFAULT_EXCLUDED_PATHS
         )
+        self._duration = _get_or_create_duration_histogram(duration_buckets)
 
     async def __call__(self, scope, receive, send):
         # Pass through non-HTTP (WebSocket, lifespan)
@@ -87,9 +127,7 @@ class MetricsMiddleware:
         finally:
             duration = time.perf_counter() - start
             _http_requests_in_progress.labels(method=method).dec()
-            _http_request_duration_seconds.labels(method=method, path=path).observe(
-                duration
-            )
+            self._duration.labels(method=method, path=path).observe(duration)
             _http_requests_total.labels(
                 method=method, path=path, status_code=status_code
             ).inc()