agentspend-sdk 0.1.0__py3-none-any.whl

agentspend_sdk-0.1.0.dist-info/METADATA

@@ -0,0 +1,131 @@
+Metadata-Version: 2.4
+Name: agentspend-sdk
+Version: 0.1.0
+Summary: AgentSpend — runtime cost optimizer for AI agents. Route LLM calls to the cheapest capable model with fallbacks, loop guards, and telemetry.
+Requires-Python: <3.14,>=3.12
+Requires-Dist: aiosqlite>=0.20
+Requires-Dist: alembic>=1.14
+Requires-Dist: fastapi>=0.115
+Requires-Dist: google-auth>=2.48.0
+Requires-Dist: google-cloud-aiplatform>=1.139.0
+Requires-Dist: litellm>=1.50
+Requires-Dist: pandas>=2.2
+Requires-Dist: pydantic-settings>=2.0
+Requires-Dist: pydantic>=2.0
+Requires-Dist: python-multipart>=0.0.9
+Requires-Dist: pyyaml>=6.0
+Requires-Dist: rich>=13.0
+Requires-Dist: sqlalchemy>=2.0
+Requires-Dist: typer>=0.15
+Requires-Dist: uvicorn[standard]>=0.32
+Provides-Extra: dev
+Requires-Dist: pytest>=9.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# token-aud
+
+AI cost optimization toolkit for LLM workloads.
+
+`token-aud` now includes two complementary workflows:
+
+- **Audit mode (CLI/API):** Analyze historical usage logs and estimate savings opportunities with Student-Teacher-Judge sampling.
+- **AgentSpend SDK:** Route live agent steps (`plan`, `reason`, `tool`, `verify`, `draft`, `summarize`) to cost/quality-appropriate models with fallbacks, loop guards, and telemetry.
+
+## Installation
+
+```bash
+uv sync --no-editable
+```
+
+For local development tooling (tests):
+
+```bash
+uv sync --no-editable --extra dev
+```
+
+## Quick Start (AgentSpend)
+
+Run these three commands from repo root:
+
+```bash
+uv sync --no-editable --extra dev
+uv run --no-sync python -m pytest tests/agent -q
+uv run --no-sync python examples/agent_routing_demo.py
+```
+
+Expected results:
+
+- Agent tests pass.
+- Demo prints routed step decisions, per-step telemetry, and total run cost.
+- `agent_telemetry.jsonl` is generated locally.
+
+## AgentSpend Usage
+
+### 1) Default policy
+
+```python
+from token_aud.agent import AgentSpend
+
+agent = AgentSpend.default()
+result = agent.route_call(
+    step="plan",
+    messages=[{"role": "user", "content": "Break this task into a plan"}],
+)
+
+print(result.model_used, result.cost_usd, result.content)
+```
+
+### 2) Custom policy YAML
+
+```python
+from token_aud.agent import AgentSpend
+
+agent = AgentSpend.from_yaml("routing_policy.yaml")
+result = agent.route_call(
+    step="reason",
+    messages=[{"role": "user", "content": "Compare two architectures"}],
+)
+
+print(result.model_used, result.fallbacks_tried)
+```
+
+Built-in default policy path:
+
+- `src/token_aud/data/default_routing_policy.yaml`
+
+## AgentSpend Core Components
+
+- `src/token_aud/agent/policy.py`: Pydantic policy schema + YAML loading
+- `src/token_aud/agent/router.py`: deterministic model selection
+- `src/token_aud/agent/runtime.py`: `route_call()` execution + fallbacks
+- `src/token_aud/agent/loop_guard.py`: repeated-turn loop detection
+- `src/token_aud/agent/telemetry.py`: JSONL/HTTP telemetry sinks
+- `src/token_aud/agent/adaptive.py`: optional adaptive routing layer
+
+## AgentSpend Examples
+
+- `examples/agent_routing_demo.py`: end-to-end routed run with telemetry
+- `examples/custom_policy_demo.py`: loop escalation and hard-stop behavior
+- `examples/framework_agnostic_integration.py`: generic agent-loop integration with explicit success feedback
+- `scripts/summarize_telemetry.py`: convert `agent_telemetry.jsonl` into cost/fallback/latency summary
+
+```bash
+uv run --no-sync python scripts/summarize_telemetry.py agent_telemetry.jsonl
+```
+
+## Audit CLI (legacy + still supported)
+
+```bash
+uv run --no-sync token-aud --help
+uv run --no-sync token-aud analyze sample_data.csv --dry-run
+```
+
+## Environment Variables
+
+Common provider credentials:
+
+- `OPENAI_API_KEY`
+- `ANTHROPIC_API_KEY`
+- `GEMINI_API_KEY` or `GOOGLE_API_KEY` (depending on provider path)
+
+For Google Vertex flows, ensure ADC is configured (`gcloud auth application-default login`).

agentspend_sdk-0.1.0.dist-info/RECORD

@@ -0,0 +1,44 @@
+token_aud/__init__.py,sha256=OG88Yz6qUP3fEmFr1pMe9azFeKICUErBif0uPcZ32is,89
+token_aud/config.py,sha256=qjtn77ezAgllzGig-eP-YT9HX8XrIyKN8hqngDI9LBU,1425
+token_aud/agent/__init__.py,sha256=upJj-JKnQ0Kds4ISsS8056MGh7i-ZVDaLNw9FQrMjns,1105
+token_aud/agent/adaptive.py,sha256=dwne4Tb_fnj-vr8pw-KAzJiyC-7F3MMRJts337Lnd-I,7575
+token_aud/agent/loop_guard.py,sha256=3fFUcEEq8HcPDm93JXN3oRMpHI7OcDRTTMaT9Uvuww0,3246
+token_aud/agent/policy.py,sha256=Jcf8LsDHPGp9GwgbByGGmb_ZIhmdJ8LZdTkNAliFEEA,5120
+token_aud/agent/router.py,sha256=Nzf1-Buj3YjkbUs5dY882hfh-ElGHOtYEKhsL2nAZb0,8570
+token_aud/agent/runtime.py,sha256=alDlarqPj0UpXULbAib3QeuT0sz199Xm48ajfINf3hA,12883
+token_aud/agent/step_classifier.py,sha256=c7GOZR28-W4xM-jbJ7qcos012phn5b2_WCGQqeq8JCg,2101
+token_aud/agent/telemetry.py,sha256=JrW7rM8ykYbrBprQXLODYx5cWyhtOmEleInXM-wQhcA,4308
+token_aud/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+token_aud/api/app.py,sha256=VgFvLysKxJdPd2Awxo-t3sVTP5ag06o5TYW6IKe06c0,4830
+token_aud/api/serve.py,sha256=t20bOSJ-eYm0s7x6ctuJ7if6GpL0vvJp4E8Oxsw3hTg,194
+token_aud/api/routes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+token_aud/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+token_aud/cli/analyze.py,sha256=RAtTxtq0t1hQpbK6R2IQuNme34inaSV_DQOfVHDD7Aw,7802
+token_aud/cli/configure.py,sha256=q88mVw7LU2Vv1_asJrDyOTF_caDQYyjnwR306Ls4Ev4,3598
+token_aud/cli/main.py,sha256=pOxnOZypVp882Mf3zUW1nz0dv6EkxWIwHlP0bVuGKrw,692
+token_aud/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+token_aud/core/auditor.py,sha256=3ptT4GadRVgXa2lYYh1YM9x2NfAZ5tFJ8HOpKVZ2POA,11309
+token_aud/core/judge.py,sha256=Ew3bpEnw3q9bYctRleVpO5VkhtOIo1D7Azh9fn29R-8,6763
+token_aud/core/pricing.py,sha256=bIPlpEBymdKG0YEkUfp4QHTjS4yLkI4znnk8N8H2bA4,5108
+token_aud/core/sampler.py,sha256=Gujbb8emZWV0I1DWJ9jfR7m3HFRDntS1U4foBaa3nGY,11126
+token_aud/core/savings.py,sha256=LVdhbzT7glp2xDVKUfFaLx_ObdhJUkB7Gbgssw6OXB0,9546
+token_aud/data/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+token_aud/data/default_routing_policy.yaml,sha256=M6Rpm0bGp4MujPj73mzZMaftFuAvnAOYgfL0EPC62r8,4852
+token_aud/data/pricing.json,sha256=4xsTV_h2zFwXREgD9P4s8lEaQPf_xvJGpKIR_Y80kbM,11218
+token_aud/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+token_aud/db/session.py,sha256=0Tpm5I5eu1r5KEtlsGUmmve79uIIGhY6eirSzT8fDQQ,959
+token_aud/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+token_aud/models/db.py,sha256=U7r-oe1tr3KMGRbcjmhf3vkUD4yhMWVHrrs2OwQvE8I,3972
+token_aud/models/schemas.py,sha256=JHrH9nqwQsl0njCzd8GkS3Nta7PBr6XlvCcqs9Td-gs,4582
+token_aud/parsers/__init__.py,sha256=9EoHC62-G5nvkFUMW7vL7OHUlRUy2QL1hxV1Dj6bMo0,264
+token_aud/parsers/anthropic.py,sha256=3jgW3uykihEABLy2a-21kDEshcgh2ITIBunmelqLmxE,1527
+token_aud/parsers/base.py,sha256=KRJy_CVXpsYyFDQse_GUXch4afKFjOMQJcI7FFx2vJk,7481
+token_aud/parsers/generic_csv.py,sha256=6s7BzHPyx49tHi9GzYWGcFmr3jwaTzZuv9hCV5CQQ4Y,1409
+token_aud/parsers/openai.py,sha256=T-O6JbRM4XcNIXv4XYQqfIvvRioMkBqbe8hwwNhDPjI,1505
+token_aud/reports/__init__.py,sha256=wMW8GxhQBrN6fwfQbOEE1LBsQSH_Lgcdk_ra_dvAdV8,203
+token_aud/reports/html.py,sha256=iVGq9ecvE0VAkKriZEbJSaxd7T4LZepj2iDOTTIZgVM,9198
+token_aud/reports/terminal.py,sha256=xnWzZb8Uhv_I35Z1Htcjhg3xhItmeiL5wDOMLRvG56M,3987
+agentspend_sdk-0.1.0.dist-info/METADATA,sha256=rHvsCL2yHS7zUJ5jLcjdp1-kti87c2fSjAI3wMi3ccY,3811
+agentspend_sdk-0.1.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+agentspend_sdk-0.1.0.dist-info/entry_points.txt,sha256=Gu8y3XoWEBEcGVYCOfzRVyBKdUjjP5WzLFy1-hCn8Xg,96
+agentspend_sdk-0.1.0.dist-info/RECORD,,

agentspend_sdk-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

agentspend_sdk-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+token-aud = token_aud.cli.main:app
+token-aud-serve = token_aud.api.serve:main

token_aud/__init__.py

@@ -0,0 +1,3 @@
+"""token-aud: AI cost optimization through model benchmarking."""
+
+__version__ = "0.1.0"

token_aud/agent/__init__.py

@@ -0,0 +1,41 @@
+"""AgentSpend SDK — agent-native dynamic model routing.
+
+Public API:
+    from token_aud.agent import AgentSpend, RoutingPolicy, RouteContext
+
+    agent = AgentSpend.from_yaml("routing_policy.yaml")
+    result = agent.route_call(step="plan", messages=[...])
+"""
+
+from token_aud.agent.adaptive import AdaptiveRouter, AdaptiveSuggestion
+from token_aud.agent.loop_guard import LoopGuard
+from token_aud.agent.policy import RoutingPolicy, StepType
+from token_aud.agent.router import RouteContext, RouteDecision, decide_model
+from token_aud.agent.runtime import AgentSpend, RouteResult
+from token_aud.agent.step_classifier import classify_step
+from token_aud.agent.telemetry import (
+    CallbackSink,
+    HttpSink,
+    JsonlSink,
+    TelemetryEmitter,
+    TelemetryEvent,
+)
+
+__all__ = [
+    "AdaptiveRouter",
+    "AdaptiveSuggestion",
+    "AgentSpend",
+    "CallbackSink",
+    "HttpSink",
+    "JsonlSink",
+    "LoopGuard",
+    "RouteContext",
+    "RouteDecision",
+    "RouteResult",
+    "RoutingPolicy",
+    "StepType",
+    "TelemetryEmitter",
+    "TelemetryEvent",
+    "classify_step",
+    "decide_model",
+]

token_aud/agent/adaptive.py

@@ -0,0 +1,228 @@
+"""Adaptive routing — learns from telemetry outcomes to adjust model selection.
+
+This module provides an optional layer on top of the deterministic router.
+It tracks per-step success rates and costs, then suggests cheaper models
+when confidence is high enough.
+
+Key design constraints:
+- Deterministic mode is always the default and the safe fallback.
+- Adaptive mode requires a minimum sample size before overriding.
+- A rollback switch instantly disables adaptation.
+- All state is local and serializable (no external DB needed).
+
+Usage:
+    from token_aud.agent.adaptive import AdaptiveRouter
+
+    adaptive = AdaptiveRouter(min_samples=20, confidence_threshold=0.85)
+    adaptive.record_outcome(step="plan", model="gpt-4o-mini", success=True, cost=0.001)
+    suggestion = adaptive.suggest(step="plan", current_model="gpt-4o")
+"""
+
+from __future__ import annotations
+
+import json
+from collections import defaultdict
+from dataclasses import dataclass, field
+from pathlib import Path
+
+
+@dataclass
+class ModelStats:
+    """Accumulated statistics for a model on a specific step type."""
+
+    total_calls: int = 0
+    success_count: int = 0
+    total_cost: float = 0.0
+    total_latency_ms: float = 0.0
+
+    @property
+    def success_rate(self) -> float:
+        if self.total_calls == 0:
+            return 0.0
+        return self.success_count / self.total_calls
+
+    @property
+    def avg_cost(self) -> float:
+        if self.total_calls == 0:
+            return 0.0
+        return self.total_cost / self.total_calls
+
+    @property
+    def avg_latency_ms(self) -> float:
+        if self.total_calls == 0:
+            return 0.0
+        return self.total_latency_ms / self.total_calls
+
+
+@dataclass
+class AdaptiveSuggestion:
+    """A suggestion from the adaptive router."""
+
+    suggested_model: str | None
+    reason: str
+    confidence: float
+    stats: ModelStats | None = None
+
+
+class AdaptiveRouter:
+    """Learns from outcomes to suggest cheaper models per step type.
+
+    This is opt-in and sits alongside the deterministic router. It does not
+    replace it — the runtime checks this for suggestions before falling back
+    to the deterministic path.
+    """
+
+    def __init__(
+        self,
+        min_samples: int = 20,
+        confidence_threshold: float = 0.85,
+        enabled: bool = True,
+    ) -> None:
+        self._min_samples = min_samples
+        self._confidence_threshold = confidence_threshold
+        self._enabled = enabled
+        self._stats: dict[str, dict[str, ModelStats]] = defaultdict(
+            lambda: defaultdict(ModelStats)
+        )
+
+    @property
+    def enabled(self) -> bool:
+        return self._enabled
+
+    def enable(self) -> None:
+        self._enabled = True
+
+    def disable(self) -> None:
+        """Rollback switch — immediately disables adaptive suggestions."""
+        self._enabled = False
+
+    def record_outcome(
+        self,
+        step: str,
+        model: str,
+        success: bool,
+        cost: float = 0.0,
+        latency_ms: float = 0.0,
+    ) -> None:
+        """Record the outcome of a routed call for learning."""
+        stats = self._stats[step][model]
+        stats.total_calls += 1
+        if success:
+            stats.success_count += 1
+        stats.total_cost += cost
+        stats.total_latency_ms += latency_ms
+
+    def suggest(
+        self,
+        step: str,
+        current_model: str,
+        candidate_models: list[str] | None = None,
+    ) -> AdaptiveSuggestion:
+        """Suggest a potentially cheaper model based on historical success rates.
+
+        Returns a suggestion only if:
+        1. Adaptive routing is enabled
+        2. There's enough data (>= min_samples) for the candidate
+        3. The candidate's success rate >= confidence_threshold
+        4. The candidate is cheaper on average than the current model
+        """
+        if not self._enabled:
+            return AdaptiveSuggestion(
+                suggested_model=None,
+                reason="Adaptive routing disabled",
+                confidence=0.0,
+            )
+
+        step_stats = self._stats.get(step, {})
+        current_stats = step_stats.get(current_model)
+
+        if candidate_models is None:
+            candidate_models = [m for m in step_stats if m != current_model]
+
+        best: AdaptiveSuggestion | None = None
+
+        for candidate in candidate_models:
+            cand_stats = step_stats.get(candidate)
+            if cand_stats is None or cand_stats.total_calls < self._min_samples:
+                continue
+
+            if cand_stats.success_rate < self._confidence_threshold:
+                continue
+
+            if current_stats and cand_stats.avg_cost >= current_stats.avg_cost:
+                continue
+
+            suggestion = AdaptiveSuggestion(
+                suggested_model=candidate,
+                reason=(
+                    f"{candidate} has {cand_stats.success_rate:.0%} success rate "
+                    f"over {cand_stats.total_calls} calls at ${cand_stats.avg_cost:.6f}/call avg"
+                ),
+                confidence=cand_stats.success_rate,
+                stats=cand_stats,
+            )
+
+            if best is None or (cand_stats.avg_cost < (best.stats.avg_cost if best.stats else float("inf"))):
+                best = suggestion
+
+        if best is not None:
+            return best
+
+        return AdaptiveSuggestion(
+            suggested_model=None,
+            reason="No confident cheaper alternative found",
+            confidence=0.0,
+        )
+
+    def get_stats(self, step: str | None = None) -> dict:
+        """Return stats as a plain dict for inspection or serialization."""
+        if step:
+            return {
+                model: {
+                    "total_calls": s.total_calls,
+                    "success_rate": s.success_rate,
+                    "avg_cost": s.avg_cost,
+                    "avg_latency_ms": s.avg_latency_ms,
+                }
+                for model, s in self._stats.get(step, {}).items()
+            }
+        return {
+            step_name: {
+                model: {
+                    "total_calls": s.total_calls,
+                    "success_rate": s.success_rate,
+                    "avg_cost": s.avg_cost,
+                    "avg_latency_ms": s.avg_latency_ms,
+                }
+                for model, s in models.items()
+            }
+            for step_name, models in self._stats.items()
+        }
+
+    def save(self, path: str | Path) -> None:
+        """Persist stats to a JSON file."""
+        raw: dict = {}
+        for step_name, models in self._stats.items():
+            raw[step_name] = {}
+            for model, s in models.items():
+                raw[step_name][model] = {
+                    "total_calls": s.total_calls,
+                    "success_count": s.success_count,
+                    "total_cost": s.total_cost,
+                    "total_latency_ms": s.total_latency_ms,
+                }
+        Path(path).write_text(json.dumps(raw, indent=2))
+
+    def load(self, path: str | Path) -> None:
+        """Load stats from a previously saved JSON file."""
+        p = Path(path)
+        if not p.exists():
+            return
+        raw = json.loads(p.read_text())
+        for step_name, models in raw.items():
+            for model, data in models.items():
+                stats = self._stats[step_name][model]
+                stats.total_calls = data["total_calls"]
+                stats.success_count = data["success_count"]
+                stats.total_cost = data["total_cost"]
+                stats.total_latency_ms = data.get("total_latency_ms", 0.0)

token_aud/agent/loop_guard.py

@@ -0,0 +1,102 @@
+"""Loop guard — detects agent loops from repeated similar messages.
+
+Tracks recent message fingerprints per session. When consecutive turns
+have high textual similarity or identical content, signals a loop so the
+router can escalate or halt.
+
+Budget guard logic lives in router.py (it's part of the routing decision).
+This module focuses on turn-level repetition detection.
+"""
+
+from __future__ import annotations
+
+from collections import deque
+from difflib import SequenceMatcher
+
+from token_aud.agent.policy import LoopGuardAction, LoopGuardConfig
+
+
+class LoopGuard:
+    """Stateful loop detector for an agent run."""
+
+    def __init__(self, config: LoopGuardConfig) -> None:
+        self._config = config
+        self._recent_fingerprints: deque[str] = deque(
+            maxlen=config.repeated_turn_limit + 2
+        )
+        self._consecutive_similar: int = 0
+        self._escalation_count: int = 0
+
+    def reset(self) -> None:
+        """Clear all state between runs."""
+        self._recent_fingerprints.clear()
+        self._consecutive_similar = 0
+        self._escalation_count = 0
+
+    def check(self, messages: list[dict[str, str]]) -> bool:
+        """Check if the current turn looks like a loop.
+
+        Args:
+            messages: The message list about to be sent to the LLM.
+
+        Returns:
+            True if a loop is detected.
+        """
+        if not self._config.enabled:
+            return False
+
+        fingerprint = self._fingerprint(messages)
+        if not fingerprint:
+            return False
+
+        is_similar = self._is_similar_to_recent(fingerprint)
+        self._recent_fingerprints.append(fingerprint)
+
+        if is_similar:
+            self._consecutive_similar += 1
+        else:
+            self._consecutive_similar = 0
+
+        if self._consecutive_similar >= self._config.repeated_turn_limit:
+            return True
+
+        return False
+
+    def record_escalation(self) -> None:
+        """Record that the router escalated due to a loop detection."""
+        self._escalation_count += 1
+
+    @property
+    def should_hard_stop(self) -> bool:
+        """True if escalation count exceeds the hard_stop_after threshold."""
+        limit = self._config.on_trigger.hard_stop_after
+        return self._escalation_count >= limit
+
+    @property
+    def action(self) -> LoopGuardAction:
+        return self._config.on_trigger.action
+
+    @property
+    def escalation_count(self) -> int:
+        return self._escalation_count
+
+    # --- Internals ---
+
+    def _fingerprint(self, messages: list[dict[str, str]]) -> str:
+        """Extract a comparable fingerprint from the last message."""
+        if not messages:
+            return ""
+        for msg in reversed(messages):
+            content = msg.get("content", "")
+            if content:
+                return content.strip()[:500]
+        return ""
+
+    def _is_similar_to_recent(self, fingerprint: str) -> bool:
+        """Check if the fingerprint is similar to any recent one."""
+        threshold = self._config.similarity_threshold
+        for recent in self._recent_fingerprints:
+            ratio = SequenceMatcher(None, fingerprint, recent).ratio()
+            if ratio >= threshold:
+                return True
+        return False