traceops 0.5.0__py3-none-any.whl

trace_ops/__init__.py

@@ -0,0 +1,153 @@
+"""traceops — record and replay LLM agent traces for deterministic regression testing.
+
+Framework-agnostic. Works with OpenAI, Anthropic, LiteLLM, LangChain, CrewAI,
+or any custom agent. No LLM calls during replay — tests are fast, free, deterministic.
+
+Quick start:
+    from trace_ops import Recorder, Replayer
+
+    # Record an agent run
+    with Recorder(save_to="cassettes/test_math.yaml") as rec:
+        result = agent.run("What is 2+2?")
+
+    # Replay deterministically (zero API calls)
+    with Replayer("cassettes/test_math.yaml"):
+        result = agent.run("What is 2+2?")
+        assert result == "4"
+
+    # Also works async:
+    async with Recorder(save_to="cassettes/test.yaml") as rec:
+        result = await agent.arun("What is 2+2?")
+"""
+
+from trace_ops._types import (
+    EventType,
+    Trace,
+    TraceEvent,
+    TraceMetadata,
+)
+from trace_ops.assertions import (
+    AgentLoopError,
+    BudgetExceededError,
+    assert_cost_under,
+    assert_max_llm_calls,
+    assert_no_loops,
+    assert_tokens_under,
+)
+from trace_ops.cassette import (
+    CassetteMismatchError,
+    CassetteNotFoundError,
+    load_cassette,
+    save_cassette,
+)
+from trace_ops.diff import TraceDiff, assert_trace_unchanged, diff_traces
+from trace_ops.normalize import (
+    NormalizedResponse,
+    NormalizedToolCall,
+    normalize_for_comparison,
+    normalize_response,
+)
+from trace_ops.recorder import Recorder
+from trace_ops.replayer import Replayer
+from trace_ops.reporters.cost_dashboard import CostDashboard, CostSummary
+
+# RAG add-on (graceful degradation if not installed)
+try:
+    from trace_ops.rag.diff import RAGDiffResult, diff_rag
+    from trace_ops.rag.assertions import (
+        RAGAssertionError,
+        assert_chunk_count,
+        assert_retrieval_latency,
+        assert_min_relevance_score,
+        assert_no_retrieval_drift,
+        assert_rag_scores,
+    )
+    from trace_ops.rag.scorers import RagasScorer, DeepEvalScorer
+    from trace_ops.rag.snapshot import RetrieverSnapshot
+    from trace_ops.rag.context_analysis import analyze_context_usage
+    _RAG_AVAILABLE = True
+except ImportError:
+    _RAG_AVAILABLE = False
+
+# MCP add-on
+try:
+    from trace_ops.mcp.diff import MCPDiffResult, diff_mcp
+    _MCP_AVAILABLE = True
+except ImportError:
+    _MCP_AVAILABLE = False
+
+# Semantic add-on
+try:
+    from trace_ops.semantic.similarity import SemanticDiffResult, semantic_similarity
+    from trace_ops.semantic.assertions import SemanticRegressionError, assert_semantic_similarity
+    _SEMANTIC_AVAILABLE = True
+except ImportError:
+    _SEMANTIC_AVAILABLE = False
+
+# Export add-on
+try:
+    from trace_ops.export.finetune import to_openai_finetune, to_anthropic_finetune
+    _EXPORT_AVAILABLE = True
+except ImportError:
+    _EXPORT_AVAILABLE = False
+
+__version__ = "0.5.0"
+
+__all__ = [
+    # Core
+    "Recorder",
+    "Replayer",
+    # Types
+    "Trace",
+    "TraceEvent",
+    "TraceMetadata",
+    "EventType",
+    # Cassette
+    "save_cassette",
+    "load_cassette",
+    "CassetteNotFoundError",
+    "CassetteMismatchError",
+    # Diff
+    "TraceDiff",
+    "diff_traces",
+    "assert_trace_unchanged",
+    # Normalization
+    "NormalizedToolCall",
+    "NormalizedResponse",
+    "normalize_response",
+    "normalize_for_comparison",
+    # Assertions
+    "assert_cost_under",
+    "assert_tokens_under",
+    "assert_max_llm_calls",
+    "assert_no_loops",
+    "BudgetExceededError",
+    "AgentLoopError",
+    # Reporters
+    "CostDashboard",
+    "CostSummary",
+    # RAG (available when trace_ops[rag] installed)
+    "diff_rag",
+    "RAGDiffResult",
+    "RAGAssertionError",
+    "assert_chunk_count",
+    "assert_retrieval_latency",
+    "assert_min_relevance_score",
+    "assert_no_retrieval_drift",
+    "assert_rag_scores",
+    "RagasScorer",
+    "DeepEvalScorer",
+    "RetrieverSnapshot",
+    "analyze_context_usage",
+    # MCP
+    "diff_mcp",
+    "MCPDiffResult",
+    # Semantic
+    "semantic_similarity",
+    "SemanticDiffResult",
+    "SemanticRegressionError",
+    "assert_semantic_similarity",
+    # Export / fine-tune
+    "to_openai_finetune",
+    "to_anthropic_finetune",
+]

trace_ops/_types.py

@@ -0,0 +1,356 @@
+"""Core data model for agent execution traces.
+
+A Trace is a complete recording of an agent run. It contains a sequence of
+Events — each event is either an LLM call, a tool invocation, or an agent
+decision. The trace captures everything needed to deterministically replay
+the agent's execution without making real API calls.
+
+Key design decision: we record at the SDK level (intercepting openai.chat.completions.create,
+anthropic.messages.create, etc.) rather than at the HTTP level (like VCR.py). This gives us
+semantic understanding of what happened — we know "this was a tool call" vs "this was a
+completion" — which HTTP-level recording can't distinguish.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import threading
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+from uuid import uuid4
+
+
+class EventType(str, Enum):
+    """Types of events in an agent trace."""
+
+    LLM_REQUEST = "llm_request"
+    LLM_RESPONSE = "llm_response"
+    TOOL_CALL = "tool_call"
+    TOOL_RESULT = "tool_result"
+    AGENT_DECISION = "agent_decision"
+    ERROR = "error"
+
+    # RAG events
+    RETRIEVAL = "retrieval"
+    EMBEDDING_CALL = "embedding_call"
+    RAG_SCORES = "rag_scores"
+
+    # MCP events
+    MCP_SERVER_CONNECT = "mcp_server_connect"
+    MCP_TOOL_CALL = "mcp_tool_call"
+    MCP_TOOL_RESULT = "mcp_tool_result"
+
+
+@dataclass
+class TraceEvent:
+    """A single event in an agent execution trace.
+
+    Events are the atoms of a trace. Each event records one interaction:
+    an LLM call, a tool invocation, or an agent-level decision.
+    """
+
+    event_type: EventType
+    timestamp: float = field(default_factory=time.time)
+    event_id: str = field(default_factory=lambda: uuid4().hex[:12])
+
+    # LLM-specific fields
+    provider: str | None = None  # "openai", "anthropic", "litellm"
+    model: str | None = None
+    messages: list[dict[str, Any]] | None = None  # input messages
+    response: dict[str, Any] | None = None  # full response object
+    temperature: float | None = None
+    max_tokens: int | None = None
+    tools: list[dict[str, Any]] | None = None  # tool definitions sent to LLM
+
+    # Tool-specific fields
+    tool_name: str | None = None
+    tool_input: dict[str, Any] | None = None
+    tool_output: Any = None
+
+    # Agent decision fields
+    decision: str | None = None  # e.g., "delegate_to_agent_b", "select_tool_search"
+    reasoning: str | None = None
+
+    # Error fields
+    error_type: str | None = None
+    error_message: str | None = None
+
+    # Cost tracking
+    input_tokens: int | None = None
+    output_tokens: int | None = None
+    cost_usd: float | None = None
+
+    # Timing
+    duration_ms: float | None = None
+
+    # Metadata
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    # RAG-specific fields
+    query: str | None = None                          # retrieval query text
+    chunks: list[dict[str, Any]] | None = None        # retrieved chunks [{id, text, score, metadata}]
+    vector_store: str | None = None                   # "chromadb", "pinecone", etc.
+    collection: str | None = None                     # collection / index name
+    top_k: int | None = None
+    total_chunks_searched: int | None = None
+    dimensions: int | None = None                     # embedding dimensions
+    scores: dict[str, float] | None = None            # RAG quality scores
+
+    # MCP-specific fields
+    server_name: str | None = None
+    server_url: str | None = None
+    capabilities: list[str] | None = None
+    arguments: dict[str, Any] | None = None
+    result: Any = None
+    is_error: bool | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a dict, dropping None fields for compact storage."""
+        d: dict[str, Any] = {
+            "event_type": self.event_type.value,
+            "timestamp": self.timestamp,
+            "event_id": self.event_id,
+        }
+        for key in [
+            "provider", "model", "messages", "response", "temperature",
+            "max_tokens", "tools", "tool_name", "tool_input", "tool_output",
+            "decision", "reasoning", "error_type", "error_message",
+            "input_tokens", "output_tokens", "cost_usd", "duration_ms",
+            # RAG fields
+            "query", "chunks", "vector_store", "collection", "top_k",
+            "total_chunks_searched", "dimensions", "scores",
+            # MCP fields
+            "server_name", "server_url", "capabilities", "arguments", "result", "is_error",
+        ]:
+            val = getattr(self, key)
+            if val is not None:
+                d[key] = val
+        if self.metadata:
+            d["metadata"] = self.metadata
+        return d
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> TraceEvent:
+        """Deserialize from a dict."""
+        data = dict(data)
+        data["event_type"] = EventType(data["event_type"])
+        return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
+
+
+@dataclass
+class TraceMetadata:
+    """Metadata about the trace recording environment."""
+
+    recorded_at: float = field(default_factory=time.time)
+    trace_ops_version: str = "0.5.0"
+    python_version: str = ""
+    framework: str | None = None  # "langchain", "crewai", "openai-agents-sdk", "custom"
+    description: str = ""
+    tags: list[str] = field(default_factory=list)
+    env: dict[str, str] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        d: dict[str, Any] = {
+            "recorded_at": self.recorded_at,
+            "trace_ops_version": self.trace_ops_version,
+        }
+        if self.python_version:
+            d["python_version"] = self.python_version
+        if self.framework:
+            d["framework"] = self.framework
+        if self.description:
+            d["description"] = self.description
+        if self.tags:
+            d["tags"] = self.tags
+        if self.env:
+            d["env"] = self.env
+        return d
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> TraceMetadata:
+        return cls(**{k: v for k, v in data.items() if k in cls.__dataclass_fields__})
+
+
+@dataclass
+class Trace:
+    """A complete recording of an agent execution.
+
+    This is the top-level object that gets saved to a cassette file.
+    It contains all events from a single agent run, plus metadata
+    about the recording environment.
+    """
+
+    trace_id: str = field(default_factory=lambda: uuid4().hex[:16])
+    events: list[TraceEvent] = field(default_factory=list)
+    metadata: TraceMetadata = field(default_factory=TraceMetadata)
+
+    # Summary stats (computed after recording)
+    total_llm_calls: int = 0
+    total_tool_calls: int = 0
+    total_tokens: int = 0
+    total_cost_usd: float = 0.0
+    total_duration_ms: float = 0.0
+
+    # Thread safety — protects events list and stats
+    _lock: threading.Lock = field(
+        default_factory=threading.Lock, init=False, repr=False, compare=False
+    )
+
+    def add_event(self, event: TraceEvent) -> None:
+        """Add an event to the trace (thread-safe)."""
+        with self._lock:
+            self.events.append(event)
+            self._update_stats(event)
+
+    def _update_stats(self, event: TraceEvent) -> None:
+        """Update summary statistics after adding an event."""
+        if event.event_type == EventType.LLM_RESPONSE:
+            self.total_llm_calls += 1
+            if event.input_tokens:
+                self.total_tokens += event.input_tokens
+            if event.output_tokens:
+                self.total_tokens += event.output_tokens
+            if event.cost_usd:
+                self.total_cost_usd += event.cost_usd
+        elif event.event_type == EventType.TOOL_RESULT:
+            self.total_tool_calls += 1
+        elif event.event_type == EventType.EMBEDDING_CALL:
+            if event.cost_usd:
+                self.total_cost_usd += event.cost_usd
+        if event.duration_ms:
+            self.total_duration_ms += event.duration_ms
+
+    def finalize(self) -> None:
+        """Compute final stats after recording is complete."""
+        self.total_llm_calls = sum(
+            1 for e in self.events if e.event_type == EventType.LLM_RESPONSE
+        )
+        self.total_tool_calls = sum(
+            1 for e in self.events if e.event_type == EventType.TOOL_RESULT
+        )
+        self.total_tokens = sum(
+            (e.input_tokens or 0) + (e.output_tokens or 0)
+            for e in self.events
+            if e.event_type == EventType.LLM_RESPONSE
+        )
+        self.total_cost_usd = sum(
+            e.cost_usd or 0.0
+            for e in self.events
+            if e.cost_usd is not None
+        )
+        self.total_duration_ms = sum(
+            e.duration_ms or 0.0 for e in self.events if e.duration_ms
+        )
+
+    @property
+    def llm_events(self) -> list[TraceEvent]:
+        """Get only LLM request/response events."""
+        return [
+            e for e in self.events
+            if e.event_type in (EventType.LLM_REQUEST, EventType.LLM_RESPONSE)
+        ]
+
+    @property
+    def tool_events(self) -> list[TraceEvent]:
+        """Get only tool call/result events."""
+        return [
+            e for e in self.events
+            if e.event_type in (EventType.TOOL_CALL, EventType.TOOL_RESULT)
+        ]
+
+    @property
+    def retrieval_events(self) -> list[TraceEvent]:
+        """Get all retrieval events (RAG vector store queries)."""
+        return [e for e in self.events if e.event_type == EventType.RETRIEVAL]
+
+    @property
+    def embedding_events(self) -> list[TraceEvent]:
+        """Get all embedding call events."""
+        return [e for e in self.events if e.event_type == EventType.EMBEDDING_CALL]
+
+    @property
+    def mcp_events(self) -> list[TraceEvent]:
+        """Get all MCP-related events."""
+        return [
+            e for e in self.events
+            if e.event_type in (
+                EventType.MCP_SERVER_CONNECT,
+                EventType.MCP_TOOL_CALL,
+                EventType.MCP_TOOL_RESULT,
+            )
+        ]
+
+    @property
+    def rag_scores(self) -> dict[str, float] | None:
+        """Get cached RAG quality scores from the cassette, if any."""
+        for e in self.events:
+            if e.event_type == EventType.RAG_SCORES and e.scores:
+                return e.scores
+        return None
+
+    @property
+    def trajectory(self) -> list[str]:
+        """Get the high-level trajectory as a list of step descriptions.
+
+        Returns something like:
+            ["llm_call:gpt-4o", "tool:search_files", "llm_call:gpt-4o", "tool:read_file"]
+        """
+        steps = []
+        for event in self.events:
+            if event.event_type == EventType.LLM_REQUEST:
+                steps.append(f"llm_call:{event.model or 'unknown'}")
+            elif event.event_type == EventType.TOOL_CALL:
+                steps.append(f"tool:{event.tool_name or 'unknown'}")
+            elif event.event_type == EventType.AGENT_DECISION:
+                steps.append(f"decision:{event.decision or 'unknown'}")
+            elif event.event_type == EventType.ERROR:
+                steps.append(f"error:{event.error_type or 'unknown'}")
+            elif event.event_type == EventType.RETRIEVAL:
+                steps.append(f"retrieval:{event.vector_store or 'unknown'}")
+            elif event.event_type == EventType.EMBEDDING_CALL:
+                steps.append(f"embedding:{event.model or 'unknown'}")
+            elif event.event_type == EventType.MCP_TOOL_CALL:
+                steps.append(f"mcp:{event.server_name or 'unknown'}.{event.tool_name or 'unknown'}")
+        return steps
+
+    def fingerprint(self) -> str:
+        """Generate a hash fingerprint of the trajectory.
+
+        Two traces with the same fingerprint took the same path
+        (same sequence of LLM calls, tool calls, and decisions).
+        """
+        trajectory_str = "|".join(self.trajectory)
+        return hashlib.sha256(trajectory_str.encode()).hexdigest()[:16]
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize the full trace to a dict."""
+        return {
+            "version": "1",
+            "trace_id": self.trace_id,
+            "metadata": self.metadata.to_dict(),
+            "events": [e.to_dict() for e in self.events],
+            "summary": {
+                "total_llm_calls": self.total_llm_calls,
+                "total_tool_calls": self.total_tool_calls,
+                "total_tokens": self.total_tokens,
+                "total_cost_usd": self.total_cost_usd,
+                "total_duration_ms": self.total_duration_ms,
+                "trajectory": self.trajectory,
+                "fingerprint": self.fingerprint(),
+            },
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> Trace:
+        """Deserialize a trace from a dict."""
+        trace = cls(
+            trace_id=data.get("trace_id", uuid4().hex[:16]),
+            metadata=TraceMetadata.from_dict(data.get("metadata", {})),
+        )
+        for event_data in data.get("events", []):
+            trace.events.append(TraceEvent.from_dict(event_data))
+        trace.finalize()
+        return trace

trace_ops/assertions.py

@@ -0,0 +1,129 @@
+"""Budget and behavioural assertions for agent traces.
+
+These helpers let you guard against cost overruns, token bloat,
+excessive LLM round-trips, and infinite tool-call loops directly
+inside your test suite.
+
+Usage::
+
+    from trace_ops.assertions import assert_cost_under, assert_no_loops
+
+    with Recorder() as rec:
+        agent.run("Summarize the report")
+
+    assert_cost_under(rec.trace, max_usd=0.50)
+    assert_no_loops(rec.trace)
+"""
+
+from __future__ import annotations
+
+from trace_ops._types import EventType, Trace
+
+
+class BudgetExceededError(AssertionError):
+    """Raised when an agent trace exceeds a defined budget."""
+
+
+class AgentLoopError(AssertionError):
+    """Raised when an agent trace exhibits loop-like behaviour."""
+
+
+# ── Public assertion functions ──────────────────────────────────────
+
+
+def assert_cost_under(trace: Trace, *, max_usd: float) -> None:
+    """Assert that total cost of a trace is within budget.
+
+    Args:
+        trace: The recorded agent trace.
+        max_usd: Maximum allowed cost in US dollars.
+
+    Raises:
+        BudgetExceededError: If ``trace.total_cost_usd`` exceeds *max_usd*.
+    """
+    if trace.total_cost_usd > max_usd:
+        raise BudgetExceededError(
+            f"Trace cost ${trace.total_cost_usd:.4f} exceeds budget of ${max_usd:.4f}.\n"
+            f"The agent made {trace.total_llm_calls} LLM calls "
+            f"using {trace.total_tokens:,} tokens.\n"
+            f"Optimise prompts or reduce tool-call loops to lower cost."
+        )
+
+
+def assert_tokens_under(trace: Trace, *, max_tokens: int) -> None:
+    """Assert that total token usage is within a limit.
+
+    Args:
+        trace: The recorded agent trace.
+        max_tokens: Maximum allowed token count (input + output).
+
+    Raises:
+        BudgetExceededError: If ``trace.total_tokens`` exceeds *max_tokens*.
+    """
+    if trace.total_tokens > max_tokens:
+        raise BudgetExceededError(
+            f"Trace used {trace.total_tokens:,} tokens, "
+            f"exceeding limit of {max_tokens:,}.\n"
+            f"The agent made {trace.total_llm_calls} LLM calls.\n"
+            f"Reduce prompt size or limit tool-call depth."
+        )
+
+
+def assert_max_llm_calls(trace: Trace, *, max_calls: int) -> None:
+    """Assert that the agent didn't make too many LLM round-trips.
+
+    Args:
+        trace: The recorded agent trace.
+        max_calls: Maximum allowed LLM calls.
+
+    Raises:
+        BudgetExceededError: If ``trace.total_llm_calls`` exceeds *max_calls*.
+    """
+    if trace.total_llm_calls > max_calls:
+        raise BudgetExceededError(
+            f"Trace made {trace.total_llm_calls} LLM calls, "
+            f"exceeding limit of {max_calls}.\n"
+            f"Trajectory: {' → '.join(trace.trajectory)}\n"
+            f"The agent may be stuck in a loop or using an inefficient strategy."
+        )
+
+
+def assert_no_loops(
+    trace: Trace,
+    *,
+    max_consecutive_same_tool: int = 3,
+) -> None:
+    """Assert that the trace doesn't contain tool-call loops.
+
+    Scans for *N* consecutive ``TOOL_CALL`` events with the same
+    ``tool_name``.  Such runs typically indicate the agent is stuck
+    retrying the same action.
+
+    Args:
+        trace: The recorded agent trace.
+        max_consecutive_same_tool: Maximum allowed consecutive calls
+            to the same tool before raising.
+
+    Raises:
+        AgentLoopError: If a run of same-tool calls exceeds the limit.
+    """
+    tool_events = [
+        e for e in trace.events if e.event_type == EventType.TOOL_CALL
+    ]
+    if not tool_events:
+        return
+
+    consecutive = 1
+    for i in range(1, len(tool_events)):
+        if tool_events[i].tool_name == tool_events[i - 1].tool_name:
+            consecutive += 1
+            if consecutive > max_consecutive_same_tool:
+                raise AgentLoopError(
+                    f"Detected {consecutive} consecutive calls to tool "
+                    f"'{tool_events[i].tool_name}' "
+                    f"(limit: {max_consecutive_same_tool}).\n"
+                    f"The agent may be stuck in an infinite loop.\n"
+                    f"Check the agent's exit conditions or add loop guards."
+                )
+        else:
+            consecutive = 1