agent-trajectory-diff 0.1.0__py3-none-any.whl

agentdiff/adapters/base.py

@@ -0,0 +1,20 @@
+import json
+from abc import ABC, abstractmethod
+from typing import Any
+
+from agentdiff.models.trace import AgentTrace
+
+
+class BaseAdapter(ABC):
+    @classmethod
+    @abstractmethod
+    def from_dict(cls, data: dict[str, Any]) -> AgentTrace:
+        """Parses a raw dictionary input into a canonical AgentTrace."""
+        pass
+
+    @classmethod
+    def from_file(cls, filepath: str) -> AgentTrace:
+        """Loads a JSON file and parses it into a canonical AgentTrace."""
+        with open(filepath, encoding="utf-8") as f:
+            data = json.load(f)
+        return cls.from_dict(data)

agentdiff/adapters/deepeval.py

@@ -0,0 +1,134 @@
+from typing import Any
+
+from agentdiff.adapters.base import BaseAdapter
+from agentdiff.models.step import StepStatus, StepType, TokenUsage, TraceStep
+from agentdiff.models.trace import AgentTrace
+
+
+class DeepEvalAdapter(BaseAdapter):
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> AgentTrace:
+        """Parses a DeepEval trace dictionary into a canonical AgentTrace."""
+        if isinstance(data, list):
+            if not data:
+                raise ValueError("DeepEval trace list is empty")
+            data = data[0]
+
+        trace_id = data.get("id") or data.get("trace_id") or "deepeval_trace"
+        agent_name = data.get("name") or data.get("agent_name") or "deepeval_agent"
+
+        task_input = data.get("input")
+        if not isinstance(task_input, dict):
+            task_input = {"input": task_input}
+
+        final_output = data.get("output")
+        if not isinstance(final_output, dict):
+            final_output = {"output": final_output}
+
+        steps: list[TraceStep] = []
+
+        # Traverse child spans/nodes
+        spans = data.get("spans") or data.get("children") or data.get("nodes") or []
+        for span in spans:
+            cls._parse_span(span, None, steps)
+
+        total_latency_ms = data.get("latency") or data.get("executionTime") or 0.0
+        if total_latency_ms < 100.0 and total_latency_ms > 0:
+            total_latency_ms *= 1000.0
+
+        prompt_tokens = sum(s.tokens.prompt_tokens for s in steps)
+        completion_tokens = sum(s.tokens.completion_tokens for s in steps)
+        total_tokens = sum(s.tokens.total_tokens for s in steps)
+        cost = sum(s.tokens.estimated_cost_usd for s in steps)
+
+        total_tokens_obj = TokenUsage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens,
+            estimated_cost_usd=cost,
+        )
+
+        return AgentTrace(
+            trace_id=trace_id,
+            agent_name=agent_name,
+            task_input=task_input,
+            final_output=final_output,
+            steps=steps,
+            total_latency_ms=total_latency_ms,
+            total_tokens=total_tokens_obj,
+            metadata=data.get("metadata") or {},
+        )
+
+    @classmethod
+    def _parse_span(
+        cls, span: dict[str, Any], parent_id: str | None, steps_list: list[TraceStep]
+    ):
+        """Recursively parses DeepEval spans and adds them to steps_list."""
+        span_id = span.get("id") or span.get("span_id") or f"step_{len(steps_list)}"
+        span_type_str = str(span.get("type", "")).lower()
+
+        # Map span type to StepType
+        if "tool" in span_type_str or "retriever" in span_type_str:
+            step_type = StepType.TOOL_CALL
+        elif "llm" in span_type_str:
+            step_type = StepType.LLM_CALL
+        elif "agent" in span_type_str:
+            step_type = StepType.ROUTING
+        else:
+            step_type = StepType.THOUGHT
+
+        input_payload = span.get("input")
+        if not isinstance(input_payload, dict):
+            input_payload = {"input": input_payload}
+
+        output_payload = span.get("output")
+        if not isinstance(output_payload, dict):
+            output_payload = {"output": output_payload}
+
+        # Tokens & Cost
+        prompt_tokens = span.get("input_token_count") or span.get("prompt_tokens") or 0
+        completion_tokens = (
+            span.get("output_token_count") or span.get("completion_tokens") or 0
+        )
+        total_tokens = (
+            span.get("total_token_count")
+            or span.get("total_tokens")
+            or (prompt_tokens + completion_tokens)
+        )
+        cost = span.get("cost") or span.get("estimated_cost_usd") or 0.0
+
+        tokens = TokenUsage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens,
+            estimated_cost_usd=cost,
+        )
+
+        latency_ms = span.get("latency") or span.get("executionTime") or 0.0
+        if latency_ms < 100.0 and latency_ms > 0:
+            latency_ms *= 1000.0
+
+        step = TraceStep(
+            step_id=span_id,
+            parent_id=parent_id,
+            step_index=len(steps_list),
+            step_type=step_type,
+            name=span.get("name") or span.get("displayName") or f"step_{span_type_str}",
+            input_payload=input_payload,
+            output_payload=output_payload,
+            status=StepStatus.SUCCESS,
+            error_message=span.get("error") or span.get("errorMessage"),
+            latency_ms=latency_ms,
+            tokens=tokens,
+            metadata=span.get("metadata") or {},
+        )
+
+        if step.error_message:
+            step.status = StepStatus.ERROR
+
+        steps_list.append(step)
+
+        # Recurse children
+        children = span.get("spans") or span.get("children") or span.get("nodes") or []
+        for child in children:
+            cls._parse_span(child, span_id, steps_list)

agentdiff/adapters/generic.py

@@ -0,0 +1,11 @@
+from typing import Any
+
+from agentdiff.adapters.base import BaseAdapter
+from agentdiff.models.trace import AgentTrace
+
+
+class GenericAdapter(BaseAdapter):
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> AgentTrace:
+        """Parses a dictionary matching the canonical schema into AgentTrace."""
+        return AgentTrace.model_validate(data)

agentdiff/adapters/langfuse.py

@@ -0,0 +1,145 @@
+from datetime import datetime
+from typing import Any
+
+from agentdiff.adapters.base import BaseAdapter
+from agentdiff.models.step import StepStatus, StepType, TokenUsage, TraceStep
+from agentdiff.models.trace import AgentTrace
+
+
+class LangfuseAdapter(BaseAdapter):
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> AgentTrace:
+        """Parses exported Langfuse trace JSON into a canonical AgentTrace."""
+        trace_id = data.get("id") or "langfuse_trace"
+        agent_name = data.get("name") or "langfuse_agent"
+
+        task_input = data.get("input")
+        if not isinstance(task_input, dict):
+            task_input = {"input": task_input}
+
+        final_output = data.get("output")
+        if not isinstance(final_output, dict):
+            final_output = {"output": final_output}
+
+        observations = data.get("observations") or []
+        steps: list[TraceStep] = []
+
+        for idx, obs in enumerate(observations):
+            obs_id = obs.get("id") or f"obs_{idx}"
+            parent_id = obs.get("parentObservationId")
+
+            # Map Langfuse type to StepType
+            obs_type = str(obs.get("type", "")).upper()
+            if obs_type == "GENERATION":
+                step_type = StepType.LLM_CALL
+            elif obs_type == "SPAN":
+                name_lower = str(obs.get("name", "")).lower()
+                if "tool" in name_lower or "call" in name_lower:
+                    step_type = StepType.TOOL_CALL
+                else:
+                    step_type = StepType.ROUTING
+            else:
+                step_type = StepType.THOUGHT
+
+            input_payload = obs.get("input")
+            if not isinstance(input_payload, dict):
+                input_payload = {"input": input_payload}
+
+            output_payload = obs.get("output")
+            if not isinstance(output_payload, dict):
+                output_payload = {"output": output_payload}
+
+            # Parse Usage
+            usage = obs.get("usage") or {}
+            prompt_tokens = usage.get("promptTokens") or usage.get("input_tokens") or 0
+            completion_tokens = (
+                usage.get("completionTokens") or usage.get("output_tokens") or 0
+            )
+            total_tokens = (
+                usage.get("totalTokens")
+                or usage.get("total_tokens")
+                or (prompt_tokens + completion_tokens)
+            )
+            cost = usage.get("cost") or obs.get("cost") or 0.0
+
+            tokens = TokenUsage(
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+                total_tokens=total_tokens,
+                estimated_cost_usd=cost,
+            )
+
+            # Latency
+            latency_ms = 0.0
+            start_str = obs.get("startTime")
+            end_str = obs.get("endTime")
+            if start_str and end_str:
+                try:
+                    t1 = datetime.fromisoformat(str(start_str).replace("Z", "+00:00"))
+                    t2 = datetime.fromisoformat(str(end_str).replace("Z", "+00:00"))
+                    latency_ms = (t2 - t1).total_seconds() * 1000.0
+                except Exception:
+                    pass
+            if latency_ms == 0.0:
+                latency_ms = obs.get("latency_ms") or (
+                    obs.get("duration", 0.0) * 1000.0
+                )
+
+            # Error Levels
+            level = str(obs.get("level", "")).upper()
+            status = StepStatus.SUCCESS
+            error_message = None
+            if "ERROR" in level or obs.get("statusMessage"):
+                status = StepStatus.ERROR
+                error_message = obs.get("statusMessage")
+
+            step = TraceStep(
+                step_id=obs_id,
+                parent_id=parent_id,
+                step_index=idx,
+                step_type=step_type,
+                name=obs.get("name") or f"obs_{obs_type.lower()}",
+                input_payload=input_payload,
+                output_payload=output_payload,
+                status=status,
+                error_message=error_message,
+                latency_ms=latency_ms,
+                tokens=tokens,
+                metadata=obs.get("metadata") or {},
+            )
+            steps.append(step)
+
+        steps.sort(key=lambda s: s.step_index)
+
+        # Total latency of trace (in seconds, convert to ms)
+        total_latency_ms = data.get("duration", 0.0) * 1000.0
+        if total_latency_ms == 0.0:
+            root_spans = [s for s in steps if not s.parent_id]
+            total_latency_ms = (
+                sum(s.latency_ms for s in root_spans)
+                if root_spans
+                else sum(s.latency_ms for s in steps)
+            )
+
+        prompt_tokens = sum(s.tokens.prompt_tokens for s in steps)
+        completion_tokens = sum(s.tokens.completion_tokens for s in steps)
+        total_tokens = sum(s.tokens.total_tokens for s in steps)
+        cost = sum(s.tokens.estimated_cost_usd for s in steps)
+
+        total_tokens_obj = TokenUsage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens,
+            estimated_cost_usd=cost,
+        )
+
+        return AgentTrace(
+            trace_id=trace_id,
+            agent_name=agent_name,
+            task_input=task_input,
+            final_output=final_output,
+            steps=steps,
+            total_latency_ms=total_latency_ms,
+            total_tokens=total_tokens_obj,
+            metadata=data.get("metadata") or {},
+        )

agentdiff/adapters/openinference.py

@@ -0,0 +1,211 @@
+import json
+from datetime import datetime
+from typing import Any
+
+from agentdiff.adapters.base import BaseAdapter
+from agentdiff.models.step import StepStatus, StepType, TokenUsage, TraceStep
+from agentdiff.models.trace import AgentTrace
+
+
+class OpenInferenceAdapter(BaseAdapter):
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> AgentTrace:
+        """Parses an OpenInference span or collection of spans into an AgentTrace."""
+        spans = []
+        if isinstance(data, dict):
+            spans = data.get("spans") or data.get("spans_list") or [data]
+        elif isinstance(data, list):
+            spans = data
+
+        if not spans:
+            raise ValueError("No spans found to parse")
+
+        steps: list[TraceStep] = []
+        for idx, span in enumerate(spans):
+            context = span.get("context") or {}
+            span_id = context.get("span_id") or span.get("span_id") or f"span_{idx}"
+            parent_id = span.get("parent_span_id") or span.get("parent_id")
+
+            attrs = span.get("attributes") or {}
+
+            # Map OpenInference span kind to StepType
+            span_kind = attrs.get("openinference.span.kind") or span.get("kind") or ""
+            span_kind_str = str(span_kind).upper()
+
+            if "TOOL" in span_kind_str or "RETRIEVER" in span_kind_str:
+                step_type = StepType.TOOL_CALL
+            elif "LLM" in span_kind_str:
+                step_type = StepType.LLM_CALL
+            elif "AGENT" in span_kind_str or "CHAIN" in span_kind_str:
+                step_type = StepType.ROUTING
+            else:
+                step_type = StepType.THOUGHT
+
+            # Parse input value
+            input_val = (
+                attrs.get("input.value")
+                or attrs.get("llm.input_messages")
+                or span.get("input")
+                or {}
+            )
+            input_payload = cls._to_payload_dict(input_val, "input")
+
+            # Parse output value
+            output_val = (
+                attrs.get("output.value")
+                or attrs.get("llm.output_messages")
+                or span.get("output")
+                or {}
+            )
+            output_payload = cls._to_payload_dict(output_val, "output")
+
+            # Tokens
+            prompt_tokens = (
+                attrs.get("llm.token_count.prompt") or attrs.get("prompt_tokens") or 0
+            )
+            completion_tokens = (
+                attrs.get("llm.token_count.completion")
+                or attrs.get("completion_tokens")
+                or 0
+            )
+            total_tokens = (
+                attrs.get("llm.token_count.total")
+                or attrs.get("total_tokens")
+                or (prompt_tokens + completion_tokens)
+            )
+            cost = attrs.get("llm.cost") or attrs.get("cost") or 0.0
+
+            tokens = TokenUsage(
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+                total_tokens=total_tokens,
+                estimated_cost_usd=cost,
+            )
+
+            # Compute Latency
+            latency_ms = 0.0
+            start_time = span.get("start_time")
+            end_time = span.get("end_time")
+            if start_time and end_time:
+                try:
+                    if isinstance(start_time, (int, float)) and isinstance(
+                        end_time, (int, float)
+                    ):
+                        latency_ms = (end_time - start_time) * 1000.0
+                    else:
+                        t1 = datetime.fromisoformat(
+                            str(start_time).replace("Z", "+00:00")
+                        )
+                        t2 = datetime.fromisoformat(
+                            str(end_time).replace("Z", "+00:00")
+                        )
+                        latency_ms = (t2 - t1).total_seconds() * 1000.0
+                except Exception:
+                    pass
+            if latency_ms == 0.0:
+                latency_ms = span.get("latency_ms") or 0.0
+
+            # Status and error messages
+            status_val = span.get("status") or {}
+            status_code = (
+                status_val.get("status_code") or span.get("status_code") or "OK"
+            )
+            status_code_str = str(status_code).upper()
+
+            status = StepStatus.SUCCESS
+            error_message = None
+            if "ERROR" in status_code_str or status_val.get("message"):
+                status = StepStatus.ERROR
+                error_message = status_val.get("message")
+
+            step = TraceStep(
+                step_id=span_id,
+                parent_id=parent_id,
+                step_index=idx,
+                step_type=step_type,
+                name=span.get("name")
+                or attrs.get("openinference.span.name")
+                or f"step_{idx}",
+                input_payload=input_payload,
+                output_payload=output_payload,
+                status=status,
+                error_message=error_message,
+                latency_ms=latency_ms,
+                tokens=tokens,
+                metadata=attrs,
+            )
+            steps.append(step)
+
+        steps.sort(key=lambda s: s.step_index)
+
+        # Retrieve trace metadata
+        trace_id = "openinference_trace"
+        if spans:
+            first_span = spans[0]
+            trace_id = (
+                first_span.get("context", {}).get("trace_id")
+                or first_span.get("trace_id")
+                or trace_id
+            )
+
+        # Task input / Final output from root span if available
+        task_input = {}
+        final_output = {}
+        agent_name = "openinference_agent"
+
+        root_spans = [s for s in steps if not s.parent_id]
+        if root_spans:
+            task_input = root_spans[0].input_payload
+            final_output = root_spans[0].output_payload
+            agent_name = root_spans[0].name
+        elif steps:
+            task_input = steps[0].input_payload
+            final_output = steps[-1].output_payload
+
+        total_latency_ms = (
+            sum(s.latency_ms for s in root_spans)
+            if root_spans
+            else sum(s.latency_ms for s in steps)
+        )
+
+        prompt_tokens = sum(s.tokens.prompt_tokens for s in steps)
+        completion_tokens = sum(s.tokens.completion_tokens for s in steps)
+        total_tokens = sum(s.tokens.total_tokens for s in steps)
+        cost = sum(s.tokens.estimated_cost_usd for s in steps)
+
+        total_tokens_obj = TokenUsage(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=total_tokens,
+            estimated_cost_usd=cost,
+        )
+
+        return AgentTrace(
+            trace_id=trace_id,
+            agent_name=agent_name,
+            task_input=task_input,
+            final_output=final_output,
+            steps=steps,
+            total_latency_ms=total_latency_ms,
+            total_tokens=total_tokens_obj,
+            metadata={},
+        )
+
+    @classmethod
+    def _to_payload_dict(cls, val: Any, key_name: str) -> dict[str, Any]:
+        """Ensures input/output values are returned as dictionary payloads."""
+        if not val:
+            return {}
+        if isinstance(val, dict):
+            return val
+        if isinstance(val, list):
+            return {f"{key_name}_list": val}
+        if isinstance(val, str):
+            try:
+                parsed = json.loads(val)
+                if isinstance(parsed, dict):
+                    return parsed
+                return {key_name: parsed}
+            except Exception:
+                return {key_name: val}
+        return {key_name: val}

agentdiff/cli.py

@@ -0,0 +1,120 @@
+import json
+import sys
+
+import typer
+
+from agentdiff.engine.comparator import compare
+from agentdiff.loader import load_trace
+from agentdiff.reporters.markdown import generate_markdown
+from agentdiff.reporters.terminal import print_report
+
+app = typer.Typer(
+    help="AgentDiff CLI - Compare multi-turn agent execution trajectories."
+)
+
+
+@app.command(name="diff")
+def diff(
+    baseline_path: str = typer.Argument(..., help="Path to baseline trace JSON file"),
+    candidate_path: str = typer.Argument(..., help="Path to candidate trace JSON file"),
+    adapter: str = typer.Option(
+        "auto",
+        help="Telemetry adapter: auto, generic, deepeval, openinference, langfuse",
+    ),
+    format: str = typer.Option(
+        "terminal", help="Output format: terminal, json, markdown"
+    ),
+    output_file: str | None = typer.Option(
+        None, help="Write output to specified file path"
+    ),
+    fail_on_regression: bool = typer.Option(
+        False, help="Return exit code 1 if regressions are detected"
+    ),
+    max_loops: int = typer.Option(
+        0, help="Maximum allowed loop count before regression"
+    ),
+    max_divergence: float = typer.Option(
+        0.3, help="Maximum allowed Trajectory Divergence Index (TDI) before regression"
+    ),
+    max_cost_delta: float = typer.Option(
+        10.0, help="Maximum allowed cost increase percentage before regression"
+    ),
+):
+    """Compares baseline and candidate agent trajectories."""
+    try:
+        # Load and parse traces
+        baseline = load_trace(baseline_path, adapter)
+        candidate = load_trace(candidate_path, adapter)
+    except (json.JSONDecodeError, ValueError, FileNotFoundError) as e:
+        typer.echo(f"Error loading or parsing trace: {e}", err=True)
+        sys.exit(2)
+    except Exception as e:
+        typer.echo(f"Ingestion error: {e}", err=True)
+        sys.exit(2)
+
+    try:
+        # Perform comparison
+        report = compare(baseline, candidate, detect_loops=True)
+
+        # Check regressions
+        loop_count = len(report.loops_detected)
+        diverged = report.trajectory_divergence_index > max_divergence
+        loop_failed = loop_count > max_loops
+        cost_failed = report.cost_delta_percentage > max_cost_delta
+
+        has_regression = diverged or loop_failed or cost_failed
+
+        if has_regression:
+            report.passed = False
+
+        # Format output
+        output_content = ""
+        if format.lower() == "terminal":
+            # For console printing we write directly, but we can capture it or format differently if output_file is active
+            if output_file:
+                # If writing terminal format to file, output the text summary representation
+                output_content = report.summary()
+            else:
+                print_report(report)
+        elif format.lower() == "json":
+            output_content = report.model_dump_json(indent=2)
+            if not output_file:
+                typer.echo(output_content)
+        elif format.lower() == "markdown":
+            output_content = generate_markdown(report)
+            if not output_file:
+                typer.echo(output_content)
+        else:
+            typer.echo(f"Unsupported format: {format}", err=True)
+            sys.exit(2)
+
+        # Write to file if requested
+        if output_file:
+            with open(output_file, "w", encoding="utf-8") as f:
+                f.write(output_content)
+
+        # Enforce exit code protocol
+        if fail_on_regression and has_regression:
+            sys.exit(1)
+
+        sys.exit(0)
+
+    except SystemExit:
+        raise
+    except Exception as e:
+        typer.echo(f"Internal comparison error: {e}", err=True)
+        sys.exit(3)
+
+
+def main():
+    try:
+        app()
+    except SystemExit as e:
+        sys.exit(e.code)
+    except Exception as e:
+        print(f"Unhandled error: {e}", file=sys.stderr)
+        sys.exit(3)
+
+
+if __name__ == "__main__":
+    main()

agentdiff/engine/__init__.py

@@ -0,0 +1,17 @@
+from agentdiff.engine.aligner import align_traces
+from agentdiff.engine.comparator import compare
+from agentdiff.engine.loop_detector import detect_all_loops
+from agentdiff.engine.metrics import (
+    calculate_delta_percentage,
+    calculate_tdi,
+    calculate_wei,
+)
+
+__all__ = [
+    "align_traces",
+    "calculate_delta_percentage",
+    "calculate_tdi",
+    "calculate_wei",
+    "compare",
+    "detect_all_loops",
+]