agent-trajectory-diff 0.1.0__py3-none-any.whl

agentdiff/engine/aligner.py

@@ -0,0 +1,146 @@
+from typing import Any
+
+import networkx as nx
+
+from agentdiff.models.report import StepDiff, StepDiffStatus
+from agentdiff.models.step import TraceStep
+from agentdiff.models.trace import AgentTrace
+
+
+def step_signature(step: TraceStep) -> tuple:
+    """Computes a unique equivalence signature for a TraceStep.
+    Signature(N) = (step_type, name, tuple(sorted(input_payload.keys())))
+    """
+    keys = tuple(sorted(step.input_payload.keys())) if step.input_payload else ()
+    return (step.step_type, step.name, keys)
+
+
+def dict_diff(dict_a: dict[str, Any], dict_b: dict[str, Any]) -> dict[str, Any] | None:
+    """Calculates the difference between two dictionaries."""
+    diff = {}
+    all_keys = set(dict_a.keys()).union(dict_b.keys())
+
+    for k in all_keys:
+        if k not in dict_a:
+            diff[k] = {"status": "added", "new_value": dict_b[k]}
+        elif k not in dict_b:
+            diff[k] = {"status": "removed", "old_value": dict_a[k]}
+        elif dict_a[k] != dict_b[k]:
+            diff[k] = {
+                "status": "changed",
+                "old_value": dict_a[k],
+                "new_value": dict_b[k],
+            }
+
+    return diff if diff else None
+
+
+def align_traces(
+    baseline: AgentTrace, candidate: AgentTrace, strict_tool_signatures: bool = False
+) -> list[StepDiff]:
+    """Aligns two AgentTrace runs using modified topological LCS alignment."""
+    # Convert to networkx to get topological ordering
+    g_a = baseline.to_networkx()
+    g_b = candidate.to_networkx()
+
+    # Get topological sort of baseline
+    try:
+        topo_a = list(nx.topological_sort(g_a))
+        seq_a = [g_a.nodes[node_id]["step"] for node_id in topo_a]
+    except nx.NetworkXUnfeasible:
+        # Fall back to step_index sorting if cycles are present
+        seq_a = sorted(baseline.steps, key=lambda s: s.step_index)
+
+    # Get topological sort of candidate
+    try:
+        topo_b = list(nx.topological_sort(g_b))
+        seq_b = [g_b.nodes[node_id]["step"] for node_id in topo_b]
+    except nx.NetworkXUnfeasible:
+        # Fall back to step_index sorting
+        seq_b = sorted(candidate.steps, key=lambda s: s.step_index)
+
+    m, n = len(seq_a), len(seq_b)
+
+    # DP table for LCS length
+    dp = [[0] * (n + 1) for _ in range(m + 1)]
+
+    def is_equivalent(sa: TraceStep, sb: TraceStep) -> bool:
+        if strict_tool_signatures:
+            return (
+                step_signature(sa) == step_signature(sb)
+                and sa.input_payload == sb.input_payload
+            )
+        return step_signature(sa) == step_signature(sb)
+
+    # Compute DP table
+    for i in range(1, m + 1):
+        for j in range(1, n + 1):
+            if is_equivalent(seq_a[i - 1], seq_b[j - 1]):
+                dp[i][j] = dp[i - 1][j - 1] + 1
+            else:
+                dp[i][j] = max(dp[i - 1][j], dp[i][j - 1])
+
+    # Backtrack alignment
+    step_diffs = []
+    i, j = m, n
+
+    while i > 0 or j > 0:
+        if i > 0 and j > 0 and is_equivalent(seq_a[i - 1], seq_b[j - 1]):
+            sa = seq_a[i - 1]
+            sb = seq_b[j - 1]
+
+            arg_diff = dict_diff(sa.input_payload or {}, sb.input_payload or {})
+            out_diff = dict_diff(sa.output_payload or {}, sb.output_payload or {})
+
+            has_diff = (
+                arg_diff is not None
+                or out_diff is not None
+                or sa.status != sb.status
+                or sa.error_message != sb.error_message
+            )
+
+            diff_status = (
+                StepDiffStatus.MODIFIED if has_diff else StepDiffStatus.MATCHED
+            )
+
+            step_diffs.append(
+                StepDiff(
+                    step_name=sa.name,
+                    diff_status=diff_status,
+                    baseline_step=sa,
+                    candidate_step=sb,
+                    argument_diff=arg_diff,
+                    output_diff=out_diff,
+                )
+            )
+            i -= 1
+            j -= 1
+        elif j > 0 and (i == 0 or dp[i][j - 1] >= dp[i - 1][j]):
+            sb = seq_b[j - 1]
+            step_diffs.append(
+                StepDiff(
+                    step_name=sb.name,
+                    diff_status=StepDiffStatus.ADDED,
+                    baseline_step=None,
+                    candidate_step=sb,
+                    argument_diff=None,
+                    output_diff=None,
+                )
+            )
+            j -= 1
+        else:
+            sa = seq_a[i - 1]
+            step_diffs.append(
+                StepDiff(
+                    step_name=sa.name,
+                    diff_status=StepDiffStatus.REMOVED,
+                    baseline_step=sa,
+                    candidate_step=None,
+                    argument_diff=None,
+                    output_diff=None,
+                )
+            )
+            i -= 1
+
+    step_diffs.reverse()
+    return step_diffs

agentdiff/engine/comparator.py

@@ -0,0 +1,68 @@
+from typing import Any
+
+from agentdiff.engine.aligner import align_traces
+from agentdiff.engine.loop_detector import detect_all_loops
+from agentdiff.engine.metrics import (
+    calculate_delta_percentage,
+    calculate_tdi,
+    calculate_wei,
+)
+from agentdiff.models.report import DiffReport, StepDiffStatus
+from agentdiff.models.trace import AgentTrace
+
+
+def compare(
+    baseline: AgentTrace,
+    candidate: AgentTrace,
+    detect_loops: bool = True,
+    strict_tool_signatures: bool = False,
+) -> DiffReport:
+    """Compares baseline and candidate AgentTrace runs, returning a DiffReport."""
+    # 1. Align execution traces
+    step_diffs = align_traces(baseline, candidate, strict_tool_signatures)
+
+    # 2. Compute LCS length for TDI
+    lcs_len = sum(
+        1
+        for sd in step_diffs
+        if sd.diff_status in (StepDiffStatus.MATCHED, StepDiffStatus.MODIFIED)
+    )
+
+    # 3. Calculate metrics
+    tdi = calculate_tdi(len(baseline.steps), len(candidate.steps), lcs_len)
+    baseline_wei = calculate_wei(baseline.steps)
+    candidate_wei = calculate_wei(candidate.steps)
+
+    # 4. Calculate Resource Deltas
+    cost_delta = calculate_delta_percentage(
+        baseline.total_tokens.estimated_cost_usd,
+        candidate.total_tokens.estimated_cost_usd,
+    )
+    latency_delta = calculate_delta_percentage(
+        baseline.total_latency_ms, candidate.total_latency_ms
+    )
+    token_delta = calculate_delta_percentage(
+        baseline.total_tokens.total_tokens, candidate.total_tokens.total_tokens
+    )
+
+    # 5. Detect loops in the candidate run
+    loops: list[dict[str, Any]] = []
+    if detect_loops:
+        loops = detect_all_loops(candidate)
+
+    # 6. Build the DiffReport
+    report = DiffReport(
+        baseline_id=baseline.trace_id,
+        candidate_id=candidate.trace_id,
+        trajectory_divergence_index=tdi,
+        baseline_wei=baseline_wei,
+        candidate_wei=candidate_wei,
+        loops_detected=loops,
+        cost_delta_percentage=cost_delta,
+        latency_delta_percentage=latency_delta,
+        token_delta_percentage=token_delta,
+        step_diffs=step_diffs,
+        passed=True,  # Default to True, assertions/plugins override this
+    )
+
+    return report

agentdiff/engine/loop_detector.py

@@ -0,0 +1,110 @@
+from typing import Any
+
+import networkx as nx
+
+from agentdiff.models.trace import AgentTrace
+
+
+def detect_graph_cycles(trace: AgentTrace) -> list[list[str]]:
+    """Detects cycles in the parent-id dependency graph of the trace."""
+    graph = trace.to_networkx()
+    try:
+        cycles = list(nx.simple_cycles(graph))
+        return cycles
+    except Exception:
+        return []
+
+
+def detect_sequence_loops(trace: AgentTrace) -> list[dict[str, Any]]:
+    """Detects consecutive repeating sub-sequences of steps (e.g., A -> B -> A -> B).
+
+    A loop is defined as a sequence of step names of length k repeating consecutively
+    2 or more times.
+    """
+    steps = sorted(trace.steps, key=lambda s: s.step_index)
+    names = [s.name for s in steps]
+    n = len(names)
+    loops = []
+
+    i = 0
+    while i < n:
+        found_loop = False
+        # Try different pattern lengths up to half the remaining sequence length
+        for k in range(1, (n - i) // 2 + 1):
+            pattern = names[i : i + k]
+
+            # Count consecutive repetitions of the pattern
+            count = 1
+            while i + (count + 1) * k <= n:
+                next_segment = names[i + count * k : i + (count + 1) * k]
+                if next_segment == pattern:
+                    count += 1
+                else:
+                    break
+
+            if count >= 2:
+                # Loop detected!
+                loop_step_ids = [s.step_id for s in steps[i : i + k]]
+
+                # Check for stagnant state (whether input payloads or outputs are unchanged)
+                # Let's compare first iteration payloads with subsequent ones
+                stagnant = True
+                for step_idx in range(k):
+                    base_step = steps[i + step_idx]
+                    for iter_idx in range(1, count):
+                        compare_step = steps[i + iter_idx * k + step_idx]
+                        if (
+                            base_step.input_payload != compare_step.input_payload
+                            or base_step.output_payload != compare_step.output_payload
+                        ):
+                            stagnant = False
+                            break
+                    if not stagnant:
+                        break
+
+                loops.append(
+                    {
+                        "steps": pattern,
+                        "step_ids": loop_step_ids,
+                        "iterations": count,
+                        "start_index": i,
+                        "length": k,
+                        "stagnant": stagnant,
+                    }
+                )
+
+                # Advance pointer past the repeated patterns
+                i += count * k
+                found_loop = True
+                break
+
+        if not found_loop:
+            i += 1
+
+    return loops
+
+
+def detect_all_loops(trace: AgentTrace) -> list[dict[str, Any]]:
+    """Runs all loop detection algorithms on the trace and returns detected loops."""
+    loops = detect_sequence_loops(trace)
+
+    # Add graph cycles if any
+    cycles = detect_graph_cycles(trace)
+    for cycle in cycles:
+        loops.append(
+            {
+                "steps": [
+                    trace.steps[idx].name
+                    for idx in range(len(trace.steps))
+                    if trace.steps[idx].step_id in cycle
+                ],
+                "step_ids": cycle,
+                "iterations": 2,  # cycle implies a recurring path
+                "start_index": -1,
+                "length": len(cycle),
+                "stagnant": True,
+                "type": "graph_cycle",
+            }
+        )
+
+    return loops

agentdiff/engine/metrics.py

@@ -0,0 +1,31 @@
+from agentdiff.models.step import StepStatus, TraceStep
+
+
+def calculate_tdi(steps_a_len: int, steps_b_len: int, lcs_len: int) -> float:
+    """Calculates Trajectory Divergence Index (TDI).
+    TDI = 1.0 - (2 * |LCS(A, B)|) / (|Steps_A| + |Steps_B|)
+    """
+    total_steps = steps_a_len + steps_b_len
+    if total_steps == 0:
+        return 0.0
+    return 1.0 - (2.0 * lcs_len) / total_steps
+
+
+def calculate_wei(steps: list[TraceStep]) -> float:
+    """Calculates Wasted Effort Index (WEI).
+    WEI = Count(Steps with status in {ERROR, RETRY, ABANDONED}) / Total Steps
+    """
+    if not steps:
+        return 0.0
+    wasted_statuses = {StepStatus.ERROR, StepStatus.RETRY, StepStatus.ABANDONED}
+    wasted_count = sum(1 for s in steps if s.status in wasted_statuses)
+    return wasted_count / len(steps)
+
+
+def calculate_delta_percentage(baseline_val: float, candidate_val: float) -> float:
+    """Calculates percentage delta between candidate and baseline values."""
+    if baseline_val == 0.0:
+        if candidate_val == 0.0:
+            return 0.0
+        return 100.0  # Standard representation for positive spike from zero
+    return ((candidate_val - baseline_val) / baseline_val) * 100.0

agentdiff/loader.py

@@ -0,0 +1,65 @@
+import json
+from typing import Any
+
+from agentdiff.adapters import (
+    DeepEvalAdapter,
+    GenericAdapter,
+    LangfuseAdapter,
+    OpenInferenceAdapter,
+)
+from agentdiff.models.trace import AgentTrace
+
+
+def parse_trace_data(data: Any, adapter_name: str = "auto") -> AgentTrace:
+    """Parses a dictionary/list into an AgentTrace, auto-detecting the adapter if specified."""
+    name = adapter_name.lower().strip()
+
+    if name == "generic":
+        return GenericAdapter.from_dict(data)
+    elif name == "deepeval":
+        return DeepEvalAdapter.from_dict(data)
+    elif name in ("openinference", "open_inference"):
+        return OpenInferenceAdapter.from_dict(data)
+    elif name == "langfuse":
+        return LangfuseAdapter.from_dict(data)
+    elif name == "auto":
+        # Auto-detect format based on structure
+        if isinstance(data, list):
+            if not data:
+                raise ValueError("Cannot auto-detect from an empty list")
+            elem = data[0]
+            if isinstance(elem, dict):
+                if "context" in elem or "attributes" in elem:
+                    return OpenInferenceAdapter.from_dict(data)
+                if "type" in elem and "input" in elem:
+                    return DeepEvalAdapter.from_dict(data)
+            return GenericAdapter.from_dict(data)
+
+        elif isinstance(data, dict):
+            # Langfuse check
+            if "observations" in data:
+                return LangfuseAdapter.from_dict(data)
+
+            # OpenInference check
+            if "spans" in data and isinstance(data["spans"], list) and data["spans"]:
+                first_span = data["spans"][0]
+                if "context" in first_span or "attributes" in first_span:
+                    return OpenInferenceAdapter.from_dict(data)
+
+            # DeepEval check
+            if ("input" in data and "output" in data) and (
+                "spans" in data or "nodes" in data
+            ):
+                return DeepEvalAdapter.from_dict(data)
+
+            # Default fallback
+            return GenericAdapter.from_dict(data)
+
+        raise ValueError(f"Unsupported raw trace data type: {type(data)}")
+
+
+def load_trace(filepath: str, adapter_name: str = "auto") -> AgentTrace:
+    """Loads and parses a trace file from disk."""
+    with open(filepath, encoding="utf-8") as f:
+        data = json.load(f)
+    return parse_trace_data(data, adapter_name)

agentdiff/models/__init__.py

@@ -0,0 +1,14 @@
+from agentdiff.models.report import DiffReport, StepDiff, StepDiffStatus
+from agentdiff.models.step import StepStatus, StepType, TokenUsage, TraceStep
+from agentdiff.models.trace import AgentTrace
+
+__all__ = [
+    "AgentTrace",
+    "DiffReport",
+    "StepDiff",
+    "StepDiffStatus",
+    "StepStatus",
+    "StepType",
+    "TokenUsage",
+    "TraceStep",
+]

agentdiff/models/report.py

@@ -0,0 +1,74 @@
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+from agentdiff.models.step import TraceStep
+
+
+class StepDiffStatus(str, Enum):
+    MATCHED = "matched"
+    ADDED = "added"
+    REMOVED = "removed"
+    MODIFIED = "modified"
+
+
+class StepDiff(BaseModel):
+    step_name: str
+    diff_status: StepDiffStatus
+    baseline_step: TraceStep | None = None
+    candidate_step: TraceStep | None = None
+    argument_diff: dict[str, Any] | None = None
+    output_diff: dict[str, Any] | None = None
+
+
+class DiffReport(BaseModel):
+    baseline_id: str
+    candidate_id: str
+    trajectory_divergence_index: float
+    baseline_wei: float
+    candidate_wei: float
+    loops_detected: list[dict[str, Any]] = Field(default_factory=list)
+    cost_delta_percentage: float
+    latency_delta_percentage: float
+    token_delta_percentage: float
+    step_diffs: list[StepDiff] = Field(default_factory=list)
+    passed: bool = True
+
+    def summary(self) -> str:
+        """Returns a string summarizing the comparison report."""
+        lines = [
+            "=========================================",
+            "           AGENTDIFF REPORT SUMMARY      ",
+            "=========================================",
+            f"Baseline ID:  {self.baseline_id}",
+            f"Candidate ID: {self.candidate_id}",
+            f"Status:       {'PASSED' if self.passed else 'FAILED'}",
+            "-----------------------------------------",
+            f"Trajectory Divergence Index (TDI): {self.trajectory_divergence_index:.4f}",
+            f"Baseline Wasted Effort Index (WEI): {self.baseline_wei:.4f}",
+            f"Candidate Wasted Effort Index (WEI): {self.candidate_wei:.4f}",
+            "-----------------------------------------",
+            "Resource Deltas:",
+            f"  Cost Delta:    {self.cost_delta_percentage:+.2f}%",
+            f"  Latency Delta: {self.latency_delta_percentage:+.2f}%",
+            f"  Token Delta:   {self.token_delta_percentage:+.2f}%",
+            "-----------------------------------------",
+            f"Loops Detected: {len(self.loops_detected)}",
+        ]
+
+        for idx, loop in enumerate(self.loops_detected):
+            lines.append(
+                f"  - Loop #{idx + 1}: Repeated steps {loop.get('steps', [])} (Count: {loop.get('iterations', 0)})"
+            )
+
+        lines.append("Step Diff Summary:")
+        counts = {status: 0 for status in StepDiffStatus}
+        for sd in self.step_diffs:
+            counts[sd.diff_status] += 1
+
+        for status, count in counts.items():
+            lines.append(f"  - {status.value.capitalize()}: {count}")
+
+        lines.append("=========================================")
+        return "\n".join(lines)

agentdiff/models/step.py

@@ -0,0 +1,40 @@
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class StepType(str, Enum):
+    TOOL_CALL = "tool_call"
+    LLM_CALL = "llm_call"
+    ROUTING = "routing"
+    THOUGHT = "thought"
+
+
+class StepStatus(str, Enum):
+    SUCCESS = "success"
+    ERROR = "error"
+    RETRY = "retry"
+    ABANDONED = "abandoned"
+
+
+class TokenUsage(BaseModel):
+    prompt_tokens: int = 0
+    completion_tokens: int = 0
+    total_tokens: int = 0
+    estimated_cost_usd: float = 0.0
+
+
+class TraceStep(BaseModel):
+    step_id: str
+    parent_id: str | None = None
+    step_index: int
+    step_type: StepType
+    name: str  # e.g. "sql_executor", "web_search", "synthesize"
+    input_payload: dict[str, Any] = Field(default_factory=dict)
+    output_payload: dict[str, Any] | None = None
+    status: StepStatus = StepStatus.SUCCESS
+    error_message: str | None = None
+    latency_ms: float = 0.0
+    tokens: TokenUsage = Field(default_factory=TokenUsage)
+    metadata: dict[str, Any] = Field(default_factory=dict)

agentdiff/models/trace.py

@@ -0,0 +1,33 @@
+from typing import Any
+
+import networkx as nx
+from pydantic import BaseModel, Field
+
+from agentdiff.models.step import TokenUsage, TraceStep
+
+
+class AgentTrace(BaseModel):
+    trace_id: str
+    agent_name: str
+    agent_version: str | None = None
+    task_input: dict[str, Any]
+    final_output: dict[str, Any] | None = None
+    steps: list[TraceStep] = Field(default_factory=list)
+    total_latency_ms: float = 0.0
+    total_tokens: TokenUsage = Field(default_factory=TokenUsage)
+    metadata: dict[str, Any] = Field(default_factory=dict)
+
+    def to_networkx(self) -> nx.DiGraph:
+        """Converts step sequence/parent_ids into a networkx.DiGraph."""
+        graph = nx.DiGraph()
+
+        # Add all nodes first
+        for step in self.steps:
+            graph.add_node(step.step_id, step=step)
+
+        # Add edges based on parent_id relationships
+        for step in self.steps:
+            if step.parent_id and step.parent_id in graph:
+                graph.add_edge(step.parent_id, step.step_id)
+
+        return graph

agentdiff/reporters/__init__.py

@@ -0,0 +1,7 @@
+from agentdiff.reporters.markdown import generate_markdown
+from agentdiff.reporters.terminal import print_report
+
+__all__ = [
+    "generate_markdown",
+    "print_report",
+]

agentdiff/reporters/markdown.py

@@ -0,0 +1,78 @@
+from agentdiff.models.report import DiffReport, StepDiffStatus
+from agentdiff.models.step import StepStatus
+
+
+def generate_markdown(report: DiffReport) -> str:
+    """Generates a markdown formatted summary of the comparison report."""
+    status_emoji = "✅ PASSED" if report.passed else "❌ FAILED"
+
+    lines = [
+        f"# AgentDiff Comparison Report: {status_emoji}",
+        "",
+        "## Summary Metrics",
+        "",
+        "| Metric | Baseline | Candidate | Delta |",
+        "| :--- | :--- | :--- | :--- |",
+        f"| **Trajectory Divergence (TDI)** | - | - | `{report.trajectory_divergence_index:.4f}` |",
+        f"| **Wasted Effort Index (WEI)** | `{report.baseline_wei:.4f}` | `{report.candidate_wei:.4f}` | - |",
+        f"| **Total Latency** | - | - | `{report.latency_delta_percentage:+.2f}%` |",
+        f"| **Total Tokens** | - | - | `{report.token_delta_percentage:+.2f}%` |",
+        f"| **Estimated Cost** | - | - | `{report.cost_delta_percentage:+.2f}%` |",
+        "",
+    ]
+
+    if report.loops_detected:
+        lines.extend(
+            [
+                "## ⚠️ Warnings: Loops Detected",
+                "",
+                "The candidate trace contains repeating step sequences:",
+                "",
+            ]
+        )
+        for idx, loop in enumerate(report.loops_detected):
+            stagnant_str = " (Stagnant state changes)" if loop.get("stagnant") else ""
+            lines.append(
+                f"- **Loop #{idx + 1}:** Repeated {loop['steps']} `{loop['iterations']}` times{stagnant_str}"
+            )
+        lines.append("")
+
+    lines.extend(
+        [
+            "## Step-by-Step Trajectory Diff",
+            "",
+            "| # | Step Name | Status | Baseline (Latency / Tokens / Cost) | Candidate (Latency / Tokens / Cost) |",
+            "| :--- | :--- | :--- | :--- | :--- |",
+        ]
+    )
+
+    for idx, sd in enumerate(report.step_diffs):
+        status_emoji_map = {
+            StepDiffStatus.MATCHED: "🟢 MATCHED",
+            StepDiffStatus.ADDED: "🔵 ADDED",
+            StepDiffStatus.REMOVED: "🔴 REMOVED",
+            StepDiffStatus.MODIFIED: "🟡 MODIFIED",
+        }
+        status_str = status_emoji_map.get(sd.diff_status, sd.diff_status.value.upper())
+
+        base_info = "-"
+        if sd.baseline_step:
+            s = sd.baseline_step
+            err_str = (
+                f" ({s.status.value.upper()})" if s.status != StepStatus.SUCCESS else ""
+            )
+            base_info = f"{s.latency_ms:.0f}ms / {s.tokens.total_tokens}t / ${s.tokens.estimated_cost_usd:.4f}{err_str}"
+
+        cand_info = "-"
+        if sd.candidate_step:
+            s = sd.candidate_step
+            err_str = (
+                f" ({s.status.value.upper()})" if s.status != StepStatus.SUCCESS else ""
+            )
+            cand_info = f"{s.latency_ms:.0f}ms / {s.tokens.total_tokens}t / ${s.tokens.estimated_cost_usd:.4f}{err_str}"
+
+        lines.append(
+            f"| {idx + 1} | {sd.step_name} | {status_str} | {base_info} | {cand_info} |"
+        )
+
+    return "\n".join(lines)