evaldeck 0.1.0__py3-none-any.whl

evaldeck/integrations/opentelemetry.py

@@ -0,0 +1,416 @@
+"""OpenTelemetry/OpenInference integration for Evaldeck.
+
+This module provides an OpenTelemetry SpanProcessor that captures traces from
+any OpenInference-instrumented framework (LangChain, CrewAI, LiteLLM, OpenAI SDK,
+Anthropic SDK, etc.) and converts them to Evaldeck's Trace format.
+
+Installation:
+    pip install opentelemetry-sdk openinference-semantic-conventions
+
+    # Then install instrumentors for your framework(s):
+    pip install openinference-instrumentation-langchain
+    pip install openinference-instrumentation-crewai
+    pip install openinference-instrumentation-litellm
+    pip install openinference-instrumentation-openai
+
+Usage:
+    from evaldeck.integrations import EvaldeckSpanProcessor, setup_otel_tracing
+    from openinference.instrumentation.langchain import LangChainInstrumentor
+
+    # Setup tracing
+    processor = setup_otel_tracing()
+    LangChainInstrumentor().instrument()
+
+    # Run your agent
+    result = agent.invoke({"input": "Book a flight to NYC"})
+
+    # Get the evaldeck trace and evaluate
+    trace = processor.get_latest_trace()
+    result = evaluator.evaluate(trace, test_case)
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any
+
+from evaldeck.trace import Step, StepStatus, StepType, TokenUsage, Trace, TraceStatus
+
+# OpenTelemetry imports
+try:
+    from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor
+    from opentelemetry.trace import StatusCode
+
+    OTEL_AVAILABLE = True
+except ImportError:
+    OTEL_AVAILABLE = False
+
+    if TYPE_CHECKING:
+        from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor
+        from opentelemetry.trace import StatusCode
+    else:
+        SpanProcessor = object
+        ReadableSpan = object
+
+
+# OpenInference span kinds
+SPAN_KIND_LLM = "LLM"
+SPAN_KIND_TOOL = "TOOL"
+SPAN_KIND_CHAIN = "CHAIN"
+SPAN_KIND_EMBEDDING = "EMBEDDING"
+SPAN_KIND_RETRIEVER = "RETRIEVER"
+SPAN_KIND_RERANKER = "RERANKER"
+SPAN_KIND_GUARDRAIL = "GUARDRAIL"
+SPAN_KIND_AGENT = "AGENT"
+
+
+class EvaldeckSpanProcessor(SpanProcessor):
+    """OpenTelemetry SpanProcessor that builds Evaldeck Traces from OpenInference spans.
+
+    This processor intercepts OpenTelemetry spans as they complete and converts them
+    to Evaldeck's Trace/Step format. It supports all OpenInference span kinds:
+    LLM, TOOL, CHAIN, EMBEDDING, RETRIEVER, RERANKER, GUARDRAIL, AGENT.
+
+    Example:
+        from evaldeck.integrations import EvaldeckSpanProcessor
+        from opentelemetry import trace
+        from opentelemetry.sdk.trace import TracerProvider
+
+        processor = EvaldeckSpanProcessor()
+        provider = TracerProvider()
+        provider.add_span_processor(processor)
+        trace.set_tracer_provider(provider)
+
+        # After running instrumented code:
+        evaldeck_trace = processor.get_latest_trace()
+    """
+
+    def __init__(self) -> None:
+        if not OTEL_AVAILABLE:
+            raise ImportError(
+                "OpenTelemetry is not installed. Install with: "
+                "pip install opentelemetry-sdk openinference-semantic-conventions"
+            )
+
+        self._traces: dict[str, Trace] = {}
+        self._trace_order: list[str] = []  # Track order for get_latest_trace
+
+    def on_start(self, span: ReadableSpan, parent_context: Any = None) -> None:
+        """Called when a span starts. We don't need to do anything here."""
+        pass
+
+    def on_end(self, span: ReadableSpan) -> None:
+        """Called when a span ends. Convert to Evaldeck format."""
+        attrs = dict(span.attributes or {})
+        span_kind = str(attrs.get("openinference.span.kind", "")).upper()
+
+        # Skip spans without OpenInference kind
+        if not span_kind:
+            return
+
+        trace_id = format(span.context.trace_id, "032x")
+
+        # Ensure trace exists
+        if trace_id not in self._traces:
+            self._traces[trace_id] = Trace(
+                id=trace_id,
+                input="",
+                framework="openinference",
+            )
+            self._trace_order.append(trace_id)
+
+        trace = self._traces[trace_id]
+
+        # CHAIN/AGENT spans with no parent become the root trace
+        if span_kind in (SPAN_KIND_CHAIN, SPAN_KIND_AGENT) and span.parent is None:
+            self._update_trace_from_root_span(trace, span, attrs)
+            return
+
+        # Convert other spans to Steps
+        step = self._span_to_step(span, span_kind, attrs)
+        if step:
+            trace.add_step(step)
+
+    def _update_trace_from_root_span(
+        self, trace: Trace, span: ReadableSpan, attrs: dict[str, Any]
+    ) -> None:
+        """Update trace metadata from the root CHAIN/AGENT span."""
+        trace.input = str(attrs.get("input.value", trace.input or ""))
+        trace.output = attrs.get("output.value")
+        trace.status = self._map_trace_status(span)
+        trace.started_at = self._ns_to_datetime(span.start_time)
+        trace.completed_at = self._ns_to_datetime(span.end_time)
+        trace.duration_ms = (span.end_time - span.start_time) / 1_000_000
+
+        # Extract agent/framework info
+        if "llm.system" in attrs:
+            trace.framework = str(attrs["llm.system"])
+
+        trace.metadata["otel_trace_id"] = format(span.context.trace_id, "032x")
+        trace.metadata["otel_root_span_id"] = format(span.context.span_id, "016x")
+
+    def _span_to_step(
+        self, span: ReadableSpan, kind: str, attrs: dict[str, Any]
+    ) -> Step | None:
+        """Convert an OpenTelemetry span to an Evaldeck Step."""
+
+        if kind == SPAN_KIND_LLM:
+            return self._convert_llm_span(span, attrs)
+
+        elif kind == SPAN_KIND_TOOL:
+            return self._convert_tool_span(span, attrs)
+
+        elif kind in (SPAN_KIND_EMBEDDING, SPAN_KIND_RETRIEVER, SPAN_KIND_RERANKER):
+            return self._convert_retrieval_span(span, kind, attrs)
+
+        elif kind == SPAN_KIND_GUARDRAIL:
+            return self._convert_guardrail_span(span, attrs)
+
+        elif kind in (SPAN_KIND_CHAIN, SPAN_KIND_AGENT):
+            # Nested chains/agents become reasoning steps
+            return self._convert_chain_span(span, attrs)
+
+        return None
+
+    def _convert_llm_span(self, span: ReadableSpan, attrs: dict[str, Any]) -> Step:
+        """Convert an LLM span to a Step."""
+        return Step(
+            type=StepType.LLM_CALL,
+            model=attrs.get("llm.model_name") or attrs.get("gen_ai.request.model"),
+            input=self._extract_messages(attrs, "input"),
+            output=self._extract_messages(attrs, "output"),
+            tokens=TokenUsage(
+                prompt_tokens=int(attrs.get("llm.token_count.prompt", 0)),
+                completion_tokens=int(attrs.get("llm.token_count.completion", 0)),
+                total_tokens=int(attrs.get("llm.token_count.total", 0)),
+            ),
+            status=self._map_step_status(span),
+            duration_ms=self._calc_duration_ms(span),
+            error=self._extract_error(span),
+            metadata={
+                "otel_span_id": format(span.context.span_id, "016x"),
+                "llm_provider": attrs.get("llm.provider") or attrs.get("llm.system"),
+            },
+        )
+
+    def _convert_tool_span(self, span: ReadableSpan, attrs: dict[str, Any]) -> Step:
+        """Convert a TOOL span to a Step."""
+        tool_name = (
+            attrs.get("tool.name")
+            or attrs.get("tool_call.function.name")
+            or "unknown_tool"
+        )
+
+        tool_args = self._parse_json(
+            attrs.get("tool.parameters")
+            or attrs.get("tool_call.function.arguments")
+            or attrs.get("input.value")
+        )
+
+        return Step(
+            type=StepType.TOOL_CALL,
+            tool_name=str(tool_name),
+            tool_args=tool_args if isinstance(tool_args, dict) else {"input": tool_args},
+            tool_result=attrs.get("output.value"),
+            status=self._map_step_status(span),
+            duration_ms=self._calc_duration_ms(span),
+            error=self._extract_error(span),
+            metadata={
+                "otel_span_id": format(span.context.span_id, "016x"),
+                "tool_id": attrs.get("tool.id") or attrs.get("tool_call.id"),
+            },
+        )
+
+    def _convert_retrieval_span(
+        self, span: ReadableSpan, kind: str, attrs: dict[str, Any]
+    ) -> Step:
+        """Convert EMBEDDING/RETRIEVER/RERANKER spans to tool call Steps."""
+        return Step(
+            type=StepType.TOOL_CALL,
+            tool_name=kind.lower(),  # "embedding", "retriever", "reranker"
+            tool_args={"input": attrs.get("input.value")},
+            tool_result=attrs.get("output.value"),
+            status=self._map_step_status(span),
+            duration_ms=self._calc_duration_ms(span),
+            error=self._extract_error(span),
+            metadata={
+                "otel_span_id": format(span.context.span_id, "016x"),
+                "span_kind": kind,
+            },
+        )
+
+    def _convert_guardrail_span(self, span: ReadableSpan, attrs: dict[str, Any]) -> Step:
+        """Convert GUARDRAIL spans to reasoning Steps."""
+        return Step(
+            type=StepType.REASONING,
+            reasoning_text=f"Guardrail check: {attrs.get('output.value', 'passed')}",
+            status=self._map_step_status(span),
+            duration_ms=self._calc_duration_ms(span),
+            error=self._extract_error(span),
+            metadata={
+                "otel_span_id": format(span.context.span_id, "016x"),
+                "guardrail_input": attrs.get("input.value"),
+            },
+        )
+
+    def _convert_chain_span(self, span: ReadableSpan, attrs: dict[str, Any]) -> Step:
+        """Convert nested CHAIN/AGENT spans to reasoning Steps."""
+        return Step(
+            type=StepType.REASONING,
+            reasoning_text=f"Chain: {span.name} - {attrs.get('output.value', '')}",
+            status=self._map_step_status(span),
+            duration_ms=self._calc_duration_ms(span),
+            metadata={
+                "otel_span_id": format(span.context.span_id, "016x"),
+                "chain_input": attrs.get("input.value"),
+            },
+        )
+
+    def _extract_messages(self, attrs: dict[str, Any], direction: str) -> str:
+        """Extract message content from OpenInference indexed attributes.
+
+        OpenInference uses indexed prefixes like:
+            llm.input_messages.0.message.content
+            llm.input_messages.1.message.content
+        """
+        messages = []
+        i = 0
+        while True:
+            content_key = f"llm.{direction}_messages.{i}.message.content"
+            if content_key in attrs:
+                content = attrs[content_key]
+                role_key = f"llm.{direction}_messages.{i}.message.role"
+                role = attrs.get(role_key, "")
+                if role:
+                    messages.append(f"[{role}]: {content}")
+                else:
+                    messages.append(str(content))
+                i += 1
+            else:
+                break
+
+        if messages:
+            return "\n".join(messages)
+
+        # Fallback to simple input/output value
+        return str(attrs.get(f"{direction}.value", ""))
+
+    def _parse_json(self, value: Any) -> Any:
+        """Parse JSON string if possible, return as-is otherwise."""
+        if value is None:
+            return {}
+        if isinstance(value, str):
+            try:
+                return json.loads(value)
+            except (json.JSONDecodeError, TypeError):
+                return value
+        return value
+
+    def _map_trace_status(self, span: ReadableSpan) -> TraceStatus:
+        """Map OTel span status to Evaldeck TraceStatus."""
+        if span.status.status_code == StatusCode.ERROR:
+            return TraceStatus.ERROR
+        return TraceStatus.SUCCESS
+
+    def _map_step_status(self, span: ReadableSpan) -> StepStatus:
+        """Map OTel span status to Evaldeck StepStatus."""
+        if span.status.status_code == StatusCode.ERROR:
+            return StepStatus.FAILURE
+        return StepStatus.SUCCESS
+
+    def _extract_error(self, span: ReadableSpan) -> str | None:
+        """Extract error message from span if present."""
+        if span.status.status_code == StatusCode.ERROR:
+            return span.status.description
+        return None
+
+    def _calc_duration_ms(self, span: ReadableSpan) -> float:
+        """Calculate span duration in milliseconds."""
+        return (span.end_time - span.start_time) / 1_000_000
+
+    def _ns_to_datetime(self, ns: int) -> datetime:
+        """Convert nanoseconds timestamp to datetime."""
+        return datetime.fromtimestamp(ns / 1_000_000_000, tz=timezone.utc)
+
+    # -------------------------------------------------------------------------
+    # Public API
+    # -------------------------------------------------------------------------
+
+    def get_trace(self, trace_id: str) -> Trace | None:
+        """Get a trace by its ID.
+
+        Args:
+            trace_id: The OpenTelemetry trace ID (32 hex chars)
+
+        Returns:
+            The Evaldeck Trace, or None if not found
+        """
+        return self._traces.get(trace_id)
+
+    def get_latest_trace(self) -> Trace | None:
+        """Get the most recently completed trace.
+
+        Returns:
+            The most recent Evaldeck Trace, or None if no traces captured
+        """
+        if self._trace_order:
+            return self._traces.get(self._trace_order[-1])
+        return None
+
+    def get_all_traces(self) -> list[Trace]:
+        """Get all captured traces in order.
+
+        Returns:
+            List of all Evaldeck Traces
+        """
+        return [self._traces[tid] for tid in self._trace_order if tid in self._traces]
+
+    def reset(self) -> None:
+        """Clear all captured traces."""
+        self._traces.clear()
+        self._trace_order.clear()
+
+    def shutdown(self) -> None:
+        """Shutdown the processor (required by SpanProcessor interface)."""
+        pass
+
+    def force_flush(self, timeout_millis: int = 30000) -> bool:
+        """Force flush (required by SpanProcessor interface)."""
+        return True
+
+
+def setup_tracing(processor: EvaldeckSpanProcessor | None = None) -> EvaldeckSpanProcessor:
+    """Setup OpenTelemetry tracing with the Evaldeck processor.
+
+    This is a convenience function that sets up the tracer provider
+    with the Evaldeck processor.
+
+    Args:
+        processor: Optional existing processor. If None, creates a new one.
+
+    Returns:
+        The EvaldeckSpanProcessor (for later trace retrieval)
+
+    Example:
+        from evaldeck.integrations import setup_otel_tracing
+        from openinference.instrumentation.langchain import LangChainInstrumentor
+
+        processor = setup_otel_tracing()
+        LangChainInstrumentor().instrument()
+
+        # Run agent...
+
+        trace = processor.get_latest_trace()
+    """
+    from opentelemetry import trace
+    from opentelemetry.sdk.trace import TracerProvider
+
+    if processor is None:
+        processor = EvaldeckSpanProcessor()
+
+    provider = TracerProvider()
+    provider.add_span_processor(processor)
+    trace.set_tracer_provider(provider)
+
+    return processor

evaldeck/metrics/__init__.py

@@ -0,0 +1,25 @@
+"""Metrics for measuring agent performance."""
+
+from evaldeck.metrics.base import BaseMetric
+from evaldeck.metrics.builtin import (
+    DurationMetric,
+    ErrorRateMetric,
+    LLMCallCountMetric,
+    StepCountMetric,
+    StepEfficiencyMetric,
+    TokenUsageMetric,
+    ToolCallCountMetric,
+    ToolDiversityMetric,
+)
+
+__all__ = [
+    "BaseMetric",
+    "StepCountMetric",
+    "TokenUsageMetric",
+    "ToolCallCountMetric",
+    "DurationMetric",
+    "ToolDiversityMetric",
+    "StepEfficiencyMetric",
+    "LLMCallCountMetric",
+    "ErrorRateMetric",
+]

evaldeck/metrics/base.py

@@ -0,0 +1,62 @@
+"""Base metric class."""
+
+from __future__ import annotations
+
+import asyncio
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+from evaldeck.results import MetricResult
+
+if TYPE_CHECKING:
+    from evaldeck.test_case import EvalCase
+    from evaldeck.trace import Trace
+
+
+class BaseMetric(ABC):
+    """Base class for all metrics.
+
+    Metrics calculate quantitative measurements from traces.
+    Unlike graders, metrics don't pass/fail - they just measure.
+
+    Supports both sync and async calculation. Override calculate_async()
+    for metrics that need to make async I/O calls (e.g., fetching external
+    benchmark data).
+    """
+
+    name: str = "base"
+    unit: str | None = None
+
+    @abstractmethod
+    def calculate(self, trace: Trace, test_case: EvalCase | None = None) -> MetricResult:
+        """Calculate the metric value (sync).
+
+        Args:
+            trace: The execution trace to measure.
+            test_case: Optional test case for context.
+
+        Returns:
+            MetricResult with the calculated value.
+        """
+        pass
+
+    async def calculate_async(
+        self, trace: Trace, test_case: EvalCase | None = None
+    ) -> MetricResult:
+        """Calculate the metric value (async).
+
+        Default implementation runs sync calculate() in a thread pool.
+        Override this method for true async behavior (e.g., async API calls
+        for external benchmarking services).
+
+        Args:
+            trace: The execution trace to measure.
+            test_case: Optional test case for context.
+
+        Returns:
+            MetricResult with the calculated value.
+        """
+        return await asyncio.to_thread(self.calculate, trace, test_case)
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}()"

evaldeck/metrics/builtin.py

@@ -0,0 +1,195 @@
+"""Built-in metrics for agent evaluation."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from evaldeck.metrics.base import BaseMetric
+from evaldeck.results import MetricResult
+
+if TYPE_CHECKING:
+    from evaldeck.test_case import EvalCase
+    from evaldeck.trace import Trace
+
+
+class StepCountMetric(BaseMetric):
+    """Count total number of steps in the trace."""
+
+    name = "step_count"
+    unit = "steps"
+
+    def calculate(self, trace: Trace, test_case: EvalCase | None = None) -> MetricResult:
+        return MetricResult(
+            metric_name=self.name,
+            value=float(trace.step_count),
+            unit=self.unit,
+        )
+
+
+class TokenUsageMetric(BaseMetric):
+    """Total token usage across all LLM calls."""
+
+    name = "token_usage"
+    unit = "tokens"
+
+    def calculate(self, trace: Trace, test_case: EvalCase | None = None) -> MetricResult:
+        return MetricResult(
+            metric_name=self.name,
+            value=float(trace.total_tokens),
+            unit=self.unit,
+            details={
+                "llm_calls": len(trace.llm_calls),
+            },
+        )
+
+
+class ToolCallCountMetric(BaseMetric):
+    """Count number of tool calls."""
+
+    name = "tool_call_count"
+    unit = "calls"
+
+    def calculate(self, trace: Trace, test_case: EvalCase | None = None) -> MetricResult:
+        return MetricResult(
+            metric_name=self.name,
+            value=float(len(trace.tool_calls)),
+            unit=self.unit,
+            details={
+                "tools": trace.tools_called,
+            },
+        )
+
+
+class DurationMetric(BaseMetric):
+    """Total execution duration."""
+
+    name = "duration"
+    unit = "ms"
+
+    def calculate(self, trace: Trace, test_case: EvalCase | None = None) -> MetricResult:
+        duration = trace.duration_ms or 0.0
+        return MetricResult(
+            metric_name=self.name,
+            value=duration,
+            unit=self.unit,
+        )
+
+
+class ToolDiversityMetric(BaseMetric):
+    """Measure diversity of tools used (unique tools / total calls)."""
+
+    name = "tool_diversity"
+    unit = "ratio"
+
+    def calculate(self, trace: Trace, test_case: EvalCase | None = None) -> MetricResult:
+        tool_calls = trace.tool_calls
+        if not tool_calls:
+            return MetricResult(
+                metric_name=self.name,
+                value=0.0,
+                unit=self.unit,
+            )
+
+        unique_tools = len(set(trace.tools_called))
+        total_calls = len(tool_calls)
+        diversity = unique_tools / total_calls
+
+        return MetricResult(
+            metric_name=self.name,
+            value=diversity,
+            unit=self.unit,
+            details={
+                "unique_tools": unique_tools,
+                "total_calls": total_calls,
+            },
+        )
+
+
+class StepEfficiencyMetric(BaseMetric):
+    """Measure step efficiency compared to expected max steps.
+
+    Returns 1.0 if within expected steps, <1.0 if exceeded.
+    """
+
+    name = "step_efficiency"
+    unit = "ratio"
+
+    def calculate(self, trace: Trace, test_case: EvalCase | None = None) -> MetricResult:
+        actual_steps = trace.step_count
+
+        # Get expected max from test case
+        max_steps = None
+        if test_case and test_case.expected.max_steps:
+            max_steps = test_case.expected.max_steps
+
+        if max_steps is None:
+            # No baseline, just return step count as negative efficiency
+            return MetricResult(
+                metric_name=self.name,
+                value=1.0,
+                unit=self.unit,
+                details={
+                    "actual_steps": actual_steps,
+                    "max_steps": None,
+                },
+            )
+
+        # Calculate efficiency (1.0 = at or under budget, <1.0 = over budget)
+        if actual_steps <= max_steps:
+            efficiency = 1.0
+        else:
+            efficiency = max_steps / actual_steps
+
+        return MetricResult(
+            metric_name=self.name,
+            value=efficiency,
+            unit=self.unit,
+            details={
+                "actual_steps": actual_steps,
+                "max_steps": max_steps,
+            },
+        )
+
+
+class LLMCallCountMetric(BaseMetric):
+    """Count number of LLM calls."""
+
+    name = "llm_call_count"
+    unit = "calls"
+
+    def calculate(self, trace: Trace, test_case: EvalCase | None = None) -> MetricResult:
+        return MetricResult(
+            metric_name=self.name,
+            value=float(len(trace.llm_calls)),
+            unit=self.unit,
+        )
+
+
+class ErrorRateMetric(BaseMetric):
+    """Calculate error rate across steps."""
+
+    name = "error_rate"
+    unit = "ratio"
+
+    def calculate(self, trace: Trace, test_case: EvalCase | None = None) -> MetricResult:
+        from evaldeck.trace import StepStatus
+
+        if not trace.steps:
+            return MetricResult(
+                metric_name=self.name,
+                value=0.0,
+                unit=self.unit,
+            )
+
+        error_count = sum(1 for s in trace.steps if s.status == StepStatus.FAILURE)
+        error_rate = error_count / len(trace.steps)
+
+        return MetricResult(
+            metric_name=self.name,
+            value=error_rate,
+            unit=self.unit,
+            details={
+                "error_count": error_count,
+                "total_steps": len(trace.steps),
+            },
+        )