agentevals-cli 0.5.2__py3-none-any.whl

agentevals/output.py

@@ -0,0 +1,204 @@
+"""Output formatting for evaluation results."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from .runner import MetricResult, RunResult
+
+
+def format_results(run_result: RunResult, fmt: str = "table") -> str:
+    if fmt == "json":
+        return _format_json(run_result)
+    elif fmt == "summary":
+        return _format_summary(run_result)
+    else:
+        return _format_table(run_result)
+
+
+def _format_table(run_result: RunResult) -> str:
+    try:
+        from tabulate import tabulate
+    except ImportError:
+        return _format_summary(run_result)
+
+    lines: list[str] = []
+
+    if run_result.errors:
+        lines.append("Errors:")
+        for err in run_result.errors:
+            lines.append(f"  - {err}")
+        lines.append("")
+
+    for trace_result in run_result.trace_results:
+        lines.append(f"Trace: {trace_result.trace_id}")
+        lines.append(f"Invocations: {trace_result.num_invocations}")
+
+        if trace_result.conversion_warnings:
+            for w in trace_result.conversion_warnings:
+                lines.append(f"  Warning: {w}")
+
+        rows = []
+        for mr in trace_result.metric_results:
+            status_icon = _status_icon(mr.eval_status)
+            score_str = f"{mr.score:.4f}" if mr.score is not None else "N/A"
+            error_str = mr.error or ""
+            per_inv = (
+                ", ".join(f"{s:.4f}" if s is not None else "N/A" for s in mr.per_invocation_scores)
+                if mr.per_invocation_scores
+                else ""
+            )
+            rows.append(
+                [
+                    status_icon,
+                    mr.metric_name,
+                    score_str,
+                    mr.eval_status,
+                    per_inv,
+                    error_str,
+                ]
+            )
+
+        if rows:
+            table = tabulate(
+                rows,
+                headers=["", "Metric", "Score", "Status", "Per-Invocation", "Error"],
+                tablefmt="simple",
+            )
+            lines.append(table)
+
+        for mr in trace_result.metric_results:
+            if mr.details and mr.eval_status == "FAILED":
+                lines.append(_format_metric_details(mr))
+                lines.append("")
+
+        if trace_result.performance_metrics:
+            perf = trace_result.performance_metrics
+            lines.append("\n  Performance Metrics:")
+
+            lat = perf["latency"]
+            lines.append(
+                f"    Overall Latency: p50={lat['overall']['p50']:.0f}ms, p95={lat['overall']['p95']:.0f}ms, p99={lat['overall']['p99']:.0f}ms"
+            )
+            lines.append(
+                f"    LLM Latency:     p50={lat['llm_calls']['p50']:.0f}ms, p95={lat['llm_calls']['p95']:.0f}ms, p99={lat['llm_calls']['p99']:.0f}ms"
+            )
+            lines.append(
+                f"    Tool Latency:    p50={lat['tool_executions']['p50']:.0f}ms, p95={lat['tool_executions']['p95']:.0f}ms, p99={lat['tool_executions']['p99']:.0f}ms"
+            )
+
+            tok = perf["tokens"]
+            lines.append(
+                f"    Tokens: {tok['total']} total ({tok['total_prompt']} prompt + {tok['total_output']} output)"
+            )
+            lines.append(
+                f"    Per LLM Call:    p50={tok['per_llm_call']['p50']:.0f}, p95={tok['per_llm_call']['p95']:.0f}, p99={tok['per_llm_call']['p99']:.0f}"
+            )
+
+        lines.append("")
+
+    if run_result.performance_metrics:
+        lines.append("Overall Performance:")
+        perf = run_result.performance_metrics
+        lines.append(
+            f"  Total Tokens: {perf['tokens']['total']} ({perf['tokens']['total_prompt']} prompt + {perf['tokens']['total_output']} output)"
+        )
+        lines.append(
+            f"  Avg per Trace: {perf['tokens']['avg_per_trace']['prompt']:.0f} prompt, {perf['tokens']['avg_per_trace']['output']:.0f} output"
+        )
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def _format_json(run_result: RunResult) -> str:
+    data: dict[str, Any] = {
+        "traces": [],
+        "errors": run_result.errors,
+    }
+
+    for tr in run_result.trace_results:
+        trace_data: dict[str, Any] = {
+            "trace_id": tr.trace_id,
+            "num_invocations": tr.num_invocations,
+            "conversion_warnings": tr.conversion_warnings,
+            "metrics": [],
+        }
+        for mr in tr.metric_results:
+            metric_data = {
+                "metric_name": mr.metric_name,
+                "score": mr.score,
+                "eval_status": mr.eval_status,
+                "per_invocation_scores": mr.per_invocation_scores,
+                "error": mr.error,
+            }
+            if mr.details:
+                metric_data["details"] = mr.details
+            trace_data["metrics"].append(metric_data)
+        if tr.performance_metrics:
+            trace_data["performance_metrics"] = tr.performance_metrics
+        data["traces"].append(trace_data)
+
+    if run_result.performance_metrics:
+        data["performance_metrics"] = run_result.performance_metrics
+
+    return json.dumps(data, indent=2)
+
+
+def _format_summary(run_result: RunResult) -> str:
+    lines: list[str] = []
+
+    if run_result.errors:
+        lines.append("Errors:")
+        for err in run_result.errors:
+            lines.append(f"  - {err}")
+        lines.append("")
+
+    for tr in run_result.trace_results:
+        lines.append(f"Trace {tr.trace_id} ({tr.num_invocations} invocations):")
+        for mr in tr.metric_results:
+            icon = _status_icon(mr.eval_status)
+            if mr.error:
+                lines.append(f"  {icon} {mr.metric_name}: ERROR - {mr.error}")
+            elif mr.score is not None:
+                lines.append(f"  {icon} {mr.metric_name}: {mr.score:.4f} ({mr.eval_status})")
+            else:
+                lines.append(f"  {icon} {mr.metric_name}: N/A ({mr.eval_status})")
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def _format_metric_details(mr: MetricResult) -> str:
+    """Format detailed comparison for metrics with details field."""
+    lines = []
+
+    if mr.metric_name == "tool_trajectory_avg_score" and mr.details:
+        comparisons = mr.details.get("comparisons", [])
+        for i, comp in enumerate(comparisons, 1):
+            if not comp.get("matched", True):
+                lines.append(f"  Invocation {i} trajectory mismatch:")
+                lines.append("    Expected:")
+                for tool in comp.get("expected", []):
+                    args_str = json.dumps(tool["args"]) if tool["args"] else "{}"
+                    lines.append(f"      - {tool['name']}({args_str})")
+                if not comp.get("expected"):
+                    lines.append("      (none)")
+
+                lines.append("    Actual:")
+                for tool in comp.get("actual", []):
+                    args_str = json.dumps(tool["args"]) if tool["args"] else "{}"
+                    lines.append(f"      - {tool['name']}({args_str})")
+                if not comp.get("actual"):
+                    lines.append("      (none)")
+
+    return "\n".join(lines)
+
+
+def _status_icon(status: str) -> str:
+    return {
+        "PASSED": "[PASS]",
+        "FAILED": "[FAIL]",
+        "NOT_EVALUATED": "[----]",
+    }.get(status, "[????]")

agentevals/runner.py

@@ -0,0 +1,310 @@
+"""Evaluation runner — orchestrates trace loading, conversion, and scoring."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_set import EvalSet
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic.alias_generators import to_camel
+
+from .builtin_metrics import evaluate_builtin_metric
+from .config import (
+    CustomEvaluatorDef,
+    EvalRunConfig,
+)
+from .converter import ConversionResult, convert_traces
+from .loader.base import TraceLoader
+from .loader.jaeger import JaegerJsonLoader
+from .loader.otlp import OtlpJsonLoader
+from .trace_metrics import extract_performance_metrics
+
+logger = logging.getLogger(__name__)
+
+ProgressCallback = Callable[[str], Awaitable[None]]
+TraceProgressCallback = Callable[["TraceResult"], Awaitable[None]]
+
+
+class MetricResult(BaseModel):
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+
+    metric_name: str
+    score: float | None = None
+    eval_status: str = "NOT_EVALUATED"
+    per_invocation_scores: list[float | None] = Field(default_factory=list)
+    error: str | None = None
+    details: dict[str, Any] | None = None
+
+
+class TraceResult(BaseModel):
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+
+    trace_id: str
+    num_invocations: int = 0
+    metric_results: list[MetricResult] = Field(default_factory=list)
+    conversion_warnings: list[str] = Field(default_factory=list)
+    performance_metrics: dict[str, Any] | None = None
+
+
+class RunResult(BaseModel):
+    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+
+    trace_results: list[TraceResult] = Field(default_factory=list)
+    errors: list[str] = Field(default_factory=list)
+    performance_metrics: dict[str, Any] | None = None
+
+
+def get_loader(format_name: str) -> TraceLoader:
+    loaders: dict[str, type[TraceLoader]] = {
+        "jaeger-json": JaegerJsonLoader,
+        "otlp-json": OtlpJsonLoader,
+    }
+    if format_name not in loaders:
+        raise ValueError(f"Unknown trace format '{format_name}'. Available: {list(loaders.keys())}")
+    return loaders[format_name]()
+
+
+def load_eval_set(path: str) -> EvalSet:
+    with open(path) as f:
+        data = json.load(f)
+    return EvalSet.model_validate(data)
+
+
+async def run_evaluation(
+    config: EvalRunConfig,
+    progress_callback: ProgressCallback | None = None,
+    trace_progress_callback: TraceProgressCallback | None = None,
+) -> RunResult:
+    result = RunResult()
+
+    loader = get_loader(config.trace_format)
+    all_traces = []
+    for trace_file in config.trace_files:
+        try:
+            traces = loader.load(trace_file)
+            all_traces.extend(traces)
+        except Exception as exc:
+            msg = f"Failed to load trace file '{trace_file}': {exc}"
+            logger.error(msg)
+            result.errors.append(msg)
+
+    if not all_traces:
+        result.errors.append("No traces loaded.")
+        return result
+
+    conversion_results = convert_traces(all_traces)
+
+    trace_map = {t.trace_id: t for t in all_traces}
+
+    perf_metrics_map: dict[str, dict[str, Any]] = {}
+    for trace in all_traces:
+        perf_metrics_map[trace.trace_id] = extract_performance_metrics(trace)
+
+    eval_set: EvalSet | None = None
+    if config.eval_set_file:
+        try:
+            eval_set = load_eval_set(config.eval_set_file)
+        except Exception as exc:
+            msg = f"Failed to load eval set '{config.eval_set_file}': {exc}"
+            logger.error(msg)
+            result.errors.append(msg)
+
+    total_traces = len(conversion_results)
+    if progress_callback:
+        await progress_callback(f"Evaluating {total_traces} trace{'s' if total_traces != 1 else ''}...")
+
+    trace_semaphore = asyncio.Semaphore(config.max_concurrent_traces)
+    eval_semaphore = asyncio.Semaphore(config.max_concurrent_evals)
+
+    async def _evaluate_trace_bounded(idx: int, conv_result: ConversionResult) -> TraceResult:
+        async with trace_semaphore:
+            if progress_callback:
+                trace_id_short = (
+                    conv_result.trace_id[:12] + "..." if len(conv_result.trace_id) > 12 else conv_result.trace_id
+                )
+                await progress_callback(f"Trace {idx + 1}/{total_traces}: {trace_id_short}")
+
+            trace = trace_map.get(conv_result.trace_id)
+
+            return await _evaluate_trace(
+                conv_result=conv_result,
+                metrics=config.metrics,
+                custom_evaluators=config.custom_evaluators,
+                eval_set=eval_set,
+                judge_model=config.judge_model,
+                threshold=config.threshold,
+                eval_semaphore=eval_semaphore,
+                progress_callback=progress_callback,
+                trace_progress_callback=trace_progress_callback,
+                trace=trace,
+                performance_metrics=perf_metrics_map.get(conv_result.trace_id),
+            )
+
+    trace_results = await asyncio.gather(
+        *[_evaluate_trace_bounded(idx, conv_result) for idx, conv_result in enumerate(conversion_results)],
+        return_exceptions=True,
+    )
+
+    for tr in trace_results:
+        if isinstance(tr, Exception):
+            logger.error("Unexpected error evaluating trace: %s", tr)
+            result.errors.append(str(tr))
+        else:
+            result.trace_results.append(tr)
+
+    if progress_callback:
+        await progress_callback("Evaluation complete")
+
+    if result.trace_results:
+        all_tokens = {"prompt": [], "output": [], "total": []}
+
+        for tr in result.trace_results:
+            if tr.performance_metrics:
+                perf = tr.performance_metrics
+                all_tokens["prompt"].append(perf["tokens"]["total_prompt"])
+                all_tokens["output"].append(perf["tokens"]["total_output"])
+                all_tokens["total"].append(perf["tokens"]["total"])
+
+        if all_tokens["total"]:
+            result.performance_metrics = {
+                "tokens": {
+                    "total_prompt": sum(all_tokens["prompt"]),
+                    "total_output": sum(all_tokens["output"]),
+                    "total": sum(all_tokens["total"]),
+                    "avg_per_trace": {
+                        "prompt": sum(all_tokens["prompt"]) / len(all_tokens["prompt"]),
+                        "output": sum(all_tokens["output"]) / len(all_tokens["output"]),
+                    },
+                },
+                "trace_count": len(result.trace_results),
+            }
+
+    return result
+
+
+async def _evaluate_trace(
+    conv_result: ConversionResult,
+    metrics: list[str],
+    custom_evaluators: list[CustomEvaluatorDef],
+    eval_set: EvalSet | None,
+    judge_model: str | None,
+    threshold: float | None,
+    eval_semaphore: asyncio.Semaphore,
+    progress_callback: ProgressCallback | None = None,
+    trace_progress_callback: TraceProgressCallback | None = None,
+    trace=None,
+    performance_metrics: dict[str, Any] | None = None,
+) -> TraceResult:
+    trace_result = TraceResult(
+        trace_id=conv_result.trace_id,
+        num_invocations=len(conv_result.invocations),
+        conversion_warnings=conv_result.warnings,
+    )
+
+    if performance_metrics:
+        trace_result.performance_metrics = performance_metrics
+
+    if not conv_result.invocations:
+        trace_result.metric_results.append(
+            MetricResult(
+                metric_name="(all)",
+                error="No invocations extracted from trace.",
+            )
+        )
+        return trace_result
+
+    actual_invocations = conv_result.invocations
+
+    expected_invocations: list[Invocation] | None = None
+    if eval_set:
+        expected_invocations = _find_expected_invocations(actual_invocations, eval_set)
+
+    async def _append_result(result: MetricResult) -> MetricResult:
+        trace_result.metric_results.append(result)
+        if trace_progress_callback:
+            await trace_progress_callback(trace_result)
+        return result
+
+    async def _eval_builtin_with_semaphore(metric_name: str) -> MetricResult:
+        async with eval_semaphore:
+            if progress_callback:
+                await progress_callback(f"Running {metric_name}...")
+            result = await evaluate_builtin_metric(
+                metric_name=metric_name,
+                actual_invocations=actual_invocations,
+                expected_invocations=expected_invocations,
+                judge_model=judge_model,
+                threshold=threshold,
+            )
+        return await _append_result(result)
+
+    async def _eval_custom_with_semaphore(evaluator_def: CustomEvaluatorDef) -> MetricResult:
+        async with eval_semaphore:
+            if progress_callback:
+                await progress_callback(f"Running {evaluator_def.name}...")
+            from .custom_evaluators import evaluate_custom_evaluator
+
+            result = await evaluate_custom_evaluator(
+                evaluator_def=evaluator_def,
+                actual_invocations=actual_invocations,
+                expected_invocations=expected_invocations,
+            )
+        return await _append_result(result)
+
+    tasks = [_eval_builtin_with_semaphore(m) for m in metrics]
+    tasks.extend(_eval_custom_with_semaphore(g) for g in custom_evaluators)
+
+    await asyncio.gather(*tasks)
+
+    return trace_result
+
+
+def _find_expected_invocations(
+    actual_invocations: list[Invocation],
+    eval_set: EvalSet,
+) -> list[Invocation] | None:
+    """Match actual invocations to an eval case. Uses the sole eval case if
+    there's only one, otherwise matches by user content text."""
+    if not eval_set.eval_cases:
+        return None
+
+    if len(eval_set.eval_cases) == 1:
+        case = eval_set.eval_cases[0]
+        if case.conversation:
+            return case.conversation
+        return None
+
+    actual_user_text = _get_user_text(actual_invocations[0]) if actual_invocations else None
+    if not actual_user_text:
+        case = eval_set.eval_cases[0]
+        return case.conversation if case.conversation else None
+
+    for case in eval_set.eval_cases:
+        if not case.conversation:
+            continue
+        expected_user_text = _get_user_text(case.conversation[0])
+        if expected_user_text and _text_matches(actual_user_text, expected_user_text):
+            return case.conversation
+
+    logger.warning(
+        "No matching eval case found for user text: '%s'. Using first eval case.",
+        actual_user_text[:100],
+    )
+    case = eval_set.eval_cases[0]
+    return case.conversation if case.conversation else None
+
+
+def _get_user_text(invocation: Invocation) -> str | None:
+    if not invocation.user_content or not invocation.user_content.parts:
+        return None
+    texts = [p.text for p in invocation.user_content.parts if p.text]
+    return " ".join(texts) if texts else None
+
+
+def _text_matches(a: str, b: str) -> bool:
+    return a.strip().lower() == b.strip().lower()