agentevals-cli 0.5.2__py3-none-any.whl

agentevals/api/streaming_routes.py

@@ -0,0 +1,373 @@
+"""API routes for streaming session management."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import FileResponse
+from pydantic import BaseModel
+
+from ..config import EvalRunConfig
+from ..converter import convert_traces
+from ..loader.otlp import OtlpJsonLoader
+from ..runner import run_evaluation
+from ..trace_attrs import OTEL_GENAI_INPUT_MESSAGES, OTEL_GENAI_REQUEST_MODEL
+from ..utils.log_enrichment import enrich_spans_with_logs
+from .models import (
+    CreateEvalSetData,
+    EvaluateSessionsData,
+    GetTraceData,
+    PrepareEvaluationData,
+    SessionEvalResult,
+    SessionInfo,
+    StandardResponse,
+)
+
+logger = logging.getLogger(__name__)
+
+streaming_router = APIRouter()
+
+trace_manager = None
+
+
+def set_trace_manager(manager):
+    """Set the trace manager instance."""
+    global trace_manager
+    trace_manager = manager
+
+
+class CreateEvalSetRequest(BaseModel):
+    session_id: str
+    eval_set_id: str
+
+
+class EvaluateSessionsRequest(BaseModel):
+    golden_session_id: str
+    eval_set_id: str
+    metrics: list[str] = ["tool_trajectory_avg_score"]
+    judge_model: str = "gemini-2.5-flash"
+
+
+class PrepareEvaluationRequest(BaseModel):
+    golden_session_id: str
+    session_ids: list[str]
+
+
+class GetTraceRequest(BaseModel):
+    session_id: str
+
+
+@streaming_router.get("/sessions", response_model=StandardResponse[list[SessionInfo]])
+async def list_sessions():
+    sessions_data = []
+
+    for session_id, session in trace_manager.sessions.items():
+        info = SessionInfo(
+            session_id=session_id,
+            trace_id=session.trace_id,
+            eval_set_id=session.eval_set_id,
+            span_count=len(session.spans),
+            is_complete=session.is_complete,
+            started_at=session.started_at.isoformat(),
+            metadata=session.metadata,
+            invocations=session.invocations if session.is_complete and session.invocations else None,
+        )
+        sessions_data.append(info)
+
+    return StandardResponse(data=sessions_data)
+
+
+@streaming_router.post("/create-eval-set", response_model=StandardResponse[CreateEvalSetData])
+async def create_eval_set_from_session(request: CreateEvalSetRequest):
+    """Convert a session's trace into an EvalSet."""
+    session = trace_manager.sessions.get(request.session_id)
+    if not session:
+        raise HTTPException(status_code=404, detail="Session not found")
+
+    try:
+        trace_file = await trace_manager._save_spans_to_temp_file(session)
+        logger.debug(
+            "Session %s: %d spans, %d logs saved to %s",
+            request.session_id,
+            len(session.spans),
+            len(session.logs),
+            trace_file,
+        )
+        loader = OtlpJsonLoader()
+        traces = loader.load(str(trace_file))
+
+        if not traces:
+            raise HTTPException(
+                status_code=400,
+                detail=(
+                    f"No traces found in session (spans={len(session.spans)}, "
+                    f"logs={len(session.logs)}). If using the SDK with langchain/openai, "
+                    f"ensure opentelemetry-instrumentation-openai-v2 is installed."
+                ),
+            )
+
+        conversion_results = convert_traces(traces)
+        if not conversion_results:
+            raise HTTPException(status_code=400, detail="Failed to convert trace")
+
+        all_invocations = []
+        for conv_result in conversion_results:
+            all_invocations.extend(conv_result.invocations)
+
+        logger.debug(f"Creating eval set from {len(all_invocations)} invocations")
+        for i, inv in enumerate(all_invocations):
+            tool_count = len(inv.intermediate_data.tool_uses) if inv.intermediate_data else 0
+            logger.debug(f"  Invocation {i}: {tool_count} tool calls")
+
+        conversation = []
+        for inv in all_invocations:
+            inv_dict = {
+                "invocation_id": inv.invocation_id,
+                "user_content": inv.user_content.model_dump(exclude_none=True) if inv.user_content else None,
+            }
+            if inv.final_response:
+                inv_dict["final_response"] = inv.final_response.model_dump(exclude_none=True)
+            if inv.intermediate_data:
+                inv_dict["intermediate_data"] = inv.intermediate_data.model_dump(exclude_none=True)
+
+            conversation.append(inv_dict)
+
+        eval_set = {
+            "eval_set_id": request.eval_set_id,
+            "eval_cases": [
+                {
+                    "eval_id": "case_1",
+                    "conversation": conversation,
+                }
+            ],
+        }
+
+        return StandardResponse(
+            data=CreateEvalSetData(
+                eval_set=eval_set,
+                num_invocations=len(all_invocations),
+            )
+        )
+
+    except HTTPException:
+        raise
+    except Exception as exc:
+        logger.exception("Failed to create eval set")
+        raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+
+@streaming_router.post("/evaluate-sessions", response_model=StandardResponse[EvaluateSessionsData])
+async def evaluate_sessions(request: EvaluateSessionsRequest):
+    """Evaluate all sessions against a golden session converted to EvalSet."""
+    golden_session = trace_manager.sessions.get(request.golden_session_id)
+    if not golden_session:
+        raise HTTPException(status_code=404, detail="Golden session not found")
+
+    try:
+        eval_set_response = await create_eval_set_from_session(
+            CreateEvalSetRequest(
+                session_id=request.golden_session_id,
+                eval_set_id=request.eval_set_id,
+            )
+        )
+
+        import tempfile
+
+        eval_set_file = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False)
+        json.dump(eval_set_response.data.eval_set, eval_set_file)
+        eval_set_file.close()
+
+        sessions_to_evaluate = [
+            (session_id, session) for session_id, session in trace_manager.sessions.items() if session.is_complete
+        ]
+
+        logger.info(
+            "Evaluating %d complete sessions (of %d total)", len(sessions_to_evaluate), len(trace_manager.sessions)
+        )
+
+        sem = asyncio.Semaphore(5)
+
+        async def eval_one_session(session_id: str, session) -> SessionEvalResult:
+            async with sem:
+                try:
+                    trace_file = await trace_manager._save_spans_to_temp_file(session)
+
+                    config = EvalRunConfig(
+                        trace_files=[str(trace_file)],
+                        trace_format="otlp-json",
+                        eval_set_file=eval_set_file.name,
+                        metrics=request.metrics,
+                        judge_model=request.judge_model,
+                    )
+
+                    eval_result = await run_evaluation(config)
+
+                    if eval_result.trace_results:
+                        trace_result = eval_result.trace_results[0]
+                        return SessionEvalResult(
+                            session_id=session_id,
+                            trace_id=trace_result.trace_id,
+                            num_invocations=trace_result.num_invocations,
+                            metric_results=[
+                                {
+                                    "metricName": mr.metric_name,
+                                    "score": mr.score,
+                                    "evalStatus": mr.eval_status,
+                                    "error": mr.error,
+                                }
+                                for mr in trace_result.metric_results
+                            ],
+                        )
+                    else:
+                        logger.warning("No trace results for session %s", session_id)
+                        return SessionEvalResult(session_id=session_id, error="No trace results")
+
+                except Exception as exc:
+                    logger.error(f"Failed to evaluate session {session_id}: {exc}", exc_info=True)
+                    return SessionEvalResult(session_id=session_id, error=str(exc))
+
+        results = await asyncio.gather(*[eval_one_session(sid, sess) for sid, sess in sessions_to_evaluate])
+
+        logger.info("Evaluation complete. Total results: %d", len(results))
+
+        return StandardResponse(
+            data=EvaluateSessionsData(
+                golden_session_id=request.golden_session_id,
+                eval_set_id=request.eval_set_id,
+                results=results,
+            )
+        )
+
+    except HTTPException:
+        raise
+    except Exception as exc:
+        logger.exception("Failed to evaluate sessions")
+        raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+
+@streaming_router.post("/prepare-evaluation", response_model=StandardResponse[PrepareEvaluationData])
+async def prepare_evaluation(request: PrepareEvaluationRequest):
+    """Prepare evaluation by saving traces and eval set as downloadable files."""
+    golden_session = trace_manager.sessions.get(request.golden_session_id)
+    if not golden_session:
+        raise HTTPException(status_code=404, detail="Golden session not found")
+
+    try:
+        eval_set_response = await create_eval_set_from_session(
+            CreateEvalSetRequest(
+                session_id=request.golden_session_id,
+                eval_set_id=f"golden_{request.golden_session_id}",
+            )
+        )
+
+        import os
+        import tempfile
+
+        temp_dir = tempfile.gettempdir()
+
+        eval_set_file = os.path.join(temp_dir, f"eval_set_{request.golden_session_id}.json")
+        with open(eval_set_file, "w") as f:  # noqa: ASYNC230
+            json.dump(eval_set_response.data.eval_set, f)
+
+        trace_files = []
+        for session_id in request.session_ids:
+            session = trace_manager.sessions.get(session_id)
+            if not session or not session.is_complete:
+                continue
+
+            trace_file = await trace_manager._save_spans_to_temp_file(session)
+            trace_files.append(
+                {
+                    "session_id": session_id,
+                    "file_path": str(trace_file),
+                }
+            )
+
+        return StandardResponse(
+            data=PrepareEvaluationData(
+                eval_set_url=f"/api/streaming/download/{os.path.basename(eval_set_file)}",
+                trace_urls=[f"/api/streaming/download/{os.path.basename(tf['file_path'])}" for tf in trace_files],
+                num_traces=len(trace_files),
+            )
+        )
+
+    except HTTPException:
+        raise
+    except Exception as exc:
+        logger.exception("Failed to prepare evaluation")
+        raise HTTPException(status_code=500, detail=str(exc)) from exc
+
+
+@streaming_router.get("/download/{filename}")
+async def download_file(filename: str):
+    """Download a prepared trace or eval set file."""
+    import os
+    import tempfile
+
+    temp_dir = tempfile.gettempdir()
+    file_path = os.path.join(temp_dir, filename)
+
+    if not os.path.exists(file_path):  # noqa: ASYNC240
+        raise HTTPException(status_code=404, detail="File not found")
+
+    if not file_path.startswith(temp_dir):
+        raise HTTPException(status_code=400, detail="Invalid file path")
+
+    return FileResponse(file_path, media_type="application/json", filename=filename)
+
+
+@streaming_router.post("/get-trace", response_model=StandardResponse[GetTraceData])
+async def get_trace(request: GetTraceRequest):
+    session = trace_manager.sessions.get(request.session_id)
+    if not session:
+        raise HTTPException(status_code=404, detail="Session not found")
+
+    try:
+        import tempfile
+
+        temp_file = tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False)
+
+        unified_trace_id = session.trace_id
+
+        has_genai_spans = any(
+            span.get("attributes", [])
+            and any(
+                attr.get("key") in (OTEL_GENAI_REQUEST_MODEL, OTEL_GENAI_INPUT_MESSAGES)
+                for attr in span.get("attributes", [])
+            )
+            for span in session.spans
+        )
+
+        if has_genai_spans and not session.logs:
+            logger.warning(
+                "Session %s has GenAI spans but no logs. "
+                "Message content will be missing unless spans already enriched.",
+                request.session_id,
+            )
+
+        enriched_spans = enrich_spans_with_logs(session.spans, session.logs)
+
+        for span in enriched_spans:
+            span_copy = span.copy()
+            span_copy["traceId"] = unified_trace_id
+            temp_file.write(json.dumps(span_copy) + "\n")
+
+        temp_file.close()
+
+        with open(temp_file.name) as f:  # noqa: ASYNC230
+            trace_content = f.read()
+
+        return StandardResponse(
+            data=GetTraceData(
+                session_id=request.session_id,
+                trace_content=trace_content,
+                num_spans=len(enriched_spans),
+            )
+        )
+
+    except Exception as exc:
+        logger.exception("Failed to get trace")
+        raise HTTPException(status_code=500, detail=str(exc)) from exc

agentevals/builtin_metrics.py

@@ -0,0 +1,234 @@
+"""Built-in ADK metric evaluation — criteria construction and evaluator resolution."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import logging
+from typing import Any
+
+from google.adk.evaluation.eval_case import Invocation, get_all_tool_calls
+from google.adk.evaluation.eval_metrics import (
+    BaseCriterion,
+    EvalMetric,
+    HallucinationsCriterion,
+    JudgeModelOptions,
+    LlmAsAJudgeCriterion,
+    LlmBackedUserSimulatorCriterion,
+    RubricsBasedCriterion,
+    ToolTrajectoryCriterion,
+)
+from google.adk.evaluation.eval_rubrics import Rubric, RubricContent
+from google.adk.evaluation.evaluator import EvaluationResult, Evaluator
+
+logger = logging.getLogger(__name__)
+
+METRICS_NEEDING_EXPECTED = {
+    "tool_trajectory_avg_score",
+    "response_match_score",
+    "response_evaluation_score",
+    "final_response_match_v2",
+}
+
+METRICS_NEEDING_LLM = {
+    "final_response_match_v2",
+    "rubric_based_final_response_quality_v1",
+    "hallucinations_v1",
+    "rubric_based_tool_use_quality_v1",
+    "per_turn_user_simulator_quality_v1",
+}
+
+METRICS_NEEDING_GCP = {
+    "response_evaluation_score",
+    "safety_v1",
+}
+
+
+def rubric_strings_to_objects(rubric_texts: list[str]) -> list[Rubric]:
+    """Convert plain-text rubric strings into ADK Rubric objects."""
+    return [
+        Rubric(
+            rubric_id=f"rubric_{i}",
+            rubric_content=RubricContent(text_property=text),
+        )
+        for i, text in enumerate(rubric_texts)
+    ]
+
+
+def build_eval_metric(
+    metric_name: str,
+    judge_model: str | None,
+    threshold: float | None,
+    rubrics: list[str] | None = None,
+) -> EvalMetric:
+    """Construct an ADK ``EvalMetric`` with the appropriate criterion."""
+    effective_threshold = threshold if threshold is not None else 0.5
+
+    criterion: BaseCriterion | None = None
+
+    if metric_name == "tool_trajectory_avg_score":
+        criterion = ToolTrajectoryCriterion(threshold=effective_threshold)
+    elif metric_name == "final_response_match_v2":
+        judge_opts = JudgeModelOptions()
+        if judge_model:
+            judge_opts.judge_model = judge_model
+        criterion = LlmAsAJudgeCriterion(
+            threshold=effective_threshold,
+            judge_model_options=judge_opts,
+        )
+    elif metric_name == "hallucinations_v1":
+        judge_opts = JudgeModelOptions()
+        if judge_model:
+            judge_opts.judge_model = judge_model
+        criterion = HallucinationsCriterion(
+            threshold=effective_threshold,
+            judge_model_options=judge_opts,
+        )
+    elif metric_name in (
+        "rubric_based_final_response_quality_v1",
+        "rubric_based_tool_use_quality_v1",
+    ):
+        judge_opts = JudgeModelOptions()
+        if judge_model:
+            judge_opts.judge_model = judge_model
+        rubric_objects = rubric_strings_to_objects(rubrics) if rubrics else []
+        criterion = RubricsBasedCriterion(
+            threshold=effective_threshold,
+            judge_model_options=judge_opts,
+            rubrics=rubric_objects,
+        )
+    elif metric_name == "per_turn_user_simulator_quality_v1":
+        judge_opts = JudgeModelOptions()
+        if judge_model:
+            judge_opts.judge_model = judge_model
+        criterion = LlmBackedUserSimulatorCriterion(
+            threshold=effective_threshold,
+            judge_model_options=judge_opts,
+        )
+    elif metric_name in ("response_match_score", "response_evaluation_score", "safety_v1"):
+        criterion = BaseCriterion(threshold=effective_threshold)
+
+    return EvalMetric(
+        metric_name=metric_name,
+        threshold=effective_threshold,
+        criterion=criterion,
+    )
+
+
+def get_evaluator(eval_metric: EvalMetric) -> Evaluator:
+    """Resolve an evaluator, using direct imports for known lightweight metrics
+    to avoid pulling in heavy deps (numpy/rouge_score) via the full registry."""
+    name = eval_metric.metric_name
+
+    _DIRECT_EVALUATORS: dict[str, tuple[str, str]] = {
+        "tool_trajectory_avg_score": (
+            "google.adk.evaluation.trajectory_evaluator",
+            "TrajectoryEvaluator",
+        ),
+    }
+
+    if name in _DIRECT_EVALUATORS:
+        import importlib
+
+        mod_path, cls_name = _DIRECT_EVALUATORS[name]
+        mod = importlib.import_module(mod_path)
+        evaluator_cls = getattr(mod, cls_name)
+        return evaluator_cls(eval_metric=eval_metric)  # type: ignore[call-arg]
+
+    from google.adk.evaluation.metric_evaluator_registry import (
+        DEFAULT_METRIC_EVALUATOR_REGISTRY,
+    )
+
+    return DEFAULT_METRIC_EVALUATOR_REGISTRY.get_evaluator(eval_metric)
+
+
+def extract_trajectory_details(eval_result: EvaluationResult) -> dict[str, Any]:
+    """Extract expected vs actual tool call details from trajectory evaluation."""
+    comparisons = []
+
+    for per_inv_result in eval_result.per_invocation_results:
+        actual_inv = per_inv_result.actual_invocation
+        expected_inv = per_inv_result.expected_invocation
+
+        actual_tools = []
+        expected_tools = []
+
+        if actual_inv and actual_inv.intermediate_data:
+            tool_calls = get_all_tool_calls(actual_inv.intermediate_data)
+            actual_tools = [{"name": tc.name, "args": tc.args} for tc in tool_calls]
+
+        if expected_inv and expected_inv.intermediate_data:
+            tool_calls = get_all_tool_calls(expected_inv.intermediate_data)
+            expected_tools = [{"name": tc.name, "args": tc.args} for tc in tool_calls]
+
+        comparisons.append(
+            {
+                "invocation_id": actual_inv.invocation_id if actual_inv else None,
+                "expected": expected_tools,
+                "actual": actual_tools,
+                "matched": per_inv_result.score == 1.0,
+            }
+        )
+
+    return {"comparisons": comparisons}
+
+
+async def evaluate_builtin_metric(
+    metric_name: str,
+    actual_invocations: list[Invocation],
+    expected_invocations: list[Invocation] | None,
+    judge_model: str | None,
+    threshold: float | None,
+) -> dict[str, Any]:
+    """Evaluate a single built-in ADK metric.
+
+    Returns a dict with keys: metric_name, score, eval_status,
+    per_invocation_scores, error, details.
+    """
+    from .runner import MetricResult
+
+    if metric_name in METRICS_NEEDING_EXPECTED and not expected_invocations:
+        return MetricResult(
+            metric_name=metric_name,
+            error=(
+                f"Metric '{metric_name}' requires expected invocations "
+                f"(golden eval set), but none were provided or matched."
+            ),
+        )
+
+    try:
+        eval_metric = build_eval_metric(metric_name, judge_model, threshold)
+        evaluator: Evaluator = get_evaluator(eval_metric)
+
+        if inspect.iscoroutinefunction(evaluator.evaluate_invocations):
+            eval_result: EvaluationResult = await evaluator.evaluate_invocations(
+                actual_invocations=actual_invocations,
+                expected_invocations=expected_invocations,
+            )
+        else:
+            eval_result: EvaluationResult = await asyncio.to_thread(
+                evaluator.evaluate_invocations,
+                actual_invocations=actual_invocations,
+                expected_invocations=expected_invocations,
+            )
+
+        per_inv_scores = [r.score for r in eval_result.per_invocation_results]
+
+        details = None
+        if metric_name == "tool_trajectory_avg_score":
+            details = extract_trajectory_details(eval_result)
+
+        return MetricResult(
+            metric_name=metric_name,
+            score=eval_result.overall_score,
+            eval_status=eval_result.overall_eval_status.name,
+            per_invocation_scores=per_inv_scores,
+            details=details,
+        )
+
+    except Exception as exc:
+        logger.exception("Failed to evaluate metric '%s'", metric_name)
+        return MetricResult(
+            metric_name=metric_name,
+            error=str(exc),
+        )