agentevals-cli 0.5.2__py3-none-any.whl

agentevals/__init__.py

@@ -0,0 +1,16 @@
+"""agentevals: Standalone CLI to evaluate agent traces using ADK's scoring framework."""
+
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("agentevals")
+except PackageNotFoundError:
+    __version__ = "0.0.0-dev"
+
+
+def __getattr__(name):
+    if name == "AgentEvals":
+        from .sdk import AgentEvals
+
+        return AgentEvals
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

agentevals/_protocol.py

@@ -0,0 +1,83 @@
+"""CLI-internal protocol types for the custom evaluator JSON wire format.
+
+These mirror the types in ``agentevals_evaluator_sdk.types`` but are owned by
+the CLI so that the CLI and SDK packages can be versioned independently.  The
+JSON schema produced/consumed by these models is the contract — not the Python
+types themselves.
+
+Protocol versioning rules:
+- ``protocol_version`` uses ``"MAJOR.MINOR"`` format.
+- MINOR bumps are additive-only (new fields with defaults).  Old deserializers
+  silently ignore unknown fields.
+- MAJOR bumps signal breaking changes (removed/renamed fields, type changes).
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+
+PROTOCOL_VERSION = "1.0"
+
+
+class ToolCallData(BaseModel):
+    """A single tool call made by the agent."""
+
+    name: str
+    args: dict[str, Any] = Field(default_factory=dict)
+
+
+class ToolResponseData(BaseModel):
+    """A single tool response received by the agent."""
+
+    name: str
+    output: str = ""
+
+
+class IntermediateStepData(BaseModel):
+    """Intermediate steps between user input and final response."""
+
+    tool_calls: list[ToolCallData] = Field(default_factory=list)
+    tool_responses: list[ToolResponseData] = Field(default_factory=list)
+
+
+class InvocationData(BaseModel):
+    """Simplified, language-agnostic representation of a single agent turn."""
+
+    invocation_id: str = ""
+    user_content: str = ""
+    final_response: Optional[str] = None
+    intermediate_steps: IntermediateStepData = Field(default_factory=IntermediateStepData)
+
+
+class EvalInput(BaseModel):
+    """Input payload sent to a custom evaluator on stdin."""
+
+    protocol_version: str = PROTOCOL_VERSION
+    metric_name: str
+    threshold: float = 0.5
+    config: dict[str, Any] = Field(default_factory=dict)
+    invocations: list[InvocationData] = Field(default_factory=list)
+    expected_invocations: Optional[list[InvocationData]] = None
+
+
+class EvalStatus(str, Enum):
+    """Allowed ``status`` values on the evaluator JSON wire format (matches evaluator-sdk)."""
+
+    PASSED = "PASSED"
+    FAILED = "FAILED"
+    NOT_EVALUATED = "NOT_EVALUATED"
+
+
+class EvalResult(BaseModel):
+    """Output payload expected from a custom evaluator on stdout."""
+
+    score: float = Field(ge=0.0, le=1.0)
+    status: Optional[EvalStatus] = Field(
+        default=None,
+        description="Derived from score vs threshold if omitted.",
+    )
+    per_invocation_scores: list[Optional[float]] = Field(default_factory=list)
+    details: Optional[dict[str, Any]] = None

agentevals/api/app.py

@@ -0,0 +1,137 @@
+"""FastAPI application for agentevals REST API."""
+
+import asyncio
+import json
+import logging
+import os
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse
+
+from agentevals import __version__
+
+from ..utils.log_buffer import log_buffer
+from .debug_routes import debug_router
+from .debug_routes import set_trace_manager as set_debug_trace_manager
+from .routes import router
+
+try:
+    from dotenv import load_dotenv
+
+    env_path = Path(__file__).parent.parent.parent.parent / ".env"
+    if env_path.exists():
+        load_dotenv(env_path)
+except ImportError:
+    pass
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    log_level_str = os.getenv("AGENTEVALS_LOG_LEVEL", "INFO").upper()
+    log_level = getattr(logging, log_level_str, logging.INFO)
+    logging.basicConfig(
+        level=log_level,
+        format="%(levelname)s:%(name)s:%(message)s",
+        force=True,
+    )
+    ae_logger = logging.getLogger("agentevals")
+    ae_logger.setLevel(log_level)
+    if log_buffer not in ae_logger.handlers:
+        log_buffer.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s"))
+        ae_logger.addHandler(log_buffer)
+    if _trace_manager:
+        _trace_manager.start_cleanup_task()
+    yield
+    if _trace_manager:
+        await _trace_manager.shutdown()
+    ae_logger.removeHandler(log_buffer)
+
+
+app = FastAPI(
+    title="agentevals API",
+    version=__version__,
+    description="REST API for evaluating agent traces using ADK's scoring framework",
+    lifespan=lifespan,
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["http://localhost:5173", "http://localhost:5174"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+    expose_headers=["*"],
+)
+
+app.include_router(router, prefix="/api")
+app.include_router(debug_router, prefix="/api/debug")
+
+_live_mode = os.getenv("AGENTEVALS_LIVE") == "1"
+_trace_manager = None
+
+if _live_mode:
+    from fastapi import WebSocket
+
+    from ..streaming.ws_server import StreamingTraceManager
+    from .streaming_routes import set_trace_manager, streaming_router
+
+    app.include_router(streaming_router, prefix="/api/streaming")
+    _trace_manager = StreamingTraceManager()
+    set_trace_manager(_trace_manager)
+    set_debug_trace_manager(_trace_manager)
+
+    @app.websocket("/ws/traces")
+    async def websocket_endpoint(websocket: WebSocket):
+        await _trace_manager.handle_connection(websocket)
+
+    @app.get("/stream/ui-updates")
+    async def ui_updates_stream():
+        async def event_generator():
+            queue = _trace_manager.register_sse_client()
+            try:
+                while True:
+                    event = await queue.get()
+                    if event is None:
+                        break
+                    yield f"data: {json.dumps(event)}\n\n"
+            except asyncio.CancelledError:
+                pass
+            finally:
+                _trace_manager.unregister_sse_client(queue)
+
+        return StreamingResponse(
+            event_generator(),
+            media_type="text/event-stream",
+            headers={
+                "Cache-Control": "no-cache",
+                "Connection": "keep-alive",
+            },
+        )
+
+
+def get_trace_manager():
+    return _trace_manager
+
+
+_static_dir = Path(__file__).parent.parent / "_static"
+_has_ui = _static_dir.is_dir() and (_static_dir / "index.html").exists()
+
+if _has_ui and not os.getenv("AGENTEVALS_HEADLESS"):
+    from fastapi.responses import FileResponse
+    from fastapi.staticfiles import StaticFiles
+
+    app.mount("/assets", StaticFiles(directory=_static_dir / "assets"), name="ui-assets")
+
+    @app.get("/")
+    async def root():
+        return FileResponse(_static_dir / "index.html")
+
+    @app.get("/{path:path}")
+    async def spa_fallback(path: str):
+        file_path = _static_dir / path
+        if file_path.is_file():
+            return FileResponse(file_path)
+        return FileResponse(_static_dir / "index.html")

agentevals/api/debug_routes.py

@@ -0,0 +1,268 @@
+from __future__ import annotations
+
+import glob
+import importlib.metadata
+import io
+import json
+import logging
+import os
+import platform
+import sys
+import tempfile
+import zipfile
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+
+from fastapi import APIRouter, HTTPException, UploadFile
+from fastapi import File as FastAPIFile
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
+
+from agentevals import __version__
+
+from ..utils.log_buffer import log_buffer
+from .models import DebugLoadData, SessionInfo, StandardResponse, WSSessionCompleteEvent, WSSessionStartedEvent
+
+if TYPE_CHECKING:
+    from ..streaming.ws_server import StreamingTraceManager
+
+logger = logging.getLogger(__name__)
+
+debug_router = APIRouter()
+
+_trace_manager: StreamingTraceManager | None = None
+
+
+def set_trace_manager(manager: StreamingTraceManager) -> None:
+    global _trace_manager
+    _trace_manager = manager
+
+
+class FrontendDiagnostics(BaseModel):
+    user_description: str = ""
+    browser_info: dict = {}
+    console_logs: list[dict] = []
+    app_state: dict = {}
+    network_errors: list[dict] = []
+
+
+def _get_package_version(name: str) -> str | None:
+    try:
+        return importlib.metadata.version(name)
+    except importlib.metadata.PackageNotFoundError:
+        return None
+
+
+def _collect_environment() -> dict:
+    packages = [
+        "fastapi",
+        "uvicorn",
+        "google-adk",
+        "google-genai",
+        "opentelemetry-sdk",
+        "opentelemetry-api",
+        "pydantic",
+    ]
+    return {
+        "timestamp": datetime.now(tz=UTC).isoformat(),
+        "agentevals_version": __version__,
+        "python_version": sys.version,
+        "os": platform.system(),
+        "os_version": platform.release(),
+        "machine": platform.machine(),
+        "packages": {p: _get_package_version(p) for p in packages},
+        "config": {
+            "log_level": os.getenv("AGENTEVALS_LOG_LEVEL", "INFO"),
+            "live_mode": os.getenv("AGENTEVALS_LIVE") == "1",
+        },
+        "api_keys": {
+            "google": bool(os.getenv("GOOGLE_API_KEY") or os.getenv("GEMINI_API_KEY")),
+            "anthropic": bool(os.getenv("ANTHROPIC_API_KEY")),
+            "openai": bool(os.getenv("OPENAI_API_KEY")),
+        },
+    }
+
+
+def _collect_sessions() -> list[dict]:
+    if not _trace_manager:
+        return []
+
+    sessions_data = []
+    for session in _trace_manager.sessions.values():
+        sessions_data.append(
+            {
+                "session_id": session.session_id,
+                "trace_id": session.trace_id,
+                "eval_set_id": session.eval_set_id,
+                "started_at": session.started_at.isoformat(),
+                "is_complete": session.is_complete,
+                "span_count": len(session.spans),
+                "log_count": len(session.logs),
+                "metadata": session.metadata,
+                "spans": session.spans,
+                "logs": session.logs,
+            }
+        )
+    return sessions_data
+
+
+def _collect_temp_files(session_ids: set[str] | None = None) -> dict[str, str]:
+    """Collect temp files, filtering JSONL files to current sessions only."""
+    tmp_dir = tempfile.gettempdir()
+    files = {}
+    for pattern in ["agentevals_*.jsonl", "eval_set_*.json"]:
+        for path in glob.glob(os.path.join(tmp_dir, pattern)):
+            basename = os.path.basename(path)
+            # Filter JSONL files to only include current sessions
+            if session_ids is not None and basename.endswith(".jsonl"):
+                # Extract session ID from filename: agentevals_{session_id}.jsonl
+                sid = basename.removeprefix("agentevals_").removesuffix(".jsonl")
+                if sid not in session_ids:
+                    continue
+            try:
+                with open(path) as f:
+                    files[basename] = f.read()
+            except OSError:
+                logger.debug("Could not read temp file %s", path)
+    return files
+
+
+@debug_router.post("/bundle")
+async def create_debug_bundle(diagnostics: FrontendDiagnostics):
+    timestamp = datetime.now(tz=UTC).strftime("%Y%m%d-%H%M%S")
+    prefix = f"bug-report-{timestamp}"
+
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf:
+        env = _collect_environment()
+        metadata = {
+            **env,
+            "user_description": diagnostics.user_description,
+            "browser_info": diagnostics.browser_info,
+        }
+        zf.writestr(f"{prefix}/metadata.json", json.dumps(metadata, indent=2))
+
+        sessions = _collect_sessions()
+        for s in sessions:
+            sid = s["session_id"]
+            zf.writestr(
+                f"{prefix}/sessions/{sid}/spans.json",
+                json.dumps(s["spans"], indent=2),
+            )
+            zf.writestr(
+                f"{prefix}/sessions/{sid}/logs.json",
+                json.dumps(s["logs"], indent=2),
+            )
+            session_meta = {k: v for k, v in s.items() if k not in ("spans", "logs")}
+            zf.writestr(
+                f"{prefix}/sessions/{sid}/session_meta.json",
+                json.dumps(session_meta, indent=2),
+            )
+
+        zf.writestr(f"{prefix}/backend_logs.txt", log_buffer.get_text())
+
+        current_session_ids = {s["session_id"] for s in sessions}
+        temp_files = _collect_temp_files(session_ids=current_session_ids)
+        for filename, content in temp_files.items():
+            zf.writestr(f"{prefix}/temp_files/{filename}", content)
+
+        zf.writestr(
+            f"{prefix}/frontend_state.json",
+            json.dumps(diagnostics.app_state, indent=2),
+        )
+        zf.writestr(
+            f"{prefix}/console_logs.json",
+            json.dumps(diagnostics.console_logs, indent=2),
+        )
+        zf.writestr(
+            f"{prefix}/network_errors.json",
+            json.dumps(diagnostics.network_errors, indent=2),
+        )
+
+    buf.seek(0)
+    return StreamingResponse(
+        buf,
+        media_type="application/zip",
+        headers={"Content-Disposition": f'attachment; filename="bug-report-{timestamp}.zip"'},
+    )
+
+
+@debug_router.post("/load", response_model=StandardResponse[DebugLoadData])
+async def load_debug_bundle(file: UploadFile = FastAPIFile(...)):
+    if not _trace_manager:
+        raise HTTPException(
+            status_code=400,
+            detail="Live mode is not enabled. Start with: agentevals serve --dev",
+        )
+
+    content = await file.read()
+    try:
+        zf = zipfile.ZipFile(io.BytesIO(content))
+    except zipfile.BadZipFile as exc:
+        raise HTTPException(status_code=400, detail="Invalid ZIP file") from exc
+
+    session_dirs: dict[str, list[str]] = {}
+    for name in zf.namelist():
+        parts = name.split("/")
+        if len(parts) >= 4 and parts[-3] == "sessions":
+            sid = parts[-2]
+            session_dirs.setdefault(sid, []).append(name)
+
+    if not session_dirs:
+        raise HTTPException(status_code=400, detail="No sessions found in ZIP")
+
+    from ..streaming.session import TraceSession
+
+    loaded = []
+    for sid, files in session_dirs.items():
+        meta_file = next((f for f in files if f.endswith("session_meta.json")), None)
+        spans_file = next((f for f in files if f.endswith("spans.json")), None)
+        logs_file = next((f for f in files if f.endswith("logs.json")), None)
+
+        if not spans_file:
+            continue
+
+        meta = json.loads(zf.read(meta_file)) if meta_file else {}
+        spans = json.loads(zf.read(spans_file))
+        logs = json.loads(zf.read(logs_file)) if logs_file else []
+
+        session = TraceSession(
+            session_id=meta.get("session_id", sid),
+            trace_id=meta.get("trace_id", sid),
+            eval_set_id=meta.get("eval_set_id"),
+            spans=spans,
+            logs=logs,
+            is_complete=True,
+            metadata=meta.get("metadata", {}),
+        )
+
+        _trace_manager.sessions[session.session_id] = session
+
+        await _trace_manager.broadcast_to_ui(
+            WSSessionStartedEvent(
+                session=SessionInfo(
+                    session_id=session.session_id,
+                    trace_id=session.trace_id,
+                    eval_set_id=session.eval_set_id,
+                    span_count=len(session.spans),
+                    is_complete=False,
+                    started_at=session.started_at.isoformat(),
+                    metadata=session.metadata,
+                ),
+            ).model_dump(by_alias=True)
+        )
+
+        invocations_data = await _trace_manager._extract_invocations(session)
+        await _trace_manager._save_spans_to_temp_file(session)
+
+        await _trace_manager.broadcast_to_ui(
+            WSSessionCompleteEvent(
+                session_id=session.session_id,
+                invocations=invocations_data,
+            ).model_dump(by_alias=True)
+        )
+
+        loaded.append(session.session_id)
+        logger.info("Loaded session from bug report: %s", session.session_id)
+
+    return StandardResponse(data=DebugLoadData(loaded_sessions=loaded, count=len(loaded)))

agentevals/api/models.py

@@ -0,0 +1,204 @@
+"""Pydantic response and event models for the agentevals API.
+
+Provides a StandardResponse[T] envelope, typed REST response models,
+SSE evaluation event models, and WebSocket/UI broadcast event models.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Generic, TypeVar
+
+from pydantic import BaseModel, ConfigDict, Field
+from pydantic.alias_generators import to_camel
+
+T = TypeVar("T")
+
+
+class CamelModel(BaseModel):
+    model_config = ConfigDict(
+        alias_generator=to_camel,
+        populate_by_name=True,
+    )
+
+
+class StandardResponse(CamelModel, Generic[T]):
+    data: T
+    error: str | None = None
+
+
+# ---------------------------------------------------------------------------
+# REST response data models
+# ---------------------------------------------------------------------------
+
+
+class HealthData(CamelModel):
+    status: str
+    version: str
+
+
+class ApiKeyStatus(CamelModel):
+    google: bool
+    anthropic: bool
+    openai: bool
+
+
+class ConfigData(CamelModel):
+    api_keys: ApiKeyStatus
+
+
+class MetricInfo(CamelModel):
+    name: str
+    category: str
+    requires_eval_set: bool
+    requires_llm: bool = Field(alias="requiresLLM")
+    requires_gcp: bool = Field(alias="requiresGCP")
+    requires_rubrics: bool
+    description: str
+    working: bool
+
+
+class EvalSetValidation(CamelModel):
+    valid: bool
+    eval_set_id: str | None = None
+    num_cases: int | None = None
+    errors: list[str] = Field(default_factory=list)
+
+
+class SessionInfo(CamelModel):
+    session_id: str
+    trace_id: str
+    eval_set_id: str | None = None
+    span_count: int
+    is_complete: bool
+    started_at: str
+    metadata: dict[str, Any] = Field(default_factory=dict)
+    invocations: list[dict[str, Any]] | None = None
+
+
+class CreateEvalSetData(CamelModel):
+    eval_set: dict[str, Any]
+    num_invocations: int
+
+
+class SessionEvalResult(CamelModel):
+    session_id: str
+    trace_id: str | None = None
+    num_invocations: int | None = None
+    metric_results: list[dict[str, Any]] | None = None
+    error: str | None = None
+
+
+class EvaluateSessionsData(CamelModel):
+    golden_session_id: str
+    eval_set_id: str
+    results: list[SessionEvalResult]
+
+
+class PrepareEvaluationData(CamelModel):
+    eval_set_url: str
+    trace_urls: list[str]
+    num_traces: int
+
+
+class GetTraceData(CamelModel):
+    session_id: str
+    trace_content: str
+    num_spans: int
+
+
+class DebugLoadData(CamelModel):
+    loaded_sessions: list[str]
+    count: int
+
+
+# ---------------------------------------------------------------------------
+# SSE evaluation event models
+# ---------------------------------------------------------------------------
+
+
+class SSEProgressEvent(CamelModel):
+    message: str
+
+
+class SSETraceProgress(CamelModel):
+    trace_id: str
+    partial_result: dict[str, Any]
+
+
+class SSETraceProgressEvent(CamelModel):
+    trace_progress: SSETraceProgress
+
+
+class SSEPerformanceMetricsEvent(CamelModel):
+    trace_id: str
+    performance_metrics: dict[str, Any]
+    trace_metadata: dict[str, Any] | None = None
+
+
+class SSEDoneEvent(CamelModel):
+    done: bool = True
+    result: dict[str, Any]
+
+
+class SSEErrorEvent(CamelModel):
+    error: str
+
+
+# ---------------------------------------------------------------------------
+# WebSocket / UI broadcast event models
+# ---------------------------------------------------------------------------
+
+
+class WSSessionStartedEvent(CamelModel):
+    type: str = "session_started"
+    session: SessionInfo
+
+
+class WSSessionCompleteEvent(CamelModel):
+    type: str = "session_complete"
+    session_id: str
+    invocations: list[dict[str, Any]]
+
+
+class WSSpanReceivedEvent(CamelModel):
+    type: str = "span_received"
+    session_id: str
+    span: dict[str, Any]
+
+
+class WSUserInputEvent(CamelModel):
+    type: str = "user_input"
+    session_id: str
+    invocation_id: str
+    text: str
+    timestamp: float
+
+
+class WSAgentResponseEvent(CamelModel):
+    type: str = "agent_response"
+    session_id: str
+    invocation_id: str
+    text: str
+    timestamp: float
+
+
+class WSToolCallEvent(CamelModel):
+    type: str = "tool_call"
+    session_id: str
+    invocation_id: str
+    tool_call: dict[str, Any]
+    timestamp: float
+
+
+class WSTokenUpdateEvent(CamelModel):
+    type: str = "token_update"
+    session_id: str
+    invocation_id: str | None = None
+    input_tokens: int
+    output_tokens: int
+    model: str | None = None
+
+
+class WSErrorEvent(CamelModel):
+    type: str = "error"
+    message: str