agno 2.2.13__py3-none-any.whl

agno/os/routers/__init__.py

@@ -0,0 +1,3 @@
+from agno.os.routers.session.session import get_session_router
+
+__all__ = ["get_session_router"]

agno/os/routers/evals/__init__.py

@@ -0,0 +1,3 @@
+from agno.os.routers.evals.evals import get_eval_router
+
+__all__ = ["get_eval_router"]

agno/os/routers/evals/evals.py

@@ -0,0 +1,430 @@
+import logging
+from copy import deepcopy
+from typing import List, Optional, Union, cast
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+
+from agno.agent.agent import Agent
+from agno.db.base import AsyncBaseDb, BaseDb
+from agno.db.schemas.evals import EvalFilterType, EvalType
+from agno.models.utils import get_model
+from agno.os.auth import get_authentication_dependency
+from agno.os.routers.evals.schemas import (
+    DeleteEvalRunsRequest,
+    EvalRunInput,
+    EvalSchema,
+    UpdateEvalRunRequest,
+)
+from agno.os.routers.evals.utils import run_accuracy_eval, run_performance_eval, run_reliability_eval
+from agno.os.schema import (
+    BadRequestResponse,
+    InternalServerErrorResponse,
+    NotFoundResponse,
+    PaginatedResponse,
+    PaginationInfo,
+    SortOrder,
+    UnauthenticatedResponse,
+    ValidationErrorResponse,
+)
+from agno.os.settings import AgnoAPISettings
+from agno.os.utils import get_agent_by_id, get_db, get_team_by_id
+from agno.team.team import Team
+
+logger = logging.getLogger(__name__)
+
+
+def get_eval_router(
+    dbs: dict[str, list[Union[BaseDb, AsyncBaseDb]]],
+    agents: Optional[List[Agent]] = None,
+    teams: Optional[List[Team]] = None,
+    settings: AgnoAPISettings = AgnoAPISettings(),
+) -> APIRouter:
+    """Create eval router with comprehensive OpenAPI documentation for agent/team evaluation endpoints."""
+    router = APIRouter(
+        dependencies=[Depends(get_authentication_dependency(settings))],
+        tags=["Evals"],
+        responses={
+            400: {"description": "Bad Request", "model": BadRequestResponse},
+            401: {"description": "Unauthorized", "model": UnauthenticatedResponse},
+            404: {"description": "Not Found", "model": NotFoundResponse},
+            422: {"description": "Validation Error", "model": ValidationErrorResponse},
+            500: {"description": "Internal Server Error", "model": InternalServerErrorResponse},
+        },
+    )
+    return attach_routes(router=router, dbs=dbs, agents=agents, teams=teams)
+
+
+def attach_routes(
+    router: APIRouter,
+    dbs: dict[str, list[Union[BaseDb, AsyncBaseDb]]],
+    agents: Optional[List[Agent]] = None,
+    teams: Optional[List[Team]] = None,
+) -> APIRouter:
+    @router.get(
+        "/eval-runs",
+        response_model=PaginatedResponse[EvalSchema],
+        status_code=200,
+        operation_id="get_eval_runs",
+        summary="List Evaluation Runs",
+        description=(
+            "Retrieve paginated evaluation runs with filtering and sorting options. "
+            "Filter by agent, team, workflow, model, or evaluation type."
+        ),
+        responses={
+            200: {
+                "description": "Evaluation runs retrieved successfully",
+                "content": {
+                    "application/json": {
+                        "example": {
+                            "data": [
+                                {
+                                    "id": "a03fa2f4-900d-482d-afe0-470d4cd8d1f4",
+                                    "agent_id": "basic-agent",
+                                    "model_id": "gpt-4o",
+                                    "model_provider": "OpenAI",
+                                    "team_id": None,
+                                    "workflow_id": None,
+                                    "name": "Test ",
+                                    "evaluated_component_name": None,
+                                    "eval_type": "reliability",
+                                    "eval_data": {
+                                        "eval_status": "PASSED",
+                                        "failed_tool_calls": [],
+                                        "passed_tool_calls": ["multiply"],
+                                    },
+                                    "eval_input": {"expected_tool_calls": ["multiply"]},
+                                    "created_at": "2025-08-27T15:41:59Z",
+                                    "updated_at": "2025-08-27T15:41:59Z",
+                                }
+                            ]
+                        }
+                    }
+                },
+            }
+        },
+    )
+    async def get_eval_runs(
+        agent_id: Optional[str] = Query(default=None, description="Agent ID"),
+        team_id: Optional[str] = Query(default=None, description="Team ID"),
+        workflow_id: Optional[str] = Query(default=None, description="Workflow ID"),
+        model_id: Optional[str] = Query(default=None, description="Model ID"),
+        filter_type: Optional[EvalFilterType] = Query(default=None, description="Filter type", alias="type"),
+        eval_types: Optional[List[EvalType]] = Depends(parse_eval_types_filter),
+        limit: Optional[int] = Query(default=20, description="Number of eval runs to return"),
+        page: Optional[int] = Query(default=1, description="Page number"),
+        sort_by: Optional[str] = Query(default="created_at", description="Field to sort by"),
+        sort_order: Optional[SortOrder] = Query(default="desc", description="Sort order (asc or desc)"),
+        db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
+        table: Optional[str] = Query(default=None, description="The database table to use"),
+    ) -> PaginatedResponse[EvalSchema]:
+        db = await get_db(dbs, db_id, table)
+
+        if isinstance(db, AsyncBaseDb):
+            db = cast(AsyncBaseDb, db)
+            eval_runs, total_count = await db.get_eval_runs(
+                limit=limit,
+                page=page,
+                sort_by=sort_by,
+                sort_order=sort_order,
+                agent_id=agent_id,
+                team_id=team_id,
+                workflow_id=workflow_id,
+                model_id=model_id,
+                eval_type=eval_types,
+                filter_type=filter_type,
+                deserialize=False,
+            )
+        else:
+            eval_runs, total_count = db.get_eval_runs(  # type: ignore
+                limit=limit,
+                page=page,
+                sort_by=sort_by,
+                sort_order=sort_order,
+                agent_id=agent_id,
+                team_id=team_id,
+                workflow_id=workflow_id,
+                model_id=model_id,
+                eval_type=eval_types,
+                filter_type=filter_type,
+                deserialize=False,
+            )
+
+        return PaginatedResponse(
+            data=[EvalSchema.from_dict(eval_run) for eval_run in eval_runs],  # type: ignore
+            meta=PaginationInfo(
+                page=page,
+                limit=limit,
+                total_count=total_count,  # type: ignore
+                total_pages=(total_count + limit - 1) // limit if limit is not None and limit > 0 else 0,  # type: ignore
+            ),
+        )
+
+    @router.get(
+        "/eval-runs/{eval_run_id}",
+        response_model=EvalSchema,
+        status_code=200,
+        operation_id="get_eval_run",
+        summary="Get Evaluation Run",
+        description="Retrieve detailed results and metrics for a specific evaluation run.",
+        responses={
+            200: {
+                "description": "Evaluation run details retrieved successfully",
+                "content": {
+                    "application/json": {
+                        "example": {
+                            "id": "a03fa2f4-900d-482d-afe0-470d4cd8d1f4",
+                            "agent_id": "basic-agent",
+                            "model_id": "gpt-4o",
+                            "model_provider": "OpenAI",
+                            "team_id": None,
+                            "workflow_id": None,
+                            "name": "Test ",
+                            "evaluated_component_name": None,
+                            "eval_type": "reliability",
+                            "eval_data": {
+                                "eval_status": "PASSED",
+                                "failed_tool_calls": [],
+                                "passed_tool_calls": ["multiply"],
+                            },
+                            "eval_input": {"expected_tool_calls": ["multiply"]},
+                            "created_at": "2025-08-27T15:41:59Z",
+                            "updated_at": "2025-08-27T15:41:59Z",
+                        }
+                    }
+                },
+            },
+            404: {"description": "Evaluation run not found", "model": NotFoundResponse},
+        },
+    )
+    async def get_eval_run(
+        eval_run_id: str,
+        db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
+        table: Optional[str] = Query(default=None, description="Table to query eval run from"),
+    ) -> EvalSchema:
+        db = await get_db(dbs, db_id, table)
+        if isinstance(db, AsyncBaseDb):
+            db = cast(AsyncBaseDb, db)
+            eval_run = await db.get_eval_run(eval_run_id=eval_run_id, deserialize=False)
+        else:
+            eval_run = db.get_eval_run(eval_run_id=eval_run_id, deserialize=False)
+        if not eval_run:
+            raise HTTPException(status_code=404, detail=f"Eval run with id '{eval_run_id}' not found")
+
+        return EvalSchema.from_dict(eval_run)  # type: ignore
+
+    @router.delete(
+        "/eval-runs",
+        status_code=204,
+        operation_id="delete_eval_runs",
+        summary="Delete Evaluation Runs",
+        description="Delete multiple evaluation runs by their IDs. This action cannot be undone.",
+        responses={
+            204: {},
+            500: {"description": "Failed to delete evaluation runs", "model": InternalServerErrorResponse},
+        },
+    )
+    async def delete_eval_runs(
+        request: DeleteEvalRunsRequest,
+        db_id: Optional[str] = Query(default=None, description="Database ID to use for deletion"),
+        table: Optional[str] = Query(default=None, description="Table to use for deletion"),
+    ) -> None:
+        try:
+            db = await get_db(dbs, db_id, table)
+            if isinstance(db, AsyncBaseDb):
+                db = cast(AsyncBaseDb, db)
+                await db.delete_eval_runs(eval_run_ids=request.eval_run_ids)
+            else:
+                db.delete_eval_runs(eval_run_ids=request.eval_run_ids)
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Failed to delete eval runs: {e}")
+
+    @router.patch(
+        "/eval-runs/{eval_run_id}",
+        response_model=EvalSchema,
+        status_code=200,
+        operation_id="update_eval_run",
+        summary="Update Evaluation Run",
+        description="Update the name or other properties of an existing evaluation run.",
+        responses={
+            200: {
+                "description": "Evaluation run updated successfully",
+                "content": {
+                    "application/json": {
+                        "example": {
+                            "id": "a03fa2f4-900d-482d-afe0-470d4cd8d1f4",
+                            "agent_id": "basic-agent",
+                            "model_id": "gpt-4o",
+                            "model_provider": "OpenAI",
+                            "team_id": None,
+                            "workflow_id": None,
+                            "name": "Test ",
+                            "evaluated_component_name": None,
+                            "eval_type": "reliability",
+                            "eval_data": {
+                                "eval_status": "PASSED",
+                                "failed_tool_calls": [],
+                                "passed_tool_calls": ["multiply"],
+                            },
+                            "eval_input": {"expected_tool_calls": ["multiply"]},
+                            "created_at": "2025-08-27T15:41:59Z",
+                            "updated_at": "2025-08-27T15:41:59Z",
+                        }
+                    }
+                },
+            },
+            404: {"description": "Evaluation run not found", "model": NotFoundResponse},
+            500: {"description": "Failed to update evaluation run", "model": InternalServerErrorResponse},
+        },
+    )
+    async def update_eval_run(
+        eval_run_id: str,
+        request: UpdateEvalRunRequest,
+        db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
+        table: Optional[str] = Query(default=None, description="Table to use for rename operation"),
+    ) -> EvalSchema:
+        try:
+            db = await get_db(dbs, db_id, table)
+            if isinstance(db, AsyncBaseDb):
+                db = cast(AsyncBaseDb, db)
+                eval_run = await db.rename_eval_run(eval_run_id=eval_run_id, name=request.name, deserialize=False)
+            else:
+                eval_run = db.rename_eval_run(eval_run_id=eval_run_id, name=request.name, deserialize=False)
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Failed to rename eval run: {e}")
+
+        if not eval_run:
+            raise HTTPException(status_code=404, detail=f"Eval run with id '{eval_run_id}' not found")
+
+        return EvalSchema.from_dict(eval_run)  # type: ignore
+
+    @router.post(
+        "/eval-runs",
+        response_model=EvalSchema,
+        status_code=200,
+        operation_id="run_eval",
+        summary="Execute Evaluation",
+        description=(
+            "Run evaluation tests on agents or teams. Supports accuracy, performance, and reliability evaluations. "
+            "Requires either agent_id or team_id, but not both."
+        ),
+        responses={
+            200: {
+                "description": "Evaluation executed successfully",
+                "content": {
+                    "application/json": {
+                        "example": {
+                            "id": "f2b2d72f-e9e2-4f0e-8810-0a7e1ff58614",
+                            "agent_id": "basic-agent",
+                            "model_id": "gpt-4o",
+                            "model_provider": "OpenAI",
+                            "team_id": None,
+                            "workflow_id": None,
+                            "name": None,
+                            "evaluated_component_name": None,
+                            "eval_type": "reliability",
+                            "eval_data": {
+                                "eval_status": "PASSED",
+                                "failed_tool_calls": [],
+                                "passed_tool_calls": ["multiply"],
+                            },
+                            "created_at": "2025-08-27T15:41:59Z",
+                            "updated_at": "2025-08-27T15:41:59Z",
+                        }
+                    }
+                },
+            },
+            400: {"description": "Invalid request - provide either agent_id or team_id", "model": BadRequestResponse},
+            404: {"description": "Agent or team not found", "model": NotFoundResponse},
+        },
+    )
+    async def run_eval(
+        eval_run_input: EvalRunInput,
+        db_id: Optional[str] = Query(default=None, description="Database ID to use for evaluation"),
+        table: Optional[str] = Query(default=None, description="Table to use for evaluation"),
+    ) -> Optional[EvalSchema]:
+        db = await get_db(dbs, db_id, table)
+
+        if eval_run_input.agent_id and eval_run_input.team_id:
+            raise HTTPException(status_code=400, detail="Only one of agent_id or team_id must be provided")
+
+        if eval_run_input.agent_id:
+            agent = get_agent_by_id(agent_id=eval_run_input.agent_id, agents=agents)
+            if not agent:
+                raise HTTPException(status_code=404, detail=f"Agent with id '{eval_run_input.agent_id}' not found")
+
+            default_model = None
+            if (
+                hasattr(agent, "model")
+                and agent.model is not None
+                and eval_run_input.model_id is not None
+                and eval_run_input.model_provider is not None
+            ):
+                default_model = deepcopy(agent.model)
+                if eval_run_input.model_id != agent.model.id or eval_run_input.model_provider != agent.model.provider:
+                    model_provider = eval_run_input.model_provider.lower()
+                    model_id = eval_run_input.model_id.lower()
+                    model_string = f"{model_provider}:{model_id}"
+                    model = get_model(model_string)
+                    agent.model = model
+
+            team = None
+
+        elif eval_run_input.team_id:
+            team = get_team_by_id(team_id=eval_run_input.team_id, teams=teams)
+            if not team:
+                raise HTTPException(status_code=404, detail=f"Team with id '{eval_run_input.team_id}' not found")
+
+            default_model = None
+            if (
+                hasattr(team, "model")
+                and team.model is not None
+                and eval_run_input.model_id is not None
+                and eval_run_input.model_provider is not None
+            ):
+                default_model = deepcopy(team.model)
+                if eval_run_input.model_id != team.model.id or eval_run_input.model_provider != team.model.provider:
+                    model_provider = eval_run_input.model_provider.lower()
+                    model_id = eval_run_input.model_id.lower()
+                    model_string = f"{model_provider}:{model_id}"
+                    model = get_model(model_string)
+                    team.model = model
+
+            agent = None
+
+        else:
+            raise HTTPException(status_code=400, detail="One of agent_id or team_id must be provided")
+
+        # Run the evaluation
+        if eval_run_input.eval_type == EvalType.ACCURACY:
+            return await run_accuracy_eval(
+                eval_run_input=eval_run_input, db=db, agent=agent, team=team, default_model=default_model
+            )
+
+        elif eval_run_input.eval_type == EvalType.PERFORMANCE:
+            return await run_performance_eval(
+                eval_run_input=eval_run_input, db=db, agent=agent, team=team, default_model=default_model
+            )
+
+        else:
+            return await run_reliability_eval(
+                eval_run_input=eval_run_input, db=db, agent=agent, team=team, default_model=default_model
+            )
+
+    return router
+
+
+def parse_eval_types_filter(
+    eval_types: Optional[str] = Query(
+        default=None,
+        description="Comma-separated eval types (accuracy,performance,reliability)",
+        examples=["accuracy,performance"],
+    ),
+) -> Optional[List[EvalType]]:
+    """Parse comma-separated eval types into EvalType enums for filtering evaluation runs."""
+    if not eval_types:
+        return None
+    try:
+        return [EvalType(item.strip()) for item in eval_types.split(",")]
+    except ValueError as e:
+        valid_types = ", ".join([t.value for t in EvalType])
+        raise HTTPException(status_code=422, detail=f"Invalid eval_type: {e}. Valid types: {valid_types}")

agno/os/routers/evals/schemas.py

@@ -0,0 +1,142 @@
+from dataclasses import asdict
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+from agno.db.schemas.evals import EvalType
+from agno.eval import AccuracyResult, PerformanceResult, ReliabilityResult
+from agno.eval.accuracy import AccuracyEval
+from agno.eval.performance import PerformanceEval
+from agno.eval.reliability import ReliabilityEval
+
+
+class EvalRunInput(BaseModel):
+    agent_id: Optional[str] = Field(None, description="Agent ID to evaluate")
+    team_id: Optional[str] = Field(None, description="Team ID to evaluate")
+
+    model_id: Optional[str] = Field(None, description="Model ID to use for evaluation")
+    model_provider: Optional[str] = Field(None, description="Model provider name")
+    eval_type: EvalType = Field(..., description="Type of evaluation to run (accuracy, performance, or reliability)")
+    input: str = Field(..., description="Input text/query for the evaluation", min_length=1)
+    additional_guidelines: Optional[str] = Field(None, description="Additional guidelines for the evaluation")
+    additional_context: Optional[str] = Field(None, description="Additional context for the evaluation")
+    num_iterations: int = Field(1, description="Number of times to run the evaluation", ge=1, le=100)
+    name: Optional[str] = Field(None, description="Name for this evaluation run")
+
+    # Accuracy eval specific fields
+    expected_output: Optional[str] = Field(None, description="Expected output for accuracy evaluation")
+
+    # Performance eval specific fields
+    warmup_runs: int = Field(0, description="Number of warmup runs before measuring performance", ge=0, le=10)
+
+    # Reliability eval specific fields
+    expected_tool_calls: Optional[List[str]] = Field(None, description="Expected tool calls for reliability evaluation")
+
+
+class EvalSchema(BaseModel):
+    id: str = Field(..., description="Unique identifier for the evaluation run")
+
+    agent_id: Optional[str] = Field(None, description="Agent ID that was evaluated")
+    model_id: Optional[str] = Field(None, description="Model ID used in evaluation")
+    model_provider: Optional[str] = Field(None, description="Model provider name")
+    team_id: Optional[str] = Field(None, description="Team ID that was evaluated")
+    workflow_id: Optional[str] = Field(None, description="Workflow ID that was evaluated")
+    name: Optional[str] = Field(None, description="Name of the evaluation run")
+    evaluated_component_name: Optional[str] = Field(None, description="Name of the evaluated component")
+    eval_type: EvalType = Field(..., description="Type of evaluation (accuracy, performance, or reliability)")
+    eval_data: Dict[str, Any] = Field(..., description="Evaluation results and metrics")
+    eval_input: Optional[Dict[str, Any]] = Field(None, description="Input parameters used for the evaluation")
+    created_at: Optional[datetime] = Field(None, description="Timestamp when evaluation was created")
+    updated_at: Optional[datetime] = Field(None, description="Timestamp when evaluation was last updated")
+
+    @classmethod
+    def from_dict(cls, eval_run: Dict[str, Any]) -> "EvalSchema":
+        return cls(
+            id=eval_run["run_id"],
+            name=eval_run.get("name"),
+            agent_id=eval_run.get("agent_id"),
+            model_id=eval_run.get("model_id"),
+            model_provider=eval_run.get("model_provider"),
+            team_id=eval_run.get("team_id"),
+            workflow_id=eval_run.get("workflow_id"),
+            evaluated_component_name=eval_run.get("evaluated_component_name"),
+            eval_type=eval_run["eval_type"],
+            eval_data=eval_run["eval_data"],
+            eval_input=eval_run.get("eval_input"),
+            created_at=datetime.fromtimestamp(eval_run["created_at"], tz=timezone.utc),
+            updated_at=datetime.fromtimestamp(eval_run["updated_at"], tz=timezone.utc),
+        )
+
+    @classmethod
+    def from_accuracy_eval(cls, accuracy_eval: AccuracyEval, result: AccuracyResult) -> "EvalSchema":
+        model_provider = (
+            accuracy_eval.agent.model.provider
+            if accuracy_eval.agent and accuracy_eval.agent.model
+            else accuracy_eval.team.model.provider
+            if accuracy_eval.team and accuracy_eval.team.model
+            else None
+        )
+        return cls(
+            id=accuracy_eval.eval_id,
+            name=accuracy_eval.name,
+            agent_id=accuracy_eval.agent.id if accuracy_eval.agent else None,
+            team_id=accuracy_eval.team.id if accuracy_eval.team else None,
+            workflow_id=None,
+            model_id=accuracy_eval.agent.model.id if accuracy_eval.agent else accuracy_eval.team.model.id,  # type: ignore
+            model_provider=model_provider,
+            eval_type=EvalType.ACCURACY,
+            eval_data=asdict(result),
+        )
+
+    @classmethod
+    def from_performance_eval(
+        cls,
+        performance_eval: PerformanceEval,
+        result: PerformanceResult,
+        model_id: Optional[str] = None,
+        model_provider: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        team_id: Optional[str] = None,
+    ) -> "EvalSchema":
+        return cls(
+            id=performance_eval.eval_id,
+            name=performance_eval.name,
+            agent_id=agent_id,
+            team_id=team_id,
+            workflow_id=None,
+            model_id=model_id,
+            model_provider=model_provider,
+            eval_type=EvalType.PERFORMANCE,
+            eval_data=asdict(result),
+        )
+
+    @classmethod
+    def from_reliability_eval(
+        cls,
+        reliability_eval: ReliabilityEval,
+        result: ReliabilityResult,
+        model_id: Optional[str] = None,
+        model_provider: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        team_id: Optional[str] = None,
+    ) -> "EvalSchema":
+        return cls(
+            id=reliability_eval.eval_id,
+            name=reliability_eval.name,
+            agent_id=agent_id,
+            team_id=team_id,
+            workflow_id=None,
+            model_id=model_id,
+            model_provider=model_provider,
+            eval_type=EvalType.RELIABILITY,
+            eval_data=asdict(result),
+        )
+
+
+class DeleteEvalRunsRequest(BaseModel):
+    eval_run_ids: List[str] = Field(..., description="List of evaluation run IDs to delete", min_length=1)
+
+
+class UpdateEvalRunRequest(BaseModel):
+    name: str = Field(..., description="New name for the evaluation run", min_length=1, max_length=255)