agno 2.0.0a1__py3-none-any.whl → 2.0.0rc2__py3-none-any.whl

agno/os/routers/evals/evals.py

@@ -16,7 +16,16 @@ from agno.os.routers.evals.schemas import (
     UpdateEvalRunRequest,
 )
 from agno.os.routers.evals.utils import run_accuracy_eval, run_performance_eval, run_reliability_eval
-from agno.os.schema import PaginatedResponse, PaginationInfo, SortOrder
+from agno.os.schema import (
+    BadRequestResponse,
+    InternalServerErrorResponse,
+    NotFoundResponse,
+    PaginatedResponse,
+    PaginationInfo,
+    SortOrder,
+    UnauthenticatedResponse,
+    ValidationErrorResponse,
+)
 from agno.os.settings import AgnoAPISettings
 from agno.os.utils import get_agent_by_id, get_db, get_team_by_id
 from agno.team.team import Team
@@ -30,14 +39,67 @@ def get_eval_router(
     teams: Optional[List[Team]] = None,
     settings: AgnoAPISettings = AgnoAPISettings(),
 ) -> APIRouter:
-    router = APIRouter(dependencies=[Depends(get_authentication_dependency(settings))], tags=["Evals"])
+    """Create eval router with comprehensive OpenAPI documentation for agent/team evaluation endpoints."""
+    router = APIRouter(
+        dependencies=[Depends(get_authentication_dependency(settings))],
+        tags=["Evals"],
+        responses={
+            400: {"description": "Bad Request", "model": BadRequestResponse},
+            401: {"description": "Unauthorized", "model": UnauthenticatedResponse},
+            404: {"description": "Not Found", "model": NotFoundResponse},
+            422: {"description": "Validation Error", "model": ValidationErrorResponse},
+            500: {"description": "Internal Server Error", "model": InternalServerErrorResponse},
+        },
+    )
     return attach_routes(router=router, dbs=dbs, agents=agents, teams=teams)
 
 
 def attach_routes(
     router: APIRouter, dbs: dict[str, BaseDb], agents: Optional[List[Agent]] = None, teams: Optional[List[Team]] = None
 ) -> APIRouter:
-    @router.get("/eval-runs", response_model=PaginatedResponse[EvalSchema], status_code=200)
+    @router.get(
+        "/eval-runs",
+        response_model=PaginatedResponse[EvalSchema],
+        status_code=200,
+        operation_id="get_eval_runs",
+        summary="List Evaluation Runs",
+        description=(
+            "Retrieve paginated evaluation runs with filtering and sorting options. "
+            "Filter by agent, team, workflow, model, or evaluation type."
+        ),
+        responses={
+            200: {
+                "description": "Evaluation runs retrieved successfully",
+                "content": {
+                    "application/json": {
+                        "example": {
+                            "data": [
+                                {
+                                    "id": "a03fa2f4-900d-482d-afe0-470d4cd8d1f4",
+                                    "agent_id": "basic-agent",
+                                    "model_id": "gpt-4o",
+                                    "model_provider": "OpenAI",
+                                    "team_id": None,
+                                    "workflow_id": None,
+                                    "name": "Test ",
+                                    "evaluated_component_name": None,
+                                    "eval_type": "reliability",
+                                    "eval_data": {
+                                        "eval_status": "PASSED",
+                                        "failed_tool_calls": [],
+                                        "passed_tool_calls": ["multiply"],
+                                    },
+                                    "eval_input": {"expected_tool_calls": ["multiply"]},
+                                    "created_at": "2025-08-27T15:41:59Z",
+                                    "updated_at": "2025-08-27T15:41:59Z",
+                                }
+                            ]
+                        }
+                    }
+                },
+            }
+        },
+    )
     async def get_eval_runs(
         agent_id: Optional[str] = Query(default=None, description="Agent ID"),
         team_id: Optional[str] = Query(default=None, description="Team ID"),
@@ -76,7 +138,43 @@ def attach_routes(
             ),
         )
 
-    @router.get("/eval-runs/{eval_run_id}", response_model=EvalSchema, status_code=200)
+    @router.get(
+        "/eval-runs/{eval_run_id}",
+        response_model=EvalSchema,
+        status_code=200,
+        operation_id="get_eval_run",
+        summary="Get Evaluation Run",
+        description="Retrieve detailed results and metrics for a specific evaluation run.",
+        responses={
+            200: {
+                "description": "Evaluation run details retrieved successfully",
+                "content": {
+                    "application/json": {
+                        "example": {
+                            "id": "a03fa2f4-900d-482d-afe0-470d4cd8d1f4",
+                            "agent_id": "basic-agent",
+                            "model_id": "gpt-4o",
+                            "model_provider": "OpenAI",
+                            "team_id": None,
+                            "workflow_id": None,
+                            "name": "Test ",
+                            "evaluated_component_name": None,
+                            "eval_type": "reliability",
+                            "eval_data": {
+                                "eval_status": "PASSED",
+                                "failed_tool_calls": [],
+                                "passed_tool_calls": ["multiply"],
+                            },
+                            "eval_input": {"expected_tool_calls": ["multiply"]},
+                            "created_at": "2025-08-27T15:41:59Z",
+                            "updated_at": "2025-08-27T15:41:59Z",
+                        }
+                    }
+                },
+            },
+            404: {"description": "Evaluation run not found", "model": NotFoundResponse},
+        },
+    )
     async def get_eval_run(
         eval_run_id: str,
         db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
@@ -88,10 +186,20 @@ def attach_routes(
 
         return EvalSchema.from_dict(eval_run)  # type: ignore
 
-    @router.delete("/eval-runs", status_code=204)
+    @router.delete(
+        "/eval-runs",
+        status_code=204,
+        operation_id="delete_eval_runs",
+        summary="Delete Evaluation Runs",
+        description="Delete multiple evaluation runs by their IDs. This action cannot be undone.",
+        responses={
+            204: {},
+            500: {"description": "Failed to delete evaluation runs", "model": InternalServerErrorResponse},
+        },
+    )
     async def delete_eval_runs(
         request: DeleteEvalRunsRequest,
-        db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
+        db_id: Optional[str] = Query(default=None, description="Database ID to use for deletion"),
     ) -> None:
         try:
             db = get_db(dbs, db_id)
@@ -99,7 +207,44 @@ def attach_routes(
         except Exception as e:
             raise HTTPException(status_code=500, detail=f"Failed to delete eval runs: {e}")
 
-    @router.patch("/eval-runs/{eval_run_id}", response_model=EvalSchema, status_code=200)
+    @router.patch(
+        "/eval-runs/{eval_run_id}",
+        response_model=EvalSchema,
+        status_code=200,
+        operation_id="update_eval_run",
+        summary="Update Evaluation Run",
+        description="Update the name or other properties of an existing evaluation run.",
+        responses={
+            200: {
+                "description": "Evaluation run updated successfully",
+                "content": {
+                    "application/json": {
+                        "example": {
+                            "id": "a03fa2f4-900d-482d-afe0-470d4cd8d1f4",
+                            "agent_id": "basic-agent",
+                            "model_id": "gpt-4o",
+                            "model_provider": "OpenAI",
+                            "team_id": None,
+                            "workflow_id": None,
+                            "name": "Test ",
+                            "evaluated_component_name": None,
+                            "eval_type": "reliability",
+                            "eval_data": {
+                                "eval_status": "PASSED",
+                                "failed_tool_calls": [],
+                                "passed_tool_calls": ["multiply"],
+                            },
+                            "eval_input": {"expected_tool_calls": ["multiply"]},
+                            "created_at": "2025-08-27T15:41:59Z",
+                            "updated_at": "2025-08-27T15:41:59Z",
+                        }
+                    }
+                },
+            },
+            404: {"description": "Evaluation run not found", "model": NotFoundResponse},
+            500: {"description": "Failed to update evaluation run", "model": InternalServerErrorResponse},
+        },
+    )
     async def update_eval_run(
         eval_run_id: str,
         request: UpdateEvalRunRequest,
@@ -116,10 +261,49 @@ def attach_routes(
 
         return EvalSchema.from_dict(eval_run)  # type: ignore
 
-    @router.post("/eval-runs", response_model=EvalSchema, status_code=200)
+    @router.post(
+        "/eval-runs",
+        response_model=EvalSchema,
+        status_code=200,
+        operation_id="run_eval",
+        summary="Execute Evaluation",
+        description=(
+            "Run evaluation tests on agents or teams. Supports accuracy, performance, and reliability evaluations. "
+            "Requires either agent_id or team_id, but not both."
+        ),
+        responses={
+            200: {
+                "description": "Evaluation executed successfully",
+                "content": {
+                    "application/json": {
+                        "example": {
+                            "id": "f2b2d72f-e9e2-4f0e-8810-0a7e1ff58614",
+                            "agent_id": "basic-agent",
+                            "model_id": "gpt-4o",
+                            "model_provider": "OpenAI",
+                            "team_id": None,
+                            "workflow_id": None,
+                            "name": None,
+                            "evaluated_component_name": None,
+                            "eval_type": "reliability",
+                            "eval_data": {
+                                "eval_status": "PASSED",
+                                "failed_tool_calls": [],
+                                "passed_tool_calls": ["multiply"],
+                            },
+                            "created_at": "2025-08-27T15:41:59Z",
+                            "updated_at": "2025-08-27T15:41:59Z",
+                        }
+                    }
+                },
+            },
+            400: {"description": "Invalid request - provide either agent_id or team_id", "model": BadRequestResponse},
+            404: {"description": "Agent or team not found", "model": NotFoundResponse},
+        },
+    )
     async def run_eval(
         eval_run_input: EvalRunInput,
-        db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
+        db_id: Optional[str] = Query(default=None, description="Database ID to use for evaluation"),
     ) -> Optional[EvalSchema]:
         db = get_db(dbs, db_id)
 
@@ -193,12 +377,17 @@ def attach_routes(
 
 
 def parse_eval_types_filter(
-    eval_types: Optional[str] = Query(default=None, description="Comma-separated eval types"),
+    eval_types: Optional[str] = Query(
+        default=None,
+        description="Comma-separated eval types (accuracy,performance,reliability)",
+        example="accuracy,performance",
+    ),
 ) -> Optional[List[EvalType]]:
-    """Parse a comma-separated string of eval types into a list of EvalType enums"""
+    """Parse comma-separated eval types into EvalType enums for filtering evaluation runs."""
     if not eval_types:
         return None
     try:
         return [EvalType(item.strip()) for item in eval_types.split(",")]
     except ValueError as e:
-        raise HTTPException(status_code=422, detail=f"Invalid eval_type: {e}")
+        valid_types = ", ".join([t.value for t in EvalType])
+        raise HTTPException(status_code=422, detail=f"Invalid eval_type: {e}. Valid types: {valid_types}")