agno 2.2.13__py3-none-any.whl → 2.4.3__py3-none-any.whl

agno/os/routers/evals/evals.py

@@ -2,20 +2,25 @@ import logging
 from copy import deepcopy
 from typing import List, Optional, Union, cast
 
-from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi import APIRouter, Depends, HTTPException, Query, Request
 
-from agno.agent.agent import Agent
+from agno.agent import Agent, RemoteAgent
 from agno.db.base import AsyncBaseDb, BaseDb
 from agno.db.schemas.evals import EvalFilterType, EvalType
 from agno.models.utils import get_model
-from agno.os.auth import get_authentication_dependency
+from agno.os.auth import get_auth_token_from_request, get_authentication_dependency
 from agno.os.routers.evals.schemas import (
     DeleteEvalRunsRequest,
     EvalRunInput,
     EvalSchema,
     UpdateEvalRunRequest,
 )
-from agno.os.routers.evals.utils import run_accuracy_eval, run_performance_eval, run_reliability_eval
+from agno.os.routers.evals.utils import (
+    run_accuracy_eval,
+    run_agent_as_judge_eval,
+    run_performance_eval,
+    run_reliability_eval,
+)
 from agno.os.schema import (
     BadRequestResponse,
     InternalServerErrorResponse,
@@ -28,15 +33,17 @@ from agno.os.schema import (
 )
 from agno.os.settings import AgnoAPISettings
 from agno.os.utils import get_agent_by_id, get_db, get_team_by_id
-from agno.team.team import Team
+from agno.remote.base import RemoteDb
+from agno.team import RemoteTeam, Team
+from agno.utils.log import log_warning
 
 logger = logging.getLogger(__name__)
 
 
 def get_eval_router(
-    dbs: dict[str, list[Union[BaseDb, AsyncBaseDb]]],
-    agents: Optional[List[Agent]] = None,
-    teams: Optional[List[Team]] = None,
+    dbs: dict[str, list[Union[BaseDb, AsyncBaseDb, RemoteDb]]],
+    agents: Optional[List[Union[Agent, RemoteAgent]]] = None,
+    teams: Optional[List[Union[Team, RemoteTeam]]] = None,
     settings: AgnoAPISettings = AgnoAPISettings(),
 ) -> APIRouter:
     """Create eval router with comprehensive OpenAPI documentation for agent/team evaluation endpoints."""
@@ -56,9 +63,9 @@ def get_eval_router(
 
 def attach_routes(
     router: APIRouter,
-    dbs: dict[str, list[Union[BaseDb, AsyncBaseDb]]],
-    agents: Optional[List[Agent]] = None,
-    teams: Optional[List[Team]] = None,
+    dbs: dict[str, list[Union[BaseDb, AsyncBaseDb, RemoteDb]]],
+    agents: Optional[List[Union[Agent, RemoteAgent]]] = None,
+    teams: Optional[List[Union[Team, RemoteTeam]]] = None,
 ) -> APIRouter:
     @router.get(
         "/eval-runs",
@@ -104,14 +111,15 @@ def attach_routes(
         },
     )
     async def get_eval_runs(
+        request: Request,
         agent_id: Optional[str] = Query(default=None, description="Agent ID"),
         team_id: Optional[str] = Query(default=None, description="Team ID"),
         workflow_id: Optional[str] = Query(default=None, description="Workflow ID"),
         model_id: Optional[str] = Query(default=None, description="Model ID"),
         filter_type: Optional[EvalFilterType] = Query(default=None, description="Filter type", alias="type"),
         eval_types: Optional[List[EvalType]] = Depends(parse_eval_types_filter),
-        limit: Optional[int] = Query(default=20, description="Number of eval runs to return"),
-        page: Optional[int] = Query(default=1, description="Page number"),
+        limit: Optional[int] = Query(default=20, description="Number of eval runs to return", ge=1),
+        page: Optional[int] = Query(default=1, description="Page number", ge=0),
         sort_by: Optional[str] = Query(default="created_at", description="Field to sort by"),
         sort_order: Optional[SortOrder] = Query(default="desc", description="Sort order (asc or desc)"),
         db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
@@ -119,6 +127,23 @@ def attach_routes(
     ) -> PaginatedResponse[EvalSchema]:
         db = await get_db(dbs, db_id, table)
 
+        if isinstance(db, RemoteDb):
+            auth_token = get_auth_token_from_request(request)
+            headers = {"Authorization": f"Bearer {auth_token}"} if auth_token else None
+            return await db.get_eval_runs(
+                limit=limit,
+                page=page,
+                sort_by=sort_by,
+                sort_order=sort_order.value if sort_order else None,
+                agent_id=agent_id,
+                team_id=team_id,
+                workflow_id=workflow_id,
+                model_id=model_id,
+                eval_types=eval_types,
+                filter_type=filter_type.value if filter_type else None,
+                headers=headers,
+            )
+
         if isinstance(db, AsyncBaseDb):
             db = cast(AsyncBaseDb, db)
             eval_runs, total_count = await db.get_eval_runs(
@@ -197,11 +222,17 @@ def attach_routes(
         },
     )
     async def get_eval_run(
+        request: Request,
         eval_run_id: str,
         db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
         table: Optional[str] = Query(default=None, description="Table to query eval run from"),
     ) -> EvalSchema:
         db = await get_db(dbs, db_id, table)
+        if isinstance(db, RemoteDb):
+            auth_token = get_auth_token_from_request(request)
+            headers = {"Authorization": f"Bearer {auth_token}"} if auth_token else None
+            return await db.get_eval_run(eval_run_id=eval_run_id, db_id=db_id, table=table, headers=headers)
+
         if isinstance(db, AsyncBaseDb):
             db = cast(AsyncBaseDb, db)
             eval_run = await db.get_eval_run(eval_run_id=eval_run_id, deserialize=False)
@@ -224,12 +255,20 @@ def attach_routes(
         },
     )
     async def delete_eval_runs(
+        http_request: Request,
         request: DeleteEvalRunsRequest,
         db_id: Optional[str] = Query(default=None, description="Database ID to use for deletion"),
         table: Optional[str] = Query(default=None, description="Table to use for deletion"),
     ) -> None:
         try:
             db = await get_db(dbs, db_id, table)
+            if isinstance(db, RemoteDb):
+                auth_token = get_auth_token_from_request(http_request)
+                headers = {"Authorization": f"Bearer {auth_token}"} if auth_token else None
+                return await db.delete_eval_runs(
+                    eval_run_ids=request.eval_run_ids, db_id=db_id, table=table, headers=headers
+                )
+
             if isinstance(db, AsyncBaseDb):
                 db = cast(AsyncBaseDb, db)
                 await db.delete_eval_runs(eval_run_ids=request.eval_run_ids)
@@ -277,6 +316,7 @@ def attach_routes(
         },
     )
     async def update_eval_run(
+        http_request: Request,
         eval_run_id: str,
         request: UpdateEvalRunRequest,
         db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
@@ -284,6 +324,13 @@ def attach_routes(
     ) -> EvalSchema:
         try:
             db = await get_db(dbs, db_id, table)
+            if isinstance(db, RemoteDb):
+                auth_token = get_auth_token_from_request(http_request)
+                headers = {"Authorization": f"Bearer {auth_token}"} if auth_token else None
+                return await db.update_eval_run(
+                    eval_run_id=eval_run_id, name=request.name, db_id=db_id, table=table, headers=headers
+                )
+
             if isinstance(db, AsyncBaseDb):
                 db = cast(AsyncBaseDb, db)
                 eval_run = await db.rename_eval_run(eval_run_id=eval_run_id, name=request.name, deserialize=False)
@@ -304,7 +351,7 @@ def attach_routes(
         operation_id="run_eval",
         summary="Execute Evaluation",
         description=(
-            "Run evaluation tests on agents or teams. Supports accuracy, performance, and reliability evaluations. "
+            "Run evaluation tests on agents or teams. Supports accuracy, agent-as-judge, performance, and reliability evaluations. "
             "Requires either agent_id or team_id, but not both."
         ),
         responses={
@@ -338,11 +385,29 @@ def attach_routes(
         },
     )
     async def run_eval(
+        request: Request,
         eval_run_input: EvalRunInput,
         db_id: Optional[str] = Query(default=None, description="Database ID to use for evaluation"),
         table: Optional[str] = Query(default=None, description="Table to use for evaluation"),
     ) -> Optional[EvalSchema]:
         db = await get_db(dbs, db_id, table)
+        if isinstance(db, RemoteDb):
+            auth_token = get_auth_token_from_request(request)
+            headers = {"Authorization": f"Bearer {auth_token}"} if auth_token else None
+            return await db.create_eval_run(
+                eval_type=eval_run_input.eval_type,
+                input_text=eval_run_input.input,
+                agent_id=eval_run_input.agent_id,
+                team_id=eval_run_input.team_id,
+                model_id=eval_run_input.model_id,
+                model_provider=eval_run_input.model_provider,
+                expected_output=eval_run_input.expected_output,
+                expected_tool_calls=eval_run_input.expected_tool_calls,
+                num_iterations=eval_run_input.num_iterations,
+                db_id=db_id,
+                table=table,
+                headers=headers,
+            )
 
         if eval_run_input.agent_id and eval_run_input.team_id:
             raise HTTPException(status_code=400, detail="Only one of agent_id or team_id must be provided")
@@ -351,6 +416,9 @@ def attach_routes(
             agent = get_agent_by_id(agent_id=eval_run_input.agent_id, agents=agents)
             if not agent:
                 raise HTTPException(status_code=404, detail=f"Agent with id '{eval_run_input.agent_id}' not found")
+            if isinstance(agent, RemoteAgent):
+                log_warning("Evaluation against remote agents are not supported yet")
+                return None
 
             default_model = None
             if (
@@ -373,7 +441,11 @@ def attach_routes(
             team = get_team_by_id(team_id=eval_run_input.team_id, teams=teams)
             if not team:
                 raise HTTPException(status_code=404, detail=f"Team with id '{eval_run_input.team_id}' not found")
+            if isinstance(team, RemoteTeam):
+                log_warning("Evaluation against remote teams are not supported yet")
+                return None
 
+            # If model_id/model_provider specified, override team's model temporarily
             default_model = None
             if (
                 hasattr(team, "model")
@@ -381,13 +453,13 @@ def attach_routes(
                 and eval_run_input.model_id is not None
                 and eval_run_input.model_provider is not None
             ):
-                default_model = deepcopy(team.model)
+                default_model = deepcopy(team.model)  # Save original
                 if eval_run_input.model_id != team.model.id or eval_run_input.model_provider != team.model.provider:
                     model_provider = eval_run_input.model_provider.lower()
                     model_id = eval_run_input.model_id.lower()
                     model_string = f"{model_provider}:{model_id}"
                     model = get_model(model_string)
-                    team.model = model
+                    team.model = model  # Override temporarily
 
             agent = None
 
@@ -396,16 +468,37 @@ def attach_routes(
 
         # Run the evaluation
         if eval_run_input.eval_type == EvalType.ACCURACY:
+            if isinstance(agent, RemoteAgent) or isinstance(team, RemoteTeam):
+                # TODO: Handle remote evaluation
+                log_warning("Evaluation against remote agents are not supported yet")
+                return None
             return await run_accuracy_eval(
                 eval_run_input=eval_run_input, db=db, agent=agent, team=team, default_model=default_model
             )
 
+        elif eval_run_input.eval_type == EvalType.AGENT_AS_JUDGE:
+            return await run_agent_as_judge_eval(
+                eval_run_input=eval_run_input,
+                db=db,
+                agent=agent,
+                team=team,
+                default_model=default_model,  # type: ignore
+            )
+
         elif eval_run_input.eval_type == EvalType.PERFORMANCE:
+            if isinstance(agent, RemoteAgent) or isinstance(team, RemoteTeam):
+                # TODO: Handle remote evaluation
+                log_warning("Evaluation against remote agents are not supported yet")
+                return None
             return await run_performance_eval(
                 eval_run_input=eval_run_input, db=db, agent=agent, team=team, default_model=default_model
             )
 
         else:
+            if isinstance(agent, RemoteAgent) or isinstance(team, RemoteTeam):
+                # TODO: Handle remote evaluation
+                log_warning("Evaluation against remote agents are not supported yet")
+                return None
             return await run_reliability_eval(
                 eval_run_input=eval_run_input, db=db, agent=agent, team=team, default_model=default_model
             )
@@ -416,8 +509,8 @@ def attach_routes(
 def parse_eval_types_filter(
     eval_types: Optional[str] = Query(
         default=None,
-        description="Comma-separated eval types (accuracy,performance,reliability)",
-        examples=["accuracy,performance"],
+        description="Comma-separated eval types (accuracy,agent_as_judge,performance,reliability)",
+        examples=["accuracy,agent_as_judge,performance,reliability"],
     ),
 ) -> Optional[List[EvalType]]:
     """Parse comma-separated eval types into EvalType enums for filtering evaluation runs."""

agno/os/routers/evals/schemas.py

@@ -1,14 +1,16 @@
 from dataclasses import asdict
-from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional
+from datetime import datetime
+from typing import Any, Dict, List, Literal, Optional
 
 from pydantic import BaseModel, Field
 
 from agno.db.schemas.evals import EvalType
-from agno.eval import AccuracyResult, PerformanceResult, ReliabilityResult
+from agno.eval import AccuracyResult, AgentAsJudgeResult, PerformanceResult, ReliabilityResult
 from agno.eval.accuracy import AccuracyEval
+from agno.eval.agent_as_judge import AgentAsJudgeEval
 from agno.eval.performance import PerformanceEval
 from agno.eval.reliability import ReliabilityEval
+from agno.os.utils import to_utc_datetime
 
 
 class EvalRunInput(BaseModel):
@@ -27,6 +29,15 @@ class EvalRunInput(BaseModel):
     # Accuracy eval specific fields
     expected_output: Optional[str] = Field(None, description="Expected output for accuracy evaluation")
 
+    # AgentAsJudge eval specific fields
+    criteria: Optional[str] = Field(None, description="Evaluation criteria for agent-as-judge evaluation")
+    scoring_strategy: Optional[Literal["numeric", "binary"]] = Field(
+        "binary", description="Scoring strategy: 'numeric' (1-10 with threshold) or 'binary' (PASS/FAIL)"
+    )
+    threshold: Optional[int] = Field(
+        7, description="Score threshold for pass/fail (1-10), only used with numeric scoring", ge=1, le=10
+    )
+
     # Performance eval specific fields
     warmup_runs: int = Field(0, description="Number of warmup runs before measuring performance", ge=0, le=10)
 
@@ -64,8 +75,8 @@ class EvalSchema(BaseModel):
             eval_type=eval_run["eval_type"],
             eval_data=eval_run["eval_data"],
             eval_input=eval_run.get("eval_input"),
-            created_at=datetime.fromtimestamp(eval_run["created_at"], tz=timezone.utc),
-            updated_at=datetime.fromtimestamp(eval_run["updated_at"], tz=timezone.utc),
+            created_at=to_utc_datetime(eval_run.get("created_at")),
+            updated_at=to_utc_datetime(eval_run.get("updated_at")),
         )
 
     @classmethod
@@ -89,6 +100,28 @@ class EvalSchema(BaseModel):
             eval_data=asdict(result),
         )
 
+    @classmethod
+    def from_agent_as_judge_eval(
+        cls,
+        agent_as_judge_eval: AgentAsJudgeEval,
+        result: AgentAsJudgeResult,
+        model_id: Optional[str] = None,
+        model_provider: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        team_id: Optional[str] = None,
+    ) -> "EvalSchema":
+        return cls(
+            id=result.run_id,
+            name=agent_as_judge_eval.name,
+            agent_id=agent_id,
+            team_id=team_id,
+            workflow_id=None,
+            model_id=model_id,
+            model_provider=model_provider,
+            eval_type=EvalType.AGENT_AS_JUDGE,
+            eval_data=asdict(result),
+        )
+
     @classmethod
     def from_performance_eval(
         cls,

agno/os/routers/evals/utils.py

@@ -2,14 +2,15 @@ from typing import Optional, Union
 
 from fastapi import HTTPException
 
-from agno.agent.agent import Agent
+from agno.agent import Agent, RemoteAgent
 from agno.db.base import AsyncBaseDb, BaseDb
 from agno.eval.accuracy import AccuracyEval
+from agno.eval.agent_as_judge import AgentAsJudgeEval
 from agno.eval.performance import PerformanceEval
 from agno.eval.reliability import ReliabilityEval
 from agno.models.base import Model
 from agno.os.routers.evals.schemas import EvalRunInput, EvalSchema
-from agno.team.team import Team
+from agno.team import RemoteTeam, Team
 
 
 async def run_accuracy_eval(
@@ -36,12 +37,13 @@ async def run_accuracy_eval(
         model=default_model,
     )
 
-    result = accuracy_eval.run(print_results=False, print_summary=False)
+    result = await accuracy_eval.arun(print_results=False, print_summary=False)
     if not result:
         raise HTTPException(status_code=500, detail="Failed to run accuracy evaluation")
 
     eval_run = EvalSchema.from_accuracy_eval(accuracy_eval=accuracy_eval, result=result)
 
+    # Restore original model after eval
     if default_model is not None:
         if agent is not None:
             agent.model = default_model
@@ -51,6 +53,70 @@ async def run_accuracy_eval(
     return eval_run
 
 
+async def run_agent_as_judge_eval(
+    eval_run_input: EvalRunInput,
+    db: Union[BaseDb, AsyncBaseDb],
+    agent: Optional[Union[Agent, RemoteAgent]] = None,
+    team: Optional[Union[Team, RemoteTeam]] = None,
+    default_model: Optional[Model] = None,
+) -> EvalSchema:
+    """Run an AgentAsJudge evaluation for the given agent or team"""
+    if not eval_run_input.criteria:
+        raise HTTPException(status_code=400, detail="criteria is required for agent-as-judge evaluation")
+
+    # Run agent/team to get output
+    if agent:
+        agent_response = await agent.arun(eval_run_input.input, stream=False)
+        output = str(agent_response.content) if agent_response.content else ""
+        agent_id = agent.id
+        team_id = None
+    elif team:
+        team_response = await team.arun(eval_run_input.input, stream=False)
+        output = str(team_response.content) if team_response.content else ""
+        agent_id = None
+        team_id = team.id
+    else:
+        raise HTTPException(status_code=400, detail="Either agent_id or team_id must be provided")
+
+    agent_as_judge_eval = AgentAsJudgeEval(
+        db=db,
+        criteria=eval_run_input.criteria,
+        scoring_strategy=eval_run_input.scoring_strategy or "binary",
+        threshold=eval_run_input.threshold or 7,
+        additional_guidelines=eval_run_input.additional_guidelines,
+        name=eval_run_input.name,
+        model=default_model,
+    )
+
+    result = await agent_as_judge_eval.arun(
+        input=eval_run_input.input, output=output, print_results=False, print_summary=False
+    )
+    if not result:
+        raise HTTPException(status_code=500, detail="Failed to run agent as judge evaluation")
+
+    # Use evaluator's model
+    eval_model_id = agent_as_judge_eval.model.id if agent_as_judge_eval.model is not None else None
+    eval_model_provider = agent_as_judge_eval.model.provider if agent_as_judge_eval.model is not None else None
+
+    eval_run = EvalSchema.from_agent_as_judge_eval(
+        agent_as_judge_eval=agent_as_judge_eval,
+        result=result,
+        agent_id=agent_id,
+        team_id=team_id,
+        model_id=eval_model_id,
+        model_provider=eval_model_provider,
+    )
+
+    # Restore original model after eval
+    if default_model is not None:
+        if agent is not None and isinstance(agent, Agent):
+            agent.model = default_model
+        elif team is not None and isinstance(team, Team):
+            team.model = default_model
+
+    return eval_run
+
+
 async def run_performance_eval(
     eval_run_input: EvalRunInput,
     db: Union[BaseDb, AsyncBaseDb],
@@ -61,16 +127,16 @@ async def run_performance_eval(
     """Run a performance evaluation for the given agent or team"""
     if agent:
 
-        def run_component():  # type: ignore
-            return agent.run(eval_run_input.input)
+        async def run_component():  # type: ignore
+            return await agent.arun(eval_run_input.input, stream=False)
 
         model_id = agent.model.id if agent and agent.model else None
         model_provider = agent.model.provider if agent and agent.model else None
 
     elif team:
 
-        def run_component():
-            return team.run(eval_run_input.input)
+        async def run_component():  # type: ignore
+            return await team.arun(eval_run_input.input, stream=False)
 
         model_id = team.model.id if team and team.model else None
         model_provider = team.model.provider if team and team.model else None
@@ -86,7 +152,8 @@ async def run_performance_eval(
         model_id=model_id,
         model_provider=model_provider,
     )
-    result = performance_eval.run(print_results=False, print_summary=False)
+
+    result = await performance_eval.arun(print_results=False, print_summary=False)
     if not result:
         raise HTTPException(status_code=500, detail="Failed to run performance evaluation")
 
@@ -99,6 +166,7 @@ async def run_performance_eval(
         model_provider=model_provider,
     )
 
+    # Restore original model after eval
     if default_model is not None:
         if agent is not None:
             agent.model = default_model
@@ -120,7 +188,7 @@ async def run_reliability_eval(
         raise HTTPException(status_code=400, detail="expected_tool_calls is required for reliability evaluations")
 
     if agent:
-        agent_response = agent.run(eval_run_input.input)
+        agent_response = await agent.arun(eval_run_input.input, stream=False)
         reliability_eval = ReliabilityEval(
             db=db,
             name=eval_run_input.name,
@@ -131,7 +199,7 @@ async def run_reliability_eval(
         model_provider = agent.model.provider if agent and agent.model else None
 
     elif team:
-        team_response = team.run(eval_run_input.input)
+        team_response = await team.arun(eval_run_input.input, stream=False)
         reliability_eval = ReliabilityEval(
             db=db,
             name=eval_run_input.name,
@@ -141,7 +209,7 @@ async def run_reliability_eval(
         model_id = team.model.id if team and team.model else None
         model_provider = team.model.provider if team and team.model else None
 
-    result = reliability_eval.run(print_results=False)
+    result = await reliability_eval.arun(print_results=False)
     if not result:
         raise HTTPException(status_code=500, detail="Failed to run reliability evaluation")
 
@@ -153,6 +221,7 @@ async def run_reliability_eval(
         model_provider=model_provider,
     )
 
+    # Restore original model after eval
     if default_model is not None:
         if agent is not None:
             agent.model = default_model

agno/os/routers/health.py

@@ -8,7 +8,7 @@ from agno.os.schema import HealthResponse
 def get_health_router(health_endpoint: str = "/health") -> APIRouter:
     router = APIRouter(tags=["Health"])
 
-    started_time_stamp = datetime.now(timezone.utc).timestamp()
+    started_at = datetime.now(timezone.utc)
 
     @router.get(
         health_endpoint,
@@ -20,12 +20,12 @@ def get_health_router(health_endpoint: str = "/health") -> APIRouter:
             200: {
                 "description": "API is healthy and operational",
                 "content": {
-                    "application/json": {"example": {"status": "ok", "instantiated_at": str(started_time_stamp)}}
+                    "application/json": {"example": {"status": "ok", "instantiated_at": "2025-06-10T12:00:00Z"}}
                 },
             }
         },
     )
     async def health_check() -> HealthResponse:
-        return HealthResponse(status="ok", instantiated_at=str(started_time_stamp))
+        return HealthResponse(status="ok", instantiated_at=started_at)
 
     return router