agno 1.8.0__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

agno/os/routers/__init__.py

@@ -0,0 +1,3 @@
+from agno.os.routers.session.session import get_session_router
+
+__all__ = ["get_session_router"]

agno/os/routers/evals/__init__.py

@@ -0,0 +1,3 @@
+from agno.os.routers.evals.evals import get_eval_router
+
+__all__ = ["get_eval_router"]

agno/os/routers/evals/evals.py

@@ -0,0 +1,204 @@
+import logging
+from copy import deepcopy
+from typing import List, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+
+from agno.agent.agent import Agent
+from agno.db.base import BaseDb
+from agno.db.schemas.evals import EvalFilterType, EvalType
+from agno.models.utils import get_model
+from agno.os.auth import get_authentication_dependency
+from agno.os.routers.evals.schemas import (
+    DeleteEvalRunsRequest,
+    EvalRunInput,
+    EvalSchema,
+    UpdateEvalRunRequest,
+)
+from agno.os.routers.evals.utils import run_accuracy_eval, run_performance_eval, run_reliability_eval
+from agno.os.schema import PaginatedResponse, PaginationInfo, SortOrder
+from agno.os.settings import AgnoAPISettings
+from agno.os.utils import get_agent_by_id, get_db, get_team_by_id
+from agno.team.team import Team
+
+logger = logging.getLogger(__name__)
+
+
+def get_eval_router(
+    dbs: dict[str, BaseDb],
+    agents: Optional[List[Agent]] = None,
+    teams: Optional[List[Team]] = None,
+    settings: AgnoAPISettings = AgnoAPISettings(),
+) -> APIRouter:
+    router = APIRouter(dependencies=[Depends(get_authentication_dependency(settings))], tags=["Evals"])
+    return attach_routes(router=router, dbs=dbs, agents=agents, teams=teams)
+
+
+def attach_routes(
+    router: APIRouter, dbs: dict[str, BaseDb], agents: Optional[List[Agent]] = None, teams: Optional[List[Team]] = None
+) -> APIRouter:
+    @router.get("/eval-runs", response_model=PaginatedResponse[EvalSchema], status_code=200)
+    async def get_eval_runs(
+        agent_id: Optional[str] = Query(default=None, description="Agent ID"),
+        team_id: Optional[str] = Query(default=None, description="Team ID"),
+        workflow_id: Optional[str] = Query(default=None, description="Workflow ID"),
+        model_id: Optional[str] = Query(default=None, description="Model ID"),
+        filter_type: Optional[EvalFilterType] = Query(default=None, description="Filter type", alias="type"),
+        eval_types: Optional[List[EvalType]] = Depends(parse_eval_types_filter),
+        limit: Optional[int] = Query(default=20, description="Number of eval runs to return"),
+        page: Optional[int] = Query(default=1, description="Page number"),
+        sort_by: Optional[str] = Query(default="created_at", description="Field to sort by"),
+        sort_order: Optional[SortOrder] = Query(default="desc", description="Sort order (asc or desc)"),
+        db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
+    ) -> PaginatedResponse[EvalSchema]:
+        db = get_db(dbs, db_id)
+        eval_runs, total_count = db.get_eval_runs(
+            limit=limit,
+            page=page,
+            sort_by=sort_by,
+            sort_order=sort_order,
+            agent_id=agent_id,
+            team_id=team_id,
+            workflow_id=workflow_id,
+            model_id=model_id,
+            eval_type=eval_types,
+            filter_type=filter_type,
+            deserialize=False,
+        )
+
+        return PaginatedResponse(
+            data=[EvalSchema.from_dict(eval_run) for eval_run in eval_runs],  # type: ignore
+            meta=PaginationInfo(
+                page=page,
+                limit=limit,
+                total_count=total_count,  # type: ignore
+                total_pages=(total_count + limit - 1) // limit if limit is not None and limit > 0 else 0,  # type: ignore
+            ),
+        )
+
+    @router.get("/eval-runs/{eval_run_id}", response_model=EvalSchema, status_code=200)
+    async def get_eval_run(
+        eval_run_id: str,
+        db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
+    ) -> EvalSchema:
+        db = get_db(dbs, db_id)
+        eval_run = db.get_eval_run(eval_run_id=eval_run_id, deserialize=False)
+        if not eval_run:
+            raise HTTPException(status_code=404, detail=f"Eval run with id '{eval_run_id}' not found")
+
+        return EvalSchema.from_dict(eval_run)  # type: ignore
+
+    @router.delete("/eval-runs", status_code=204)
+    async def delete_eval_runs(
+        request: DeleteEvalRunsRequest,
+        db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
+    ) -> None:
+        try:
+            db = get_db(dbs, db_id)
+            db.delete_eval_runs(eval_run_ids=request.eval_run_ids)
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Failed to delete eval runs: {e}")
+
+    @router.patch("/eval-runs/{eval_run_id}", response_model=EvalSchema, status_code=200)
+    async def update_eval_run(
+        eval_run_id: str,
+        request: UpdateEvalRunRequest,
+        db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
+    ) -> EvalSchema:
+        try:
+            db = get_db(dbs, db_id)
+            eval_run = db.rename_eval_run(eval_run_id=eval_run_id, name=request.name, deserialize=False)
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=f"Failed to rename eval run: {e}")
+
+        if not eval_run:
+            raise HTTPException(status_code=404, detail=f"Eval run with id '{eval_run_id}' not found")
+
+        return EvalSchema.from_dict(eval_run)  # type: ignore
+
+    @router.post("/eval-runs", response_model=EvalSchema, status_code=200)
+    async def run_eval(
+        eval_run_input: EvalRunInput,
+        db_id: Optional[str] = Query(default=None, description="The ID of the database to use"),
+    ) -> Optional[EvalSchema]:
+        db = get_db(dbs, db_id)
+
+        if eval_run_input.agent_id and eval_run_input.team_id:
+            raise HTTPException(status_code=400, detail="Only one of agent_id or team_id must be provided")
+
+        if eval_run_input.agent_id:
+            agent = get_agent_by_id(agent_id=eval_run_input.agent_id, agents=agents)
+            if not agent:
+                raise HTTPException(status_code=404, detail=f"Agent with id '{eval_run_input.agent_id}' not found")
+
+            default_model = None
+            if (
+                hasattr(agent, "model")
+                and agent.model is not None
+                and eval_run_input.model_id is not None
+                and eval_run_input.model_provider is not None
+            ):
+                default_model = deepcopy(agent.model)
+                if eval_run_input.model_id != agent.model.id or eval_run_input.model_provider != agent.model.provider:
+                    model = get_model(
+                        model_id=eval_run_input.model_id.lower(),
+                        model_provider=eval_run_input.model_provider.lower(),
+                    )
+                    agent.model = model
+
+            team = None
+
+        elif eval_run_input.team_id:
+            team = get_team_by_id(team_id=eval_run_input.team_id, teams=teams)
+            if not team:
+                raise HTTPException(status_code=404, detail=f"Team with id '{eval_run_input.team_id}' not found")
+
+            default_model = None
+            if (
+                hasattr(team, "model")
+                and team.model is not None
+                and eval_run_input.model_id is not None
+                and eval_run_input.model_provider is not None
+            ):
+                default_model = deepcopy(team.model)
+                if eval_run_input.model_id != team.model.id or eval_run_input.model_provider != team.model.provider:
+                    model = get_model(
+                        model_id=eval_run_input.model_id.lower(),
+                        model_provider=eval_run_input.model_provider.lower(),
+                    )
+                    team.model = model
+
+            agent = None
+
+        else:
+            raise HTTPException(status_code=400, detail="One of agent_id or team_id must be provided")
+
+        # Run the evaluation
+        if eval_run_input.eval_type == EvalType.ACCURACY:
+            return await run_accuracy_eval(
+                eval_run_input=eval_run_input, db=db, agent=agent, team=team, default_model=default_model
+            )
+
+        elif eval_run_input.eval_type == EvalType.PERFORMANCE:
+            return await run_performance_eval(
+                eval_run_input=eval_run_input, db=db, agent=agent, team=team, default_model=default_model
+            )
+
+        else:
+            return await run_reliability_eval(
+                eval_run_input=eval_run_input, db=db, agent=agent, team=team, default_model=default_model
+            )
+
+    return router
+
+
+def parse_eval_types_filter(
+    eval_types: Optional[str] = Query(default=None, description="Comma-separated eval types"),
+) -> Optional[List[EvalType]]:
+    """Parse a comma-separated string of eval types into a list of EvalType enums"""
+    if not eval_types:
+        return None
+    try:
+        return [EvalType(item.strip()) for item in eval_types.split(",")]
+    except ValueError as e:
+        raise HTTPException(status_code=422, detail=f"Invalid eval_type: {e}")

agno/os/routers/evals/schemas.py

@@ -0,0 +1,142 @@
+from dataclasses import asdict
+from datetime import datetime, timezone
+from typing import Any, Dict, List, Optional
+
+from pydantic import BaseModel
+
+from agno.db.schemas.evals import EvalType
+from agno.eval import AccuracyResult, PerformanceResult, ReliabilityResult
+from agno.eval.accuracy import AccuracyEval
+from agno.eval.performance import PerformanceEval
+from agno.eval.reliability import ReliabilityEval
+
+
+class EvalRunInput(BaseModel):
+    agent_id: Optional[str] = None
+    team_id: Optional[str] = None
+
+    model_id: Optional[str] = None
+    model_provider: Optional[str] = None
+    eval_type: EvalType
+    input: str
+    additional_guidelines: Optional[str] = None
+    additional_context: Optional[str] = None
+    num_iterations: Optional[int] = 1
+    name: Optional[str] = None
+
+    # Accuracy eval specific fields
+    expected_output: Optional[str] = None
+
+    # Performance eval specific fields
+    warmup_runs: Optional[int] = 0
+
+    # Reliability eval specific fields
+    expected_tool_calls: Optional[List[str]] = None
+
+
+class EvalSchema(BaseModel):
+    id: str
+
+    agent_id: Optional[str] = None
+    model_id: Optional[str] = None
+    model_provider: Optional[str] = None
+    team_id: Optional[str] = None
+    workflow_id: Optional[str] = None
+    name: Optional[str] = None
+    evaluated_component_name: Optional[str] = None
+    eval_type: EvalType
+    eval_data: Dict[str, Any]
+    eval_input: Optional[Dict[str, Any]] = None
+    created_at: Optional[datetime] = None
+    updated_at: Optional[datetime] = None
+
+    @classmethod
+    def from_dict(cls, eval_run: Dict[str, Any]) -> "EvalSchema":
+        return cls(
+            id=eval_run["run_id"],
+            name=eval_run.get("name"),
+            agent_id=eval_run.get("agent_id"),
+            model_id=eval_run.get("model_id"),
+            model_provider=eval_run.get("model_provider"),
+            team_id=eval_run.get("team_id"),
+            workflow_id=eval_run.get("workflow_id"),
+            evaluated_component_name=eval_run.get("evaluated_component_name"),
+            eval_type=eval_run["eval_type"],
+            eval_data=eval_run["eval_data"],
+            eval_input=eval_run.get("eval_input"),
+            created_at=datetime.fromtimestamp(eval_run["created_at"], tz=timezone.utc),
+            updated_at=datetime.fromtimestamp(eval_run["updated_at"], tz=timezone.utc),
+        )
+
+    @classmethod
+    def from_accuracy_eval(cls, accuracy_eval: AccuracyEval, result: AccuracyResult) -> "EvalSchema":
+        model_provider = (
+            accuracy_eval.agent.model.provider
+            if accuracy_eval.agent and accuracy_eval.agent.model
+            else accuracy_eval.team.model.provider
+            if accuracy_eval.team and accuracy_eval.team.model
+            else None
+        )
+        return cls(
+            id=accuracy_eval.eval_id,
+            name=accuracy_eval.name,
+            agent_id=accuracy_eval.agent.id if accuracy_eval.agent else None,
+            team_id=accuracy_eval.team.id if accuracy_eval.team else None,
+            workflow_id=None,
+            model_id=accuracy_eval.agent.model.id if accuracy_eval.agent else accuracy_eval.team.model.id,  # type: ignore
+            model_provider=model_provider,
+            eval_type=EvalType.ACCURACY,
+            eval_data=asdict(result),
+        )
+
+    @classmethod
+    def from_performance_eval(
+        cls,
+        performance_eval: PerformanceEval,
+        result: PerformanceResult,
+        model_id: Optional[str] = None,
+        model_provider: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        team_id: Optional[str] = None,
+    ) -> "EvalSchema":
+        return cls(
+            id=performance_eval.eval_id,
+            name=performance_eval.name,
+            agent_id=agent_id,
+            team_id=team_id,
+            workflow_id=None,
+            model_id=model_id,
+            model_provider=model_provider,
+            eval_type=EvalType.PERFORMANCE,
+            eval_data=asdict(result),
+        )
+
+    @classmethod
+    def from_reliability_eval(
+        cls,
+        reliability_eval: ReliabilityEval,
+        result: ReliabilityResult,
+        model_id: Optional[str] = None,
+        model_provider: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        team_id: Optional[str] = None,
+    ) -> "EvalSchema":
+        return cls(
+            id=reliability_eval.eval_id,
+            name=reliability_eval.name,
+            agent_id=agent_id,
+            team_id=team_id,
+            workflow_id=None,
+            model_id=model_id,
+            model_provider=model_provider,
+            eval_type=EvalType.RELIABILITY,
+            eval_data=asdict(result),
+        )
+
+
+class DeleteEvalRunsRequest(BaseModel):
+    eval_run_ids: List[str]
+
+
+class UpdateEvalRunRequest(BaseModel):
+    name: str

agno/os/routers/evals/utils.py

@@ -0,0 +1,161 @@
+from typing import Optional
+
+from fastapi import HTTPException
+
+from agno.agent.agent import Agent
+from agno.db.base import BaseDb
+from agno.eval.accuracy import AccuracyEval
+from agno.eval.performance import PerformanceEval
+from agno.eval.reliability import ReliabilityEval
+from agno.models.base import Model
+from agno.os.routers.evals.schemas import EvalRunInput, EvalSchema
+from agno.team.team import Team
+
+
+async def run_accuracy_eval(
+    eval_run_input: EvalRunInput,
+    db: BaseDb,
+    agent: Optional[Agent] = None,
+    team: Optional[Team] = None,
+    default_model: Optional[Model] = None,
+) -> EvalSchema:
+    """Run an Accuracy evaluation for the given agent or team"""
+    if not eval_run_input.expected_output:
+        raise HTTPException(status_code=400, detail="expected_output is required for accuracy evaluation")
+
+    accuracy_eval = AccuracyEval(
+        db=db,
+        agent=agent,
+        team=team,
+        input=eval_run_input.input,
+        expected_output=eval_run_input.expected_output,
+        additional_guidelines=eval_run_input.additional_guidelines,
+        additional_context=eval_run_input.additional_context,
+        num_iterations=eval_run_input.num_iterations or 1,
+        name=eval_run_input.name,
+    )
+
+    result = accuracy_eval.run(print_results=False, print_summary=False)
+    if not result:
+        raise HTTPException(status_code=500, detail="Failed to run accuracy evaluation")
+
+    eval_run = EvalSchema.from_accuracy_eval(accuracy_eval=accuracy_eval, result=result)
+
+    if default_model is not None:
+        if agent is not None:
+            agent.model = default_model
+        elif team is not None:
+            team.model = default_model
+
+    return eval_run
+
+
+async def run_performance_eval(
+    eval_run_input: EvalRunInput,
+    db: BaseDb,
+    agent: Optional[Agent] = None,
+    team: Optional[Team] = None,
+    default_model: Optional[Model] = None,
+) -> EvalSchema:
+    """Run a performance evaluation for the given agent or team"""
+    if agent:
+
+        def run_component():  # type: ignore
+            return agent.run(eval_run_input.input)
+
+        model_id = agent.model.id if agent and agent.model else None
+        model_provider = agent.model.provider if agent and agent.model else None
+
+    elif team:
+
+        def run_component():
+            return team.run(eval_run_input.input)
+
+        model_id = team.model.id if team and team.model else None
+        model_provider = team.model.provider if team and team.model else None
+
+    performance_eval = PerformanceEval(
+        db=db,
+        name=eval_run_input.name,
+        func=run_component,
+        num_iterations=eval_run_input.num_iterations or 10,
+        warmup_runs=eval_run_input.warmup_runs,
+        agent_id=agent.id if agent else None,
+        team_id=team.id if team else None,
+        model_id=model_id,
+        model_provider=model_provider,
+    )
+    result = performance_eval.run(print_results=False, print_summary=False)
+    if not result:
+        raise HTTPException(status_code=500, detail="Failed to run performance evaluation")
+
+    eval_run = EvalSchema.from_performance_eval(
+        performance_eval=performance_eval,
+        result=result,
+        agent_id=agent.id if agent else None,
+        team_id=team.id if team else None,
+        model_id=model_id,
+        model_provider=model_provider,
+    )
+
+    if default_model is not None:
+        if agent is not None:
+            agent.model = default_model
+        elif team is not None:
+            team.model = default_model
+
+    return eval_run
+
+
+async def run_reliability_eval(
+    eval_run_input: EvalRunInput,
+    db: BaseDb,
+    agent: Optional[Agent] = None,
+    team: Optional[Team] = None,
+    default_model: Optional[Model] = None,
+) -> EvalSchema:
+    """Run a reliability evaluation for the given agent or team"""
+    if not eval_run_input.expected_tool_calls:
+        raise HTTPException(status_code=400, detail="expected_tool_calls is required for reliability evaluations")
+
+    if agent:
+        agent_response = agent.run(eval_run_input.input)
+        reliability_eval = ReliabilityEval(
+            db=db,
+            name=eval_run_input.name,
+            agent_response=agent_response,
+            expected_tool_calls=eval_run_input.expected_tool_calls,
+        )
+        model_id = agent.model.id if agent and agent.model else None
+        model_provider = agent.model.provider if agent and agent.model else None
+
+    elif team:
+        team_response = team.run(eval_run_input.input)
+        reliability_eval = ReliabilityEval(
+            db=db,
+            name=eval_run_input.name,
+            team_response=team_response,
+            expected_tool_calls=eval_run_input.expected_tool_calls,
+        )
+        model_id = team.model.id if team and team.model else None
+        model_provider = team.model.provider if team and team.model else None
+
+    result = reliability_eval.run(print_results=False)
+    if not result:
+        raise HTTPException(status_code=500, detail="Failed to run reliability evaluation")
+
+    eval_run = EvalSchema.from_reliability_eval(
+        reliability_eval=reliability_eval,
+        result=result,
+        agent_id=agent.id if agent else None,
+        model_id=model_id,
+        model_provider=model_provider,
+    )
+
+    if default_model is not None:
+        if agent is not None:
+            agent.model = default_model
+        elif team is not None:
+            team.model = default_model
+
+    return eval_run

agno/os/routers/knowledge/__init__.py

@@ -0,0 +1,3 @@
+from agno.os.routers.knowledge.knowledge import get_knowledge_router
+
+__all__ = ["get_knowledge_router"]