agno 2.3.8__py3-none-any.whl → 2.3.9__py3-none-any.whl

agno/os/routers/agents/schema.py

@@ -0,0 +1,261 @@
+from typing import Any, Dict, Optional
+from uuid import uuid4
+
+from pydantic import BaseModel
+
+from agno.agent import Agent
+from agno.models.message import Message
+from agno.os.schema import ModelResponse
+from agno.os.utils import (
+    format_tools,
+    get_agent_input_schema_dict,
+)
+from agno.run import RunContext
+from agno.run.agent import RunOutput
+from agno.session import AgentSession
+from agno.utils.agent import aexecute_instructions, aexecute_system_message
+
+
+class AgentResponse(BaseModel):
+    id: Optional[str] = None
+    name: Optional[str] = None
+    db_id: Optional[str] = None
+    model: Optional[ModelResponse] = None
+    tools: Optional[Dict[str, Any]] = None
+    sessions: Optional[Dict[str, Any]] = None
+    knowledge: Optional[Dict[str, Any]] = None
+    memory: Optional[Dict[str, Any]] = None
+    reasoning: Optional[Dict[str, Any]] = None
+    default_tools: Optional[Dict[str, Any]] = None
+    system_message: Optional[Dict[str, Any]] = None
+    extra_messages: Optional[Dict[str, Any]] = None
+    response_settings: Optional[Dict[str, Any]] = None
+    introduction: Optional[str] = None
+    streaming: Optional[Dict[str, Any]] = None
+    metadata: Optional[Dict[str, Any]] = None
+    input_schema: Optional[Dict[str, Any]] = None
+
+    @classmethod
+    async def from_agent(cls, agent: Agent) -> "AgentResponse":
+        def filter_meaningful_config(d: Dict[str, Any], defaults: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+            """Filter out fields that match their default values, keeping only meaningful user configurations"""
+            filtered = {}
+            for key, value in d.items():
+                if value is None:
+                    continue
+                # Skip if value matches the default exactly
+                if key in defaults and value == defaults[key]:
+                    continue
+                # Keep non-default values
+                filtered[key] = value
+            return filtered if filtered else None
+
+        # Define default values for filtering
+        agent_defaults = {
+            # Sessions defaults
+            "add_history_to_context": False,
+            "num_history_runs": 3,
+            "enable_session_summaries": False,
+            "search_session_history": False,
+            "cache_session": False,
+            # Knowledge defaults
+            "add_references": False,
+            "references_format": "json",
+            "enable_agentic_knowledge_filters": False,
+            # Memory defaults
+            "enable_agentic_memory": False,
+            "enable_user_memories": False,
+            # Reasoning defaults
+            "reasoning": False,
+            "reasoning_min_steps": 1,
+            "reasoning_max_steps": 10,
+            # Default tools defaults
+            "read_chat_history": False,
+            "search_knowledge": True,
+            "update_knowledge": False,
+            "read_tool_call_history": False,
+            # System message defaults
+            "system_message_role": "system",
+            "build_context": True,
+            "markdown": False,
+            "add_name_to_context": False,
+            "add_datetime_to_context": False,
+            "add_location_to_context": False,
+            "resolve_in_context": True,
+            # Extra messages defaults
+            "user_message_role": "user",
+            "build_user_context": True,
+            # Response settings defaults
+            "retries": 0,
+            "delay_between_retries": 1,
+            "exponential_backoff": False,
+            "parse_response": True,
+            "use_json_mode": False,
+            # Streaming defaults
+            "stream_events": False,
+            "stream_intermediate_steps": False,
+        }
+
+        session_id = str(uuid4())
+        run_id = str(uuid4())
+        agent_tools = await agent.aget_tools(
+            session=AgentSession(session_id=session_id, session_data={}),
+            run_response=RunOutput(run_id=run_id, session_id=session_id),
+            run_context=RunContext(run_id=run_id, session_id=session_id, user_id=agent.user_id),
+            check_mcp_tools=False,
+        )
+        formatted_tools = format_tools(agent_tools) if agent_tools else None
+
+        additional_input = agent.additional_input
+        if additional_input and isinstance(additional_input[0], Message):
+            additional_input = [message.to_dict() for message in additional_input]  # type: ignore
+
+        # Build model only if it has at least one non-null field
+        model_name = agent.model.name if (agent.model and agent.model.name) else None
+        model_provider = agent.model.provider if (agent.model and agent.model.provider) else None
+        model_id = agent.model.id if (agent.model and agent.model.id) else None
+        _agent_model_data: Dict[str, Any] = {}
+        if model_name is not None:
+            _agent_model_data["name"] = model_name
+        if model_id is not None:
+            _agent_model_data["model"] = model_id
+        if model_provider is not None:
+            _agent_model_data["provider"] = model_provider
+
+        session_table = agent.db.session_table_name if agent.db else None
+        knowledge_table = agent.db.knowledge_table_name if agent.db and agent.knowledge else None
+
+        tools_info = {
+            "tools": formatted_tools,
+            "tool_call_limit": agent.tool_call_limit,
+            "tool_choice": agent.tool_choice,
+        }
+
+        sessions_info = {
+            "session_table": session_table,
+            "add_history_to_context": agent.add_history_to_context,
+            "enable_session_summaries": agent.enable_session_summaries,
+            "num_history_runs": agent.num_history_runs,
+            "search_session_history": agent.search_session_history,
+            "num_history_sessions": agent.num_history_sessions,
+            "cache_session": agent.cache_session,
+        }
+
+        knowledge_info = {
+            "knowledge_table": knowledge_table,
+            "enable_agentic_knowledge_filters": agent.enable_agentic_knowledge_filters,
+            "knowledge_filters": agent.knowledge_filters,
+            "references_format": agent.references_format,
+        }
+
+        memory_info: Optional[Dict[str, Any]] = None
+        if agent.memory_manager is not None:
+            memory_info = {
+                "enable_agentic_memory": agent.enable_agentic_memory,
+                "enable_user_memories": agent.enable_user_memories,
+                "metadata": agent.metadata,
+                "memory_table": agent.db.memory_table_name if agent.db and agent.enable_user_memories else None,
+            }
+
+            if agent.memory_manager.model is not None:
+                memory_info["model"] = ModelResponse(
+                    name=agent.memory_manager.model.name,
+                    model=agent.memory_manager.model.id,
+                    provider=agent.memory_manager.model.provider,
+                ).model_dump()
+
+        reasoning_info: Dict[str, Any] = {
+            "reasoning": agent.reasoning,
+            "reasoning_agent_id": agent.reasoning_agent.id if agent.reasoning_agent else None,
+            "reasoning_min_steps": agent.reasoning_min_steps,
+            "reasoning_max_steps": agent.reasoning_max_steps,
+        }
+
+        if agent.reasoning_model:
+            reasoning_info["reasoning_model"] = ModelResponse(
+                name=agent.reasoning_model.name,
+                model=agent.reasoning_model.id,
+                provider=agent.reasoning_model.provider,
+            ).model_dump()
+
+        default_tools_info = {
+            "read_chat_history": agent.read_chat_history,
+            "search_knowledge": agent.search_knowledge,
+            "update_knowledge": agent.update_knowledge,
+            "read_tool_call_history": agent.read_tool_call_history,
+        }
+
+        instructions = agent.instructions if agent.instructions else None
+        if instructions and callable(instructions):
+            instructions = await aexecute_instructions(instructions=instructions, agent=agent)
+
+        system_message = agent.system_message if agent.system_message else None
+        if system_message and callable(system_message):
+            system_message = await aexecute_system_message(system_message=system_message, agent=agent)
+
+        system_message_info = {
+            "system_message": str(system_message) if system_message else None,
+            "system_message_role": agent.system_message_role,
+            "build_context": agent.build_context,
+            "description": agent.description,
+            "instructions": instructions,
+            "expected_output": agent.expected_output,
+            "additional_context": agent.additional_context,
+            "markdown": agent.markdown,
+            "add_name_to_context": agent.add_name_to_context,
+            "add_datetime_to_context": agent.add_datetime_to_context,
+            "add_location_to_context": agent.add_location_to_context,
+            "timezone_identifier": agent.timezone_identifier,
+            "resolve_in_context": agent.resolve_in_context,
+        }
+
+        extra_messages_info = {
+            "additional_input": additional_input,  # type: ignore
+            "user_message_role": agent.user_message_role,
+            "build_user_context": agent.build_user_context,
+        }
+
+        response_settings_info: Dict[str, Any] = {
+            "retries": agent.retries,
+            "delay_between_retries": agent.delay_between_retries,
+            "exponential_backoff": agent.exponential_backoff,
+            "output_schema_name": agent.output_schema.__name__ if agent.output_schema else None,
+            "parser_model_prompt": agent.parser_model_prompt,
+            "parse_response": agent.parse_response,
+            "structured_outputs": agent.structured_outputs,
+            "use_json_mode": agent.use_json_mode,
+            "save_response_to_file": agent.save_response_to_file,
+        }
+
+        if agent.parser_model:
+            response_settings_info["parser_model"] = ModelResponse(
+                name=agent.parser_model.name,
+                model=agent.parser_model.id,
+                provider=agent.parser_model.provider,
+            ).model_dump()
+
+        streaming_info = {
+            "stream": agent.stream,
+            "stream_events": agent.stream_events,
+            "stream_intermediate_steps": agent.stream_intermediate_steps,
+        }
+
+        return AgentResponse(
+            id=agent.id,
+            name=agent.name,
+            db_id=agent.db.id if agent.db else None,
+            model=ModelResponse(**_agent_model_data) if _agent_model_data else None,
+            tools=filter_meaningful_config(tools_info, {}),
+            sessions=filter_meaningful_config(sessions_info, agent_defaults),
+            knowledge=filter_meaningful_config(knowledge_info, agent_defaults),
+            memory=filter_meaningful_config(memory_info, agent_defaults) if memory_info else None,
+            reasoning=filter_meaningful_config(reasoning_info, agent_defaults),
+            default_tools=filter_meaningful_config(default_tools_info, agent_defaults),
+            system_message=filter_meaningful_config(system_message_info, agent_defaults),
+            extra_messages=filter_meaningful_config(extra_messages_info, agent_defaults),
+            response_settings=filter_meaningful_config(response_settings_info, agent_defaults),
+            streaming=filter_meaningful_config(streaming_info, agent_defaults),
+            introduction=agent.introduction,
+            metadata=agent.metadata,
+            input_schema=get_agent_input_schema_dict(agent),
+        )

agno/os/routers/evals/evals.py

@@ -15,7 +15,12 @@ from agno.os.routers.evals.schemas import (
     EvalSchema,
     UpdateEvalRunRequest,
 )
-from agno.os.routers.evals.utils import run_accuracy_eval, run_performance_eval, run_reliability_eval
+from agno.os.routers.evals.utils import (
+    run_accuracy_eval,
+    run_agent_as_judge_eval,
+    run_performance_eval,
+    run_reliability_eval,
+)
 from agno.os.schema import (
     BadRequestResponse,
     InternalServerErrorResponse,
@@ -119,6 +124,15 @@ def attach_routes(
     ) -> PaginatedResponse[EvalSchema]:
         db = await get_db(dbs, db_id, table)
 
+        # TODO: Delete me:
+        # Filtering out agent-as-judge by default for now,
+        # as they are not supported yet in the AgentOS UI.
+        eval_types = eval_types or [
+            EvalType.ACCURACY,
+            EvalType.PERFORMANCE,
+            EvalType.RELIABILITY,
+        ]
+
         if isinstance(db, AsyncBaseDb):
             db = cast(AsyncBaseDb, db)
             eval_runs, total_count = await db.get_eval_runs(
@@ -304,7 +318,7 @@ def attach_routes(
         operation_id="run_eval",
         summary="Execute Evaluation",
         description=(
-            "Run evaluation tests on agents or teams. Supports accuracy, performance, and reliability evaluations. "
+            "Run evaluation tests on agents or teams. Supports accuracy, agent-as-judge, performance, and reliability evaluations. "
             "Requires either agent_id or team_id, but not both."
         ),
         responses={
@@ -374,6 +388,7 @@ def attach_routes(
             if not team:
                 raise HTTPException(status_code=404, detail=f"Team with id '{eval_run_input.team_id}' not found")
 
+            # If model_id/model_provider specified, override team's model temporarily
             default_model = None
             if (
                 hasattr(team, "model")
@@ -381,13 +396,13 @@ def attach_routes(
                 and eval_run_input.model_id is not None
                 and eval_run_input.model_provider is not None
             ):
-                default_model = deepcopy(team.model)
+                default_model = deepcopy(team.model)  # Save original
                 if eval_run_input.model_id != team.model.id or eval_run_input.model_provider != team.model.provider:
                     model_provider = eval_run_input.model_provider.lower()
                     model_id = eval_run_input.model_id.lower()
                     model_string = f"{model_provider}:{model_id}"
                     model = get_model(model_string)
-                    team.model = model
+                    team.model = model  # Override temporarily
 
             agent = None
 
@@ -400,6 +415,11 @@ def attach_routes(
                 eval_run_input=eval_run_input, db=db, agent=agent, team=team, default_model=default_model
             )
 
+        elif eval_run_input.eval_type == EvalType.AGENT_AS_JUDGE:
+            return await run_agent_as_judge_eval(
+                eval_run_input=eval_run_input, db=db, agent=agent, team=team, default_model=default_model
+            )
+
         elif eval_run_input.eval_type == EvalType.PERFORMANCE:
             return await run_performance_eval(
                 eval_run_input=eval_run_input, db=db, agent=agent, team=team, default_model=default_model
@@ -416,8 +436,8 @@ def attach_routes(
 def parse_eval_types_filter(
     eval_types: Optional[str] = Query(
         default=None,
-        description="Comma-separated eval types (accuracy,performance,reliability)",
-        examples=["accuracy,performance"],
+        description="Comma-separated eval types (accuracy,agent_as_judge,performance,reliability)",
+        examples=["accuracy,agent_as_judge,performance,reliability"],
     ),
 ) -> Optional[List[EvalType]]:
     """Parse comma-separated eval types into EvalType enums for filtering evaluation runs."""

agno/os/routers/evals/schemas.py

@@ -1,12 +1,13 @@
 from dataclasses import asdict
 from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Literal, Optional
 
 from pydantic import BaseModel, Field
 
 from agno.db.schemas.evals import EvalType
-from agno.eval import AccuracyResult, PerformanceResult, ReliabilityResult
+from agno.eval import AccuracyResult, AgentAsJudgeResult, PerformanceResult, ReliabilityResult
 from agno.eval.accuracy import AccuracyEval
+from agno.eval.agent_as_judge import AgentAsJudgeEval
 from agno.eval.performance import PerformanceEval
 from agno.eval.reliability import ReliabilityEval
 
@@ -27,6 +28,15 @@ class EvalRunInput(BaseModel):
     # Accuracy eval specific fields
     expected_output: Optional[str] = Field(None, description="Expected output for accuracy evaluation")
 
+    # AgentAsJudge eval specific fields
+    criteria: Optional[str] = Field(None, description="Evaluation criteria for agent-as-judge evaluation")
+    scoring_strategy: Optional[Literal["numeric", "binary"]] = Field(
+        "binary", description="Scoring strategy: 'numeric' (1-10 with threshold) or 'binary' (PASS/FAIL)"
+    )
+    threshold: Optional[int] = Field(
+        7, description="Score threshold for pass/fail (1-10), only used with numeric scoring", ge=1, le=10
+    )
+
     # Performance eval specific fields
     warmup_runs: int = Field(0, description="Number of warmup runs before measuring performance", ge=0, le=10)
 
@@ -89,6 +99,28 @@ class EvalSchema(BaseModel):
             eval_data=asdict(result),
         )
 
+    @classmethod
+    def from_agent_as_judge_eval(
+        cls,
+        agent_as_judge_eval: AgentAsJudgeEval,
+        result: AgentAsJudgeResult,
+        model_id: Optional[str] = None,
+        model_provider: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        team_id: Optional[str] = None,
+    ) -> "EvalSchema":
+        return cls(
+            id=result.run_id,
+            name=agent_as_judge_eval.name,
+            agent_id=agent_id,
+            team_id=team_id,
+            workflow_id=None,
+            model_id=model_id,
+            model_provider=model_provider,
+            eval_type=EvalType.AGENT_AS_JUDGE,
+            eval_data=asdict(result),
+        )
+
     @classmethod
     def from_performance_eval(
         cls,

agno/os/routers/evals/utils.py

@@ -5,6 +5,7 @@ from fastapi import HTTPException
 from agno.agent.agent import Agent
 from agno.db.base import AsyncBaseDb, BaseDb
 from agno.eval.accuracy import AccuracyEval
+from agno.eval.agent_as_judge import AgentAsJudgeEval
 from agno.eval.performance import PerformanceEval
 from agno.eval.reliability import ReliabilityEval
 from agno.models.base import Model
@@ -36,15 +37,77 @@ async def run_accuracy_eval(
         model=default_model,
     )
 
-    if isinstance(db, AsyncBaseDb):
-        result = await accuracy_eval.arun(print_results=False, print_summary=False)
-    else:
-        result = accuracy_eval.run(print_results=False, print_summary=False)
+    result = await accuracy_eval.arun(print_results=False, print_summary=False)
     if not result:
         raise HTTPException(status_code=500, detail="Failed to run accuracy evaluation")
 
     eval_run = EvalSchema.from_accuracy_eval(accuracy_eval=accuracy_eval, result=result)
 
+    # Restore original model after eval
+    if default_model is not None:
+        if agent is not None:
+            agent.model = default_model
+        elif team is not None:
+            team.model = default_model
+
+    return eval_run
+
+
+async def run_agent_as_judge_eval(
+    eval_run_input: EvalRunInput,
+    db: Union[BaseDb, AsyncBaseDb],
+    agent: Optional[Agent] = None,
+    team: Optional[Team] = None,
+    default_model: Optional[Model] = None,
+) -> EvalSchema:
+    """Run an AgentAsJudge evaluation for the given agent or team"""
+    if not eval_run_input.criteria:
+        raise HTTPException(status_code=400, detail="criteria is required for agent-as-judge evaluation")
+
+    # Run agent/team to get output
+    if agent:
+        agent_response = await agent.arun(eval_run_input.input)
+        output = str(agent_response.content) if agent_response.content else ""
+        model_id = agent.model.id if agent and agent.model else None
+        model_provider = agent.model.provider if agent and agent.model else None
+        agent_id = agent.id
+        team_id = None
+    elif team:
+        team_response = await team.arun(eval_run_input.input)
+        output = str(team_response.content) if team_response.content else ""
+        model_id = team.model.id if team and team.model else None
+        model_provider = team.model.provider if team and team.model else None
+        agent_id = None
+        team_id = team.id
+    else:
+        raise HTTPException(status_code=400, detail="Either agent_id or team_id must be provided")
+
+    agent_as_judge_eval = AgentAsJudgeEval(
+        db=db,
+        criteria=eval_run_input.criteria,
+        scoring_strategy=eval_run_input.scoring_strategy or "binary",
+        threshold=eval_run_input.threshold or 7,
+        additional_guidelines=eval_run_input.additional_guidelines,
+        name=eval_run_input.name,
+        model=default_model,
+    )
+
+    result = await agent_as_judge_eval.arun(
+        input=eval_run_input.input, output=output, print_results=False, print_summary=False
+    )
+    if not result:
+        raise HTTPException(status_code=500, detail="Failed to run agent as judge evaluation")
+
+    eval_run = EvalSchema.from_agent_as_judge_eval(
+        agent_as_judge_eval=agent_as_judge_eval,
+        result=result,
+        agent_id=agent_id,
+        team_id=team_id,
+        model_id=model_id,
+        model_provider=model_provider,
+    )
+
+    # Restore original model after eval
     if default_model is not None:
         if agent is not None:
             agent.model = default_model
@@ -62,21 +125,39 @@ async def run_performance_eval(
     default_model: Optional[Model] = None,
 ) -> EvalSchema:
     """Run a performance evaluation for the given agent or team"""
-    if agent:
+    # Create sync or async function based on DB type
+    if isinstance(db, AsyncBaseDb):
+        if agent:
 
-        def run_component():  # type: ignore
-            return agent.run(eval_run_input.input)
+            async def run_component():  # type: ignore
+                return await agent.arun(eval_run_input.input)
 
-        model_id = agent.model.id if agent and agent.model else None
-        model_provider = agent.model.provider if agent and agent.model else None
+            model_id = agent.model.id if agent and agent.model else None
+            model_provider = agent.model.provider if agent and agent.model else None
 
-    elif team:
+        elif team:
 
-        def run_component():
-            return team.run(eval_run_input.input)
+            async def run_component():  # type: ignore
+                return await team.arun(eval_run_input.input)
 
-        model_id = team.model.id if team and team.model else None
-        model_provider = team.model.provider if team and team.model else None
+            model_id = team.model.id if team and team.model else None
+            model_provider = team.model.provider if team and team.model else None
+    else:
+        if agent:
+
+            def run_component():  # type: ignore
+                return agent.run(eval_run_input.input)
+
+            model_id = agent.model.id if agent and agent.model else None
+            model_provider = agent.model.provider if agent and agent.model else None
+
+        elif team:
+
+            def run_component():
+                return team.run(eval_run_input.input)
+
+            model_id = team.model.id if team and team.model else None
+            model_provider = team.model.provider if team and team.model else None
 
     performance_eval = PerformanceEval(
         db=db,
@@ -90,6 +171,7 @@ async def run_performance_eval(
         model_provider=model_provider,
     )
 
+    # PerformanceEval needs sync/async check because it wraps a function
     if isinstance(db, AsyncBaseDb):
         result = await performance_eval.arun(print_results=False, print_summary=False)
     else:
@@ -106,6 +188,7 @@ async def run_performance_eval(
         model_provider=model_provider,
     )
 
+    # Restore original model after eval
     if default_model is not None:
         if agent is not None:
             agent.model = default_model
@@ -127,7 +210,7 @@ async def run_reliability_eval(
         raise HTTPException(status_code=400, detail="expected_tool_calls is required for reliability evaluations")
 
     if agent:
-        agent_response = agent.run(eval_run_input.input)
+        agent_response = await agent.arun(eval_run_input.input)
         reliability_eval = ReliabilityEval(
             db=db,
             name=eval_run_input.name,
@@ -138,7 +221,7 @@ async def run_reliability_eval(
         model_provider = agent.model.provider if agent and agent.model else None
 
     elif team:
-        team_response = team.run(eval_run_input.input)
+        team_response = await team.arun(eval_run_input.input)
         reliability_eval = ReliabilityEval(
             db=db,
             name=eval_run_input.name,
@@ -148,10 +231,7 @@ async def run_reliability_eval(
         model_id = team.model.id if team and team.model else None
         model_provider = team.model.provider if team and team.model else None
 
-    if isinstance(db, AsyncBaseDb):
-        result = await reliability_eval.arun(print_results=False)
-    else:
-        result = reliability_eval.run(print_results=False)
+    result = await reliability_eval.arun(print_results=False)
     if not result:
         raise HTTPException(status_code=500, detail="Failed to run reliability evaluation")
 
@@ -163,6 +243,7 @@ async def run_reliability_eval(
         model_provider=model_provider,
     )
 
+    # Restore original model after eval
     if default_model is not None:
         if agent is not None:
             agent.model = default_model

agno/os/routers/knowledge/knowledge.py

@@ -981,7 +981,7 @@ async def process_content(
             log_debug(f"Set chunking strategy: {chunker}")
 
         log_debug(f"Using reader: {content.reader.__class__.__name__}")
-        await knowledge._load_content(content, upsert=False, skip_if_exists=True)
+        await knowledge._load_content_async(content, upsert=False, skip_if_exists=True)
         log_info(f"Content {content.id} processed successfully")
     except Exception as e:
         log_info(f"Error processing content: {e}")

agno/os/routers/teams/__init__.py

@@ -0,0 +1,3 @@
+from agno.os.routers.teams.router import get_team_router
+
+__all__ = ["get_team_router"]