agno 2.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

agno/eval/base.py

@@ -0,0 +1,29 @@
+from abc import ABC, abstractmethod
+from typing import Union
+
+from agno.run.agent import RunInput, RunOutput
+from agno.run.team import TeamRunInput, TeamRunOutput
+
+
+class BaseEval(ABC):
+    """Abstract base class for all evaluations."""
+
+    @abstractmethod
+    def pre_check(self, run_input: Union[RunInput, TeamRunInput]) -> None:
+        """Perform sync pre-evals."""
+        pass
+
+    @abstractmethod
+    async def async_pre_check(self, run_input: Union[RunInput, TeamRunInput]) -> None:
+        """Perform async pre-evals."""
+        pass
+
+    @abstractmethod
+    def post_check(self, run_output: Union[RunOutput, TeamRunOutput]) -> None:
+        """Perform sync post-evals."""
+        pass
+
+    @abstractmethod
+    async def async_post_check(self, run_output: Union[RunOutput, TeamRunOutput]) -> None:
+        """Perform async post-evals."""
+        pass

agno/eval/performance.py

@@ -3,10 +3,10 @@ import gc
 import tracemalloc
 from dataclasses import dataclass, field
 from os import getenv
-from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
 from uuid import uuid4
 
-from agno.db.base import BaseDb
+from agno.db.base import AsyncBaseDb, BaseDb
 from agno.db.schemas.evals import EvalType
 from agno.eval.utils import async_log_eval, log_eval_run, store_result_in_file
 from agno.utils.log import log_debug, set_log_level_to_debug, set_log_level_to_info
@@ -222,7 +222,7 @@ class PerformanceEval:
     # Enable debug logs
     debug_mode: bool = getenv("AGNO_DEBUG", "false").lower() == "true"
     # The database to store Evaluation results
-    db: Optional[BaseDb] = None
+    db: Optional[Union[BaseDb, AsyncBaseDb]] = None
 
     # Telemetry settings
     # telemetry=True logs minimal telemetry for analytics
@@ -491,17 +491,23 @@ class PerformanceEval:
         6. Print results as requested
         7. Log results to the Agno platform if requested
         """
+        if isinstance(self.db, AsyncBaseDb):
+            raise ValueError("run() is not supported with an async DB. Please use arun() instead.")
+
         from rich.console import Console
         from rich.live import Live
         from rich.status import Status
 
+        # Generate unique run_id for this execution (don't modify self.eval_id due to concurrency)
+        run_id = str(uuid4())
+
         run_times = []
         memory_usages = []
         previous_snapshot = None
 
         self._set_log_level()
 
-        log_debug(f"************ Evaluation Start: {self.eval_id} ************")
+        log_debug(f"************ Evaluation Start: {run_id} ************")
 
         # Add a spinner while running the evaluations
         console = Console()
@@ -612,7 +618,7 @@ class PerformanceEval:
                 ),
             )
 
-        log_debug(f"*********** Evaluation End: {self.eval_id} ***********")
+        log_debug(f"*********** Evaluation End: {run_id} ***********")
         return self.result
 
     async def arun(
@@ -638,13 +644,16 @@ class PerformanceEval:
         from rich.live import Live
         from rich.status import Status
 
+        # Generate unique run_id for this execution (don't modify self.eval_id due to concurrency)
+        run_id = str(uuid4())
+
         run_times = []
         memory_usages = []
         previous_snapshot = None
 
         self._set_log_level()
 
-        log_debug(f"************ Evaluation Start: {self.eval_id} ************")
+        log_debug(f"************ Evaluation Start: {run_id} ************")
 
         # Add a spinner while running the evaluations
         console = Console()
@@ -755,7 +764,7 @@ class PerformanceEval:
                 ),
             )
 
-        log_debug(f"*********** Evaluation End: {self.eval_id} ***********")
+        log_debug(f"*********** Evaluation End: {run_id} ***********")
         return self.result
 
     def _get_telemetry_data(self) -> Dict[str, Any]:

agno/eval/reliability.py

@@ -1,9 +1,10 @@
 from dataclasses import asdict, dataclass, field
 from os import getenv
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 from uuid import uuid4
 
-from agno.db.base import BaseDb
+from agno.db.base import AsyncBaseDb, BaseDb
+from agno.run.team import TeamRunOutput
 
 if TYPE_CHECKING:
     from rich.console import Console
@@ -11,7 +12,6 @@ if TYPE_CHECKING:
 from agno.agent import RunOutput
 from agno.db.schemas.evals import EvalType
 from agno.eval.utils import async_log_eval, log_eval_run, store_result_in_file
-from agno.run.team import TeamRunOutput
 from agno.utils.log import logger
 
 
@@ -63,7 +63,7 @@ class ReliabilityEval:
     # Enable debug logs
     debug_mode: bool = getenv("AGNO_DEBUG", "false").lower() == "true"
     # The database to store Evaluation results
-    db: Optional[BaseDb] = None
+    db: Optional[Union[BaseDb, AsyncBaseDb]] = None
 
     # Telemetry settings
     # telemetry=True logs minimal telemetry for analytics
@@ -71,6 +71,9 @@ class ReliabilityEval:
     telemetry: bool = True
 
     def run(self, *, print_results: bool = False) -> Optional[ReliabilityResult]:
+        if isinstance(self.db, AsyncBaseDb):
+            raise ValueError("run() is not supported with an async DB. Please use arun() instead.")
+
         if self.agent_response is None and self.team_response is None:
             raise ValueError("You need to provide 'agent_response' or 'team_response' to run the evaluation.")
 
@@ -83,6 +86,9 @@ class ReliabilityEval:
         from rich.live import Live
         from rich.status import Status
 
+        # Generate unique run_id for this execution (don't modify self.eval_id due to concurrency)
+        run_id = str(uuid4())
+
         # Add a spinner while running the evaluations
         console = Console()
         with Live(console=console, transient=True) as live_log:
@@ -115,7 +121,7 @@ class ReliabilityEval:
                     if not tool_name:
                         continue
                     else:
-                        if tool_name not in self.expected_tool_calls:  # type: ignore
+                        if self.expected_tool_calls is not None and tool_name not in self.expected_tool_calls:
                             failed_tool_calls.append(tool_call.get("function", {}).get("name"))
                         else:
                             passed_tool_calls.append(tool_call.get("function", {}).get("name"))
@@ -180,7 +186,7 @@ class ReliabilityEval:
                 ),
             )
 
-        logger.debug(f"*********** Evaluation End: {self.eval_id} ***********")
+        logger.debug(f"*********** Evaluation End: {run_id} ***********")
         return self.result
 
     async def arun(self, *, print_results: bool = False) -> Optional[ReliabilityResult]:
@@ -196,6 +202,9 @@ class ReliabilityEval:
         from rich.live import Live
         from rich.status import Status
 
+        # Generate unique run_id for this execution (don't modify self.eval_id due to concurrency)
+        run_id = str(uuid4())
+
         # Add a spinner while running the evaluations
         console = Console()
         with Live(console=console, transient=True) as live_log:
@@ -220,15 +229,18 @@ class ReliabilityEval:
 
             failed_tool_calls = []
             passed_tool_calls = []
-            for tool_call in actual_tool_calls:  # type: ignore
-                tool_name = tool_call.get("function", {}).get("name")
-                if not tool_name:
-                    continue
-                else:
-                    if tool_name not in self.expected_tool_calls:  # type: ignore
-                        failed_tool_calls.append(tool_call.get("function", {}).get("name"))
+            if not actual_tool_calls:
+                failed_tool_calls = self.expected_tool_calls or []
+            else:
+                for tool_call in actual_tool_calls:  # type: ignore
+                    tool_name = tool_call.get("function", {}).get("name")
+                    if not tool_name:
+                        continue
                     else:
-                        passed_tool_calls.append(tool_call.get("function", {}).get("name"))
+                        if self.expected_tool_calls is not None and tool_name not in self.expected_tool_calls:
+                            failed_tool_calls.append(tool_call.get("function", {}).get("name"))
+                        else:
+                            passed_tool_calls.append(tool_call.get("function", {}).get("name"))
 
             self.result = ReliabilityResult(
                 eval_status="PASSED" if len(failed_tool_calls) == 0 else "FAILED",
@@ -241,7 +253,7 @@ class ReliabilityEval:
             store_result_in_file(
                 file_path=self.file_path_to_save_results,
                 name=self.name,
-                eval_id=self.eval_id,
+                eval_id=run_id,
                 result=self.result,
             )
 
@@ -290,7 +302,7 @@ class ReliabilityEval:
                 ),
             )
 
-        logger.debug(f"*********** Evaluation End: {self.eval_id} ***********")
+        logger.debug(f"*********** Evaluation End: {run_id} ***********")
         return self.result
 
     def _get_telemetry_data(self) -> Dict[str, Any]:

agno/eval/utils.py

@@ -2,12 +2,13 @@ from dataclasses import asdict
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional, Union
 
-from agno.db.base import BaseDb
+from agno.db.base import AsyncBaseDb, BaseDb
 from agno.db.schemas.evals import EvalRunRecord, EvalType
 from agno.utils.log import log_debug, logger
 
 if TYPE_CHECKING:
     from agno.eval.accuracy import AccuracyResult
+    from agno.eval.agent_as_judge import AgentAsJudgeResult
     from agno.eval.performance import PerformanceResult
     from agno.eval.reliability import ReliabilityResult
 
@@ -49,7 +50,7 @@ def log_eval_run(
 
 
 async def async_log_eval(
-    db: BaseDb,
+    db: Union[BaseDb, AsyncBaseDb],
     run_id: str,
     run_data: dict,
     eval_type: EvalType,
@@ -65,28 +66,45 @@ async def async_log_eval(
     """Call the API to create an evaluation run."""
 
     try:
-        db.create_eval_run(
-            EvalRunRecord(
-                run_id=run_id,
-                eval_type=eval_type,
-                eval_data=run_data,
-                eval_input=eval_input,
-                agent_id=agent_id,
-                model_id=model_id,
-                model_provider=model_provider,
-                name=name,
-                evaluated_component_name=evaluated_component_name,
-                team_id=team_id,
-                workflow_id=workflow_id,
+        if isinstance(db, AsyncBaseDb):
+            await db.create_eval_run(
+                EvalRunRecord(
+                    run_id=run_id,
+                    eval_type=eval_type,
+                    eval_data=run_data,
+                    eval_input=eval_input,
+                    agent_id=agent_id,
+                    model_id=model_id,
+                    model_provider=model_provider,
+                    name=name,
+                    evaluated_component_name=evaluated_component_name,
+                    team_id=team_id,
+                    workflow_id=workflow_id,
+                )
+            )
+        else:
+            db.create_eval_run(
+                EvalRunRecord(
+                    run_id=run_id,
+                    eval_type=eval_type,
+                    eval_data=run_data,
+                    eval_input=eval_input,
+                    agent_id=agent_id,
+                    model_id=model_id,
+                    model_provider=model_provider,
+                    name=name,
+                    evaluated_component_name=evaluated_component_name,
+                    team_id=team_id,
+                    workflow_id=workflow_id,
+                )
             )
-        )
     except Exception as e:
         log_debug(f"Could not create agent event: {e}")
 
 
 def store_result_in_file(
     file_path: str,
-    result: Union["AccuracyResult", "PerformanceResult", "ReliabilityResult"],
+    result: Union["AccuracyResult", "AgentAsJudgeResult", "PerformanceResult", "ReliabilityResult"],
     eval_id: Optional[str] = None,
     name: Optional[str] = None,
 ):

agno/exceptions.py

@@ -1,3 +1,4 @@
+from dataclasses import dataclass
 from enum import Enum
 from typing import Any, Dict, List, Optional, Union
 
@@ -77,6 +78,17 @@ class AgnoError(Exception):
         return str(self.message)
 
 
+class ModelAuthenticationError(AgnoError):
+    """Raised when model authentication fails."""
+
+    def __init__(self, message: str, status_code: int = 401, model_name: Optional[str] = None):
+        super().__init__(message, status_code)
+        self.model_name = model_name
+
+        self.type = "model_authentication_error"
+        self.error_id = "model_authentication_error"
+
+
 class ModelProviderError(AgnoError):
     """Exception raised when a model provider returns an error."""
 
@@ -130,7 +142,10 @@ class InputCheckError(Exception):
     ):
         super().__init__(message)
         self.type = "input_check_error"
-        self.error_id = check_trigger.value
+        if isinstance(check_trigger, CheckTrigger):
+            self.error_id = check_trigger.value
+        else:
+            self.error_id = str(check_trigger)
 
         self.message = message
         self.check_trigger = check_trigger
@@ -148,8 +163,18 @@ class OutputCheckError(Exception):
     ):
         super().__init__(message)
         self.type = "output_check_error"
-        self.error_id = check_trigger.value
+        if isinstance(check_trigger, CheckTrigger):
+            self.error_id = check_trigger.value
+        else:
+            self.error_id = str(check_trigger)
 
         self.message = message
         self.check_trigger = check_trigger
         self.additional_data = additional_data
+
+
+@dataclass
+class RetryableModelProviderError(Exception):
+    original_error: Optional[str] = None
+    # Guidance message to retry a model invocation after an error
+    retry_guidance_message: Optional[str] = None