agno 0.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

agno/eval/reliability.py

@@ -1,14 +1,18 @@
-from dataclasses import asdict, dataclass
+from dataclasses import asdict, dataclass, field
 from os import getenv
-from pathlib import Path
-from typing import TYPE_CHECKING, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 from uuid import uuid4
 
+from agno.db.base import AsyncBaseDb, BaseDb
+from agno.run.team import TeamRunOutput
+
 if TYPE_CHECKING:
     from rich.console import Console
 
-from agno.run.response import RunResponse
-from agno.utils.log import logger, set_log_level_to_debug, set_log_level_to_info
+from agno.agent import RunOutput
+from agno.db.schemas.evals import EvalType
+from agno.eval.utils import async_log_eval, log_eval_run, store_result_in_file
+from agno.utils.log import logger
 
 
 @dataclass
@@ -40,52 +44,50 @@ class ReliabilityEval:
 
     # Evaluation name
     name: Optional[str] = None
-
-    # Evaluation UUID (autogenerated if not set)
-    eval_id: Optional[str] = None
+    # Evaluation UUID
+    eval_id: str = field(default_factory=lambda: str(uuid4()))
 
     # Agent response
-    agent_response: Optional[RunResponse] = None
-
+    agent_response: Optional[RunOutput] = None
+    # Team response
+    team_response: Optional[TeamRunOutput] = None
     # Expected tool calls
     expected_tool_calls: Optional[List[str]] = None
-
     # Result of the evaluation
     result: Optional[ReliabilityResult] = None
 
-    # Print summary of results
-    print_summary: bool = False
     # Print detailed results
     print_results: bool = False
-    # Save the result to a file
-    save_result_to_file: Optional[str] = None
-
-    # debug_mode=True enables debug logs
-    debug_mode: bool = False
-
-    def set_eval_id(self) -> str:
-        if self.eval_id is None:
-            self.eval_id = str(uuid4())
-        logger.debug(f"*********** Evaluation ID: {self.eval_id} ***********")
-        return self.eval_id
-
-    def set_debug_mode(self) -> None:
-        if self.debug_mode or getenv("AGNO_DEBUG", "false").lower() == "true":
-            self.debug_mode = True
-            set_log_level_to_debug()
-            logger.debug("Debug logs enabled")
-        else:
-            set_log_level_to_info()
-
-    def run(self, *, print_summary: bool = False, print_results: bool = False) -> Optional[ReliabilityResult]:
+    # If set, results will be saved in the given file path
+    file_path_to_save_results: Optional[str] = None
+    # Enable debug logs
+    debug_mode: bool = getenv("AGNO_DEBUG", "false").lower() == "true"
+    # The database to store Evaluation results
+    db: Optional[Union[BaseDb, AsyncBaseDb]] = None
+
+    # Telemetry settings
+    # telemetry=True logs minimal telemetry for analytics
+    # This helps us improve our Evals and provide better support
+    telemetry: bool = True
+
+    def run(self, *, print_results: bool = False) -> Optional[ReliabilityResult]:
+        if isinstance(self.db, AsyncBaseDb):
+            raise ValueError("run() is not supported with an async DB. Please use arun() instead.")
+
+        if self.agent_response is None and self.team_response is None:
+            raise ValueError("You need to provide 'agent_response' or 'team_response' to run the evaluation.")
+
+        if self.agent_response is not None and self.team_response is not None:
+            raise ValueError(
+                "You need to provide only one of 'agent_response' or 'team_response' to run the evaluation."
+            )
+
         from rich.console import Console
         from rich.live import Live
         from rich.status import Status
 
-        self.set_eval_id()
-        self.set_debug_mode()
-        self.print_results = print_results
-        self.print_summary = print_summary
+        # Generate unique run_id for this execution (don't modify self.eval_id due to concurrency)
+        run_id = str(uuid4())
 
         # Add a spinner while running the evaluations
         console = Console()
@@ -95,20 +97,34 @@ class ReliabilityEval:
 
             actual_tool_calls = None
             if self.agent_response is not None:
-                for message in reversed(self.agent_response.messages):  # type: ignore
-                    if message.tool_calls:
-                        if actual_tool_calls is None:
-                            actual_tool_calls = message.tool_calls
-                        else:
-                            actual_tool_calls.append(message.tool_calls[0])  # type: ignore
+                messages = self.agent_response.messages
+            elif self.team_response is not None:
+                messages = self.team_response.messages or []
+                for member_response in self.team_response.member_responses:
+                    if member_response.messages is not None:
+                        messages += member_response.messages
+
+            for message in reversed(messages):  # type: ignore
+                if message.tool_calls:
+                    if actual_tool_calls is None:
+                        actual_tool_calls = message.tool_calls
+                    else:
+                        actual_tool_calls.append(message.tool_calls[0])  # type: ignore
 
             failed_tool_calls = []
             passed_tool_calls = []
-            for tool_call in actual_tool_calls:  # type: ignore
-                if tool_call.get("function", {}).get("name") not in self.expected_tool_calls:  # type: ignore
-                    failed_tool_calls.append(tool_call.get("function", {}).get("name"))
-                else:
-                    passed_tool_calls.append(tool_call.get("function", {}).get("name"))
+            if not actual_tool_calls:
+                failed_tool_calls = self.expected_tool_calls or []
+            else:
+                for tool_call in actual_tool_calls:  # type: ignore
+                    tool_name = tool_call.get("function", {}).get("name")
+                    if not tool_name:
+                        continue
+                    else:
+                        if self.expected_tool_calls is not None and tool_name not in self.expected_tool_calls:
+                            failed_tool_calls.append(tool_call.get("function", {}).get("name"))
+                        else:
+                            passed_tool_calls.append(tool_call.get("function", {}).get("name"))
 
             self.result = ReliabilityResult(
                 eval_status="PASSED" if len(failed_tool_calls) == 0 else "FAILED",
@@ -116,21 +132,184 @@ class ReliabilityEval:
                 passed_tool_calls=passed_tool_calls,
             )
 
-        # -*- Save result to file if save_result_to_file is set
-        if self.save_result_to_file is not None and self.result is not None:
-            try:
-                import json
+        # Save result to file if requested
+        if self.file_path_to_save_results is not None and self.result is not None:
+            store_result_in_file(
+                file_path=self.file_path_to_save_results,
+                name=self.name,
+                eval_id=self.eval_id,
+                result=self.result,
+            )
+
+        # Print results if requested
+        if self.print_results or print_results:
+            self.result.print_eval(console)
+
+        # Log results to the Agno platform if requested
+        if self.db:
+            if self.agent_response is not None:
+                agent_id = self.agent_response.agent_id
+                team_id = None
+                model_id = self.agent_response.model
+                model_provider = self.agent_response.model_provider
+            elif self.team_response is not None:
+                agent_id = None
+                team_id = self.team_response.team_id
+                model_id = self.team_response.model
+                model_provider = self.team_response.model_provider
+
+            eval_input = {
+                "expected_tool_calls": self.expected_tool_calls,
+            }
+
+            log_eval_run(
+                db=self.db,
+                run_id=self.eval_id,  # type: ignore
+                run_data=asdict(self.result),
+                eval_type=EvalType.RELIABILITY,
+                name=self.name if self.name is not None else None,
+                agent_id=agent_id,
+                team_id=team_id,
+                model_id=model_id,
+                model_provider=model_provider,
+                eval_input=eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, create_eval_run_telemetry
+
+            create_eval_run_telemetry(
+                eval_run=EvalRunCreate(
+                    run_id=self.eval_id,
+                    eval_type=EvalType.RELIABILITY,
+                    data=self._get_telemetry_data(),
+                ),
+            )
+
+        logger.debug(f"*********** Evaluation End: {run_id} ***********")
+        return self.result
+
+    async def arun(self, *, print_results: bool = False) -> Optional[ReliabilityResult]:
+        if self.agent_response is None and self.team_response is None:
+            raise ValueError("You need to provide 'agent_response' or 'team_response' to run the evaluation.")
+
+        if self.agent_response is not None and self.team_response is not None:
+            raise ValueError(
+                "You need to provide only one of 'agent_response' or 'team_response' to run the evaluation."
+            )
+
+        from rich.console import Console
+        from rich.live import Live
+        from rich.status import Status
+
+        # Generate unique run_id for this execution (don't modify self.eval_id due to concurrency)
+        run_id = str(uuid4())
+
+        # Add a spinner while running the evaluations
+        console = Console()
+        with Live(console=console, transient=True) as live_log:
+            status = Status("Running evaluation...", spinner="dots", speed=1.0, refresh_per_second=10)
+            live_log.update(status)
+
+            actual_tool_calls = None
+            if self.agent_response is not None:
+                messages = self.agent_response.messages
+            elif self.team_response is not None:
+                messages = self.team_response.messages or []
+                for member_response in self.team_response.member_responses:
+                    if member_response.messages is not None:
+                        messages += member_response.messages
+
+            for message in reversed(messages):  # type: ignore
+                if message.tool_calls:
+                    if actual_tool_calls is None:
+                        actual_tool_calls = message.tool_calls
+                    else:
+                        actual_tool_calls.append(message.tool_calls[0])  # type: ignore
+
+            failed_tool_calls = []
+            passed_tool_calls = []
+            if not actual_tool_calls:
+                failed_tool_calls = self.expected_tool_calls or []
+            else:
+                for tool_call in actual_tool_calls:  # type: ignore
+                    tool_name = tool_call.get("function", {}).get("name")
+                    if not tool_name:
+                        continue
+                    else:
+                        if self.expected_tool_calls is not None and tool_name not in self.expected_tool_calls:
+                            failed_tool_calls.append(tool_call.get("function", {}).get("name"))
+                        else:
+                            passed_tool_calls.append(tool_call.get("function", {}).get("name"))
+
+            self.result = ReliabilityResult(
+                eval_status="PASSED" if len(failed_tool_calls) == 0 else "FAILED",
+                failed_tool_calls=failed_tool_calls,
+                passed_tool_calls=passed_tool_calls,
+            )
 
-                fn_path = Path(self.save_result_to_file.format(name=self.name, eval_id=self.eval_id))
-                if not fn_path.parent.exists():
-                    fn_path.parent.mkdir(parents=True, exist_ok=True)
-                fn_path.write_text(json.dumps(asdict(self.result), indent=4))
-            except Exception as e:
-                logger.warning(f"Failed to save result to file: {e}")
+        # Save result to file if requested
+        if self.file_path_to_save_results is not None and self.result is not None:
+            store_result_in_file(
+                file_path=self.file_path_to_save_results,
+                name=self.name,
+                eval_id=run_id,
+                result=self.result,
+            )
 
-        # Show results
-        if self.print_summary or self.print_results:
+        # Print results if requested
+        if self.print_results or print_results:
             self.result.print_eval(console)
 
-        logger.debug(f"*********** Evaluation End: {self.eval_id} ***********")
+        # Log results to the Agno platform if requested
+        if self.db:
+            if self.agent_response is not None:
+                agent_id = self.agent_response.agent_id
+                team_id = None
+                model_id = self.agent_response.model
+                model_provider = self.agent_response.model_provider
+            elif self.team_response is not None:
+                agent_id = None
+                team_id = self.team_response.team_id
+                model_id = self.team_response.model
+                model_provider = self.team_response.model_provider
+
+            eval_input = {
+                "expected_tool_calls": self.expected_tool_calls,
+            }
+
+            await async_log_eval(
+                db=self.db,
+                run_id=self.eval_id,  # type: ignore
+                run_data=asdict(self.result),
+                eval_type=EvalType.RELIABILITY,
+                name=self.name if self.name is not None else None,
+                agent_id=agent_id,
+                team_id=team_id,
+                model_id=model_id,
+                model_provider=model_provider,
+                eval_input=eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, async_create_eval_run_telemetry
+
+            await async_create_eval_run_telemetry(
+                eval_run=EvalRunCreate(
+                    run_id=self.eval_id,
+                    eval_type=EvalType.RELIABILITY,
+                    data=self._get_telemetry_data(),
+                ),
+            )
+
+        logger.debug(f"*********** Evaluation End: {run_id} ***********")
         return self.result
+
+    def _get_telemetry_data(self) -> Dict[str, Any]:
+        """Get the telemetry data for the evaluation"""
+        return {
+            "team_id": self.team_response.team_id if self.team_response else None,
+            "agent_id": self.agent_response.agent_id if self.agent_response else None,
+            "model_id": self.agent_response.model if self.agent_response else None,
+            "model_provider": self.agent_response.model_provider if self.agent_response else None,
+        }

agno/eval/utils.py

@@ -0,0 +1,120 @@
+from dataclasses import asdict
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional, Union
+
+from agno.db.base import AsyncBaseDb, BaseDb
+from agno.db.schemas.evals import EvalRunRecord, EvalType
+from agno.utils.log import log_debug, logger
+
+if TYPE_CHECKING:
+    from agno.eval.accuracy import AccuracyResult
+    from agno.eval.agent_as_judge import AgentAsJudgeResult
+    from agno.eval.performance import PerformanceResult
+    from agno.eval.reliability import ReliabilityResult
+
+
+def log_eval_run(
+    db: BaseDb,
+    run_id: str,
+    run_data: dict,
+    eval_type: EvalType,
+    eval_input: dict,
+    agent_id: Optional[str] = None,
+    model_id: Optional[str] = None,
+    model_provider: Optional[str] = None,
+    name: Optional[str] = None,
+    evaluated_component_name: Optional[str] = None,
+    team_id: Optional[str] = None,
+    workflow_id: Optional[str] = None,
+) -> None:
+    """Call the API to create an evaluation run."""
+
+    try:
+        db.create_eval_run(
+            EvalRunRecord(
+                run_id=run_id,
+                eval_type=eval_type,
+                eval_data=run_data,
+                eval_input=eval_input,
+                agent_id=agent_id,
+                model_id=model_id,
+                model_provider=model_provider,
+                name=name,
+                evaluated_component_name=evaluated_component_name,
+                team_id=team_id,
+                workflow_id=workflow_id,
+            )
+        )
+    except Exception as e:
+        log_debug(f"Could not create agent event: {e}")
+
+
+async def async_log_eval(
+    db: Union[BaseDb, AsyncBaseDb],
+    run_id: str,
+    run_data: dict,
+    eval_type: EvalType,
+    eval_input: dict,
+    agent_id: Optional[str] = None,
+    model_id: Optional[str] = None,
+    model_provider: Optional[str] = None,
+    name: Optional[str] = None,
+    evaluated_component_name: Optional[str] = None,
+    team_id: Optional[str] = None,
+    workflow_id: Optional[str] = None,
+) -> None:
+    """Call the API to create an evaluation run."""
+
+    try:
+        if isinstance(db, AsyncBaseDb):
+            await db.create_eval_run(
+                EvalRunRecord(
+                    run_id=run_id,
+                    eval_type=eval_type,
+                    eval_data=run_data,
+                    eval_input=eval_input,
+                    agent_id=agent_id,
+                    model_id=model_id,
+                    model_provider=model_provider,
+                    name=name,
+                    evaluated_component_name=evaluated_component_name,
+                    team_id=team_id,
+                    workflow_id=workflow_id,
+                )
+            )
+        else:
+            db.create_eval_run(
+                EvalRunRecord(
+                    run_id=run_id,
+                    eval_type=eval_type,
+                    eval_data=run_data,
+                    eval_input=eval_input,
+                    agent_id=agent_id,
+                    model_id=model_id,
+                    model_provider=model_provider,
+                    name=name,
+                    evaluated_component_name=evaluated_component_name,
+                    team_id=team_id,
+                    workflow_id=workflow_id,
+                )
+            )
+    except Exception as e:
+        log_debug(f"Could not create agent event: {e}")
+
+
+def store_result_in_file(
+    file_path: str,
+    result: Union["AccuracyResult", "AgentAsJudgeResult", "PerformanceResult", "ReliabilityResult"],
+    eval_id: Optional[str] = None,
+    name: Optional[str] = None,
+):
+    """Store the given result in the given file path"""
+    try:
+        import json
+
+        fn_path = Path(file_path.format(name=name, eval_id=eval_id))
+        if not fn_path.parent.exists():
+            fn_path.parent.mkdir(parents=True, exist_ok=True)
+        fn_path.write_text(json.dumps(asdict(result), indent=4))
+    except Exception as e:
+        logger.warning(f"Failed to save result to file: {e}")

agno/exceptions.py

@@ -1,4 +1,6 @@
-from typing import List, Optional, Union
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Dict, List, Optional, Union
 
 from agno.models.message import Message
 
@@ -17,11 +19,25 @@ class AgentRunException(Exception):
         self.agent_message = agent_message
         self.messages = messages
         self.stop_execution = stop_execution
+        self.type = "agent_run_error"
+        self.error_id = "agent_run_error"
 
 
 class RetryAgentRun(AgentRunException):
     """Exception raised when a tool call should be retried."""
 
+    def __init__(
+        self,
+        exc,
+        user_message: Optional[Union[str, Message]] = None,
+        agent_message: Optional[Union[str, Message]] = None,
+        messages: Optional[List[Union[dict, Message]]] = None,
+    ):
+        super().__init__(
+            exc, user_message=user_message, agent_message=agent_message, messages=messages, stop_execution=False
+        )
+        self.error_id = "retry_agent_run_error"
+
 
 class StopAgentRun(AgentRunException):
     """Exception raised when an agent should stop executing entirely."""
@@ -36,3 +52,129 @@ class StopAgentRun(AgentRunException):
         super().__init__(
             exc, user_message=user_message, agent_message=agent_message, messages=messages, stop_execution=True
         )
+        self.error_id = "stop_agent_run_error"
+
+
+class RunCancelledException(Exception):
+    """Exception raised when a run is cancelled."""
+
+    def __init__(self, message: str = "Operation cancelled by user"):
+        super().__init__(message)
+        self.type = "run_cancelled_error"
+        self.error_id = "run_cancelled_error"
+
+
+class AgnoError(Exception):
+    """Exception raised when an internal error occurs."""
+
+    def __init__(self, message: str, status_code: int = 500):
+        super().__init__(message)
+        self.message = message
+        self.status_code = status_code
+        self.type = "agno_error"
+        self.error_id = "agno_error"
+
+    def __str__(self) -> str:
+        return str(self.message)
+
+
+class ModelAuthenticationError(AgnoError):
+    """Raised when model authentication fails."""
+
+    def __init__(self, message: str, status_code: int = 401, model_name: Optional[str] = None):
+        super().__init__(message, status_code)
+        self.model_name = model_name
+
+        self.type = "model_authentication_error"
+        self.error_id = "model_authentication_error"
+
+
+class ModelProviderError(AgnoError):
+    """Exception raised when a model provider returns an error."""
+
+    def __init__(
+        self, message: str, status_code: int = 502, model_name: Optional[str] = None, model_id: Optional[str] = None
+    ):
+        super().__init__(message, status_code)
+        self.model_name = model_name
+        self.model_id = model_id
+
+        self.type = "model_provider_error"
+        self.error_id = "model_provider_error"
+
+
+class ModelRateLimitError(ModelProviderError):
+    """Exception raised when a model provider returns a rate limit error."""
+
+    def __init__(
+        self, message: str, status_code: int = 429, model_name: Optional[str] = None, model_id: Optional[str] = None
+    ):
+        super().__init__(message, status_code, model_name, model_id)
+        self.error_id = "model_rate_limit_error"
+
+
+class EvalError(Exception):
+    """Exception raised when an evaluation fails."""
+
+    pass
+
+
+class CheckTrigger(Enum):
+    """Enum for guardrail triggers."""
+
+    OFF_TOPIC = "off_topic"
+    INPUT_NOT_ALLOWED = "input_not_allowed"
+    OUTPUT_NOT_ALLOWED = "output_not_allowed"
+    VALIDATION_FAILED = "validation_failed"
+
+    PROMPT_INJECTION = "prompt_injection"
+    PII_DETECTED = "pii_detected"
+
+
+class InputCheckError(Exception):
+    """Exception raised when an input check fails."""
+
+    def __init__(
+        self,
+        message: str,
+        check_trigger: CheckTrigger = CheckTrigger.INPUT_NOT_ALLOWED,
+        additional_data: Optional[Dict[str, Any]] = None,
+    ):
+        super().__init__(message)
+        self.type = "input_check_error"
+        if isinstance(check_trigger, CheckTrigger):
+            self.error_id = check_trigger.value
+        else:
+            self.error_id = str(check_trigger)
+
+        self.message = message
+        self.check_trigger = check_trigger
+        self.additional_data = additional_data
+
+
+class OutputCheckError(Exception):
+    """Exception raised when an output check fails."""
+
+    def __init__(
+        self,
+        message: str,
+        check_trigger: CheckTrigger = CheckTrigger.OUTPUT_NOT_ALLOWED,
+        additional_data: Optional[Dict[str, Any]] = None,
+    ):
+        super().__init__(message)
+        self.type = "output_check_error"
+        if isinstance(check_trigger, CheckTrigger):
+            self.error_id = check_trigger.value
+        else:
+            self.error_id = str(check_trigger)
+
+        self.message = message
+        self.check_trigger = check_trigger
+        self.additional_data = additional_data
+
+
+@dataclass
+class RetryableModelProviderError(Exception):
+    original_error: Optional[str] = None
+    # Guidance message to retry a model invocation after an error
+    retry_guidance_message: Optional[str] = None