agno 2.2.13__py3-none-any.whl

agno/eval/reliability.py

@@ -0,0 +1,306 @@
+from dataclasses import asdict, dataclass, field
+from os import getenv
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from uuid import uuid4
+
+from agno.db.base import AsyncBaseDb, BaseDb
+
+if TYPE_CHECKING:
+    from rich.console import Console
+
+from agno.agent import RunOutput
+from agno.db.schemas.evals import EvalType
+from agno.eval.utils import async_log_eval, log_eval_run, store_result_in_file
+from agno.run.team import TeamRunOutput
+from agno.utils.log import logger
+
+
+@dataclass
+class ReliabilityResult:
+    eval_status: str
+    failed_tool_calls: List[str]
+    passed_tool_calls: List[str]
+
+    def print_eval(self, console: Optional["Console"] = None):
+        from rich.console import Console
+        from rich.table import Table
+
+        if console is None:
+            console = Console()
+
+        results_table = Table(title="Reliability Summary", show_header=True, header_style="bold magenta")
+        results_table.add_row("Evaluation Status", self.eval_status)
+        results_table.add_row("Failed Tool Calls", str(self.failed_tool_calls))
+        results_table.add_row("Passed Tool Calls", str(self.passed_tool_calls))
+        console.print(results_table)
+
+    def assert_passed(self):
+        assert self.eval_status == "PASSED"
+
+
+@dataclass
+class ReliabilityEval:
+    """Evaluate the reliability of a model by checking the tool calls"""
+
+    # Evaluation name
+    name: Optional[str] = None
+    # Evaluation UUID
+    eval_id: str = field(default_factory=lambda: str(uuid4()))
+
+    # Agent response
+    agent_response: Optional[RunOutput] = None
+    # Team response
+    team_response: Optional[TeamRunOutput] = None
+    # Expected tool calls
+    expected_tool_calls: Optional[List[str]] = None
+    # Result of the evaluation
+    result: Optional[ReliabilityResult] = None
+
+    # Print detailed results
+    print_results: bool = False
+    # If set, results will be saved in the given file path
+    file_path_to_save_results: Optional[str] = None
+    # Enable debug logs
+    debug_mode: bool = getenv("AGNO_DEBUG", "false").lower() == "true"
+    # The database to store Evaluation results
+    db: Optional[Union[BaseDb, AsyncBaseDb]] = None
+
+    # Telemetry settings
+    # telemetry=True logs minimal telemetry for analytics
+    # This helps us improve our Evals and provide better support
+    telemetry: bool = True
+
+    def run(self, *, print_results: bool = False) -> Optional[ReliabilityResult]:
+        if isinstance(self.db, AsyncBaseDb):
+            raise ValueError("run() is not supported with an async DB. Please use arun() instead.")
+
+        if self.agent_response is None and self.team_response is None:
+            raise ValueError("You need to provide 'agent_response' or 'team_response' to run the evaluation.")
+
+        if self.agent_response is not None and self.team_response is not None:
+            raise ValueError(
+                "You need to provide only one of 'agent_response' or 'team_response' to run the evaluation."
+            )
+
+        from rich.console import Console
+        from rich.live import Live
+        from rich.status import Status
+
+        # Add a spinner while running the evaluations
+        console = Console()
+        with Live(console=console, transient=True) as live_log:
+            status = Status("Running evaluation...", spinner="dots", speed=1.0, refresh_per_second=10)
+            live_log.update(status)
+
+            actual_tool_calls = None
+            if self.agent_response is not None:
+                messages = self.agent_response.messages
+            elif self.team_response is not None:
+                messages = self.team_response.messages or []
+                for member_response in self.team_response.member_responses:
+                    if member_response.messages is not None:
+                        messages += member_response.messages
+
+            for message in reversed(messages):  # type: ignore
+                if message.tool_calls:
+                    if actual_tool_calls is None:
+                        actual_tool_calls = message.tool_calls
+                    else:
+                        actual_tool_calls.append(message.tool_calls[0])  # type: ignore
+
+            failed_tool_calls = []
+            passed_tool_calls = []
+            if not actual_tool_calls:
+                failed_tool_calls = self.expected_tool_calls or []
+            else:
+                for tool_call in actual_tool_calls:  # type: ignore
+                    tool_name = tool_call.get("function", {}).get("name")
+                    if not tool_name:
+                        continue
+                    else:
+                        if tool_name not in self.expected_tool_calls:  # type: ignore
+                            failed_tool_calls.append(tool_call.get("function", {}).get("name"))
+                        else:
+                            passed_tool_calls.append(tool_call.get("function", {}).get("name"))
+
+            self.result = ReliabilityResult(
+                eval_status="PASSED" if len(failed_tool_calls) == 0 else "FAILED",
+                failed_tool_calls=failed_tool_calls,
+                passed_tool_calls=passed_tool_calls,
+            )
+
+        # Save result to file if requested
+        if self.file_path_to_save_results is not None and self.result is not None:
+            store_result_in_file(
+                file_path=self.file_path_to_save_results,
+                name=self.name,
+                eval_id=self.eval_id,
+                result=self.result,
+            )
+
+        # Print results if requested
+        if self.print_results or print_results:
+            self.result.print_eval(console)
+
+        # Log results to the Agno platform if requested
+        if self.db:
+            if self.agent_response is not None:
+                agent_id = self.agent_response.agent_id
+                team_id = None
+                model_id = self.agent_response.model
+                model_provider = self.agent_response.model_provider
+            elif self.team_response is not None:
+                agent_id = None
+                team_id = self.team_response.team_id
+                model_id = self.team_response.model
+                model_provider = self.team_response.model_provider
+
+            eval_input = {
+                "expected_tool_calls": self.expected_tool_calls,
+            }
+
+            log_eval_run(
+                db=self.db,
+                run_id=self.eval_id,  # type: ignore
+                run_data=asdict(self.result),
+                eval_type=EvalType.RELIABILITY,
+                name=self.name if self.name is not None else None,
+                agent_id=agent_id,
+                team_id=team_id,
+                model_id=model_id,
+                model_provider=model_provider,
+                eval_input=eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, create_eval_run_telemetry
+
+            create_eval_run_telemetry(
+                eval_run=EvalRunCreate(
+                    run_id=self.eval_id,
+                    eval_type=EvalType.RELIABILITY,
+                    data=self._get_telemetry_data(),
+                ),
+            )
+
+        logger.debug(f"*********** Evaluation End: {self.eval_id} ***********")
+        return self.result
+
+    async def arun(self, *, print_results: bool = False) -> Optional[ReliabilityResult]:
+        if self.agent_response is None and self.team_response is None:
+            raise ValueError("You need to provide 'agent_response' or 'team_response' to run the evaluation.")
+
+        if self.agent_response is not None and self.team_response is not None:
+            raise ValueError(
+                "You need to provide only one of 'agent_response' or 'team_response' to run the evaluation."
+            )
+
+        from rich.console import Console
+        from rich.live import Live
+        from rich.status import Status
+
+        # Add a spinner while running the evaluations
+        console = Console()
+        with Live(console=console, transient=True) as live_log:
+            status = Status("Running evaluation...", spinner="dots", speed=1.0, refresh_per_second=10)
+            live_log.update(status)
+
+            actual_tool_calls = None
+            if self.agent_response is not None:
+                messages = self.agent_response.messages
+            elif self.team_response is not None:
+                messages = self.team_response.messages or []
+                for member_response in self.team_response.member_responses:
+                    if member_response.messages is not None:
+                        messages += member_response.messages
+
+            for message in reversed(messages):  # type: ignore
+                if message.tool_calls:
+                    if actual_tool_calls is None:
+                        actual_tool_calls = message.tool_calls
+                    else:
+                        actual_tool_calls.append(message.tool_calls[0])  # type: ignore
+
+            failed_tool_calls = []
+            passed_tool_calls = []
+            for tool_call in actual_tool_calls:  # type: ignore
+                tool_name = tool_call.get("function", {}).get("name")
+                if not tool_name:
+                    continue
+                else:
+                    if tool_name not in self.expected_tool_calls:  # type: ignore
+                        failed_tool_calls.append(tool_call.get("function", {}).get("name"))
+                    else:
+                        passed_tool_calls.append(tool_call.get("function", {}).get("name"))
+
+            self.result = ReliabilityResult(
+                eval_status="PASSED" if len(failed_tool_calls) == 0 else "FAILED",
+                failed_tool_calls=failed_tool_calls,
+                passed_tool_calls=passed_tool_calls,
+            )
+
+        # Save result to file if requested
+        if self.file_path_to_save_results is not None and self.result is not None:
+            store_result_in_file(
+                file_path=self.file_path_to_save_results,
+                name=self.name,
+                eval_id=self.eval_id,
+                result=self.result,
+            )
+
+        # Print results if requested
+        if self.print_results or print_results:
+            self.result.print_eval(console)
+
+        # Log results to the Agno platform if requested
+        if self.db:
+            if self.agent_response is not None:
+                agent_id = self.agent_response.agent_id
+                team_id = None
+                model_id = self.agent_response.model
+                model_provider = self.agent_response.model_provider
+            elif self.team_response is not None:
+                agent_id = None
+                team_id = self.team_response.team_id
+                model_id = self.team_response.model
+                model_provider = self.team_response.model_provider
+
+            eval_input = {
+                "expected_tool_calls": self.expected_tool_calls,
+            }
+
+            await async_log_eval(
+                db=self.db,
+                run_id=self.eval_id,  # type: ignore
+                run_data=asdict(self.result),
+                eval_type=EvalType.RELIABILITY,
+                name=self.name if self.name is not None else None,
+                agent_id=agent_id,
+                team_id=team_id,
+                model_id=model_id,
+                model_provider=model_provider,
+                eval_input=eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, async_create_eval_run_telemetry
+
+            await async_create_eval_run_telemetry(
+                eval_run=EvalRunCreate(
+                    run_id=self.eval_id,
+                    eval_type=EvalType.RELIABILITY,
+                    data=self._get_telemetry_data(),
+                ),
+            )
+
+        logger.debug(f"*********** Evaluation End: {self.eval_id} ***********")
+        return self.result
+
+    def _get_telemetry_data(self) -> Dict[str, Any]:
+        """Get the telemetry data for the evaluation"""
+        return {
+            "team_id": self.team_response.team_id if self.team_response else None,
+            "agent_id": self.agent_response.agent_id if self.agent_response else None,
+            "model_id": self.agent_response.model if self.agent_response else None,
+            "model_provider": self.agent_response.model_provider if self.agent_response else None,
+        }

agno/eval/utils.py

@@ -0,0 +1,119 @@
+from dataclasses import asdict
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional, Union
+
+from agno.db.base import AsyncBaseDb, BaseDb
+from agno.db.schemas.evals import EvalRunRecord, EvalType
+from agno.utils.log import log_debug, logger
+
+if TYPE_CHECKING:
+    from agno.eval.accuracy import AccuracyResult
+    from agno.eval.performance import PerformanceResult
+    from agno.eval.reliability import ReliabilityResult
+
+
+def log_eval_run(
+    db: BaseDb,
+    run_id: str,
+    run_data: dict,
+    eval_type: EvalType,
+    eval_input: dict,
+    agent_id: Optional[str] = None,
+    model_id: Optional[str] = None,
+    model_provider: Optional[str] = None,
+    name: Optional[str] = None,
+    evaluated_component_name: Optional[str] = None,
+    team_id: Optional[str] = None,
+    workflow_id: Optional[str] = None,
+) -> None:
+    """Call the API to create an evaluation run."""
+
+    try:
+        db.create_eval_run(
+            EvalRunRecord(
+                run_id=run_id,
+                eval_type=eval_type,
+                eval_data=run_data,
+                eval_input=eval_input,
+                agent_id=agent_id,
+                model_id=model_id,
+                model_provider=model_provider,
+                name=name,
+                evaluated_component_name=evaluated_component_name,
+                team_id=team_id,
+                workflow_id=workflow_id,
+            )
+        )
+    except Exception as e:
+        log_debug(f"Could not create agent event: {e}")
+
+
+async def async_log_eval(
+    db: Union[BaseDb, AsyncBaseDb],
+    run_id: str,
+    run_data: dict,
+    eval_type: EvalType,
+    eval_input: dict,
+    agent_id: Optional[str] = None,
+    model_id: Optional[str] = None,
+    model_provider: Optional[str] = None,
+    name: Optional[str] = None,
+    evaluated_component_name: Optional[str] = None,
+    team_id: Optional[str] = None,
+    workflow_id: Optional[str] = None,
+) -> None:
+    """Call the API to create an evaluation run."""
+
+    try:
+        if isinstance(db, AsyncBaseDb):
+            await db.create_eval_run(
+                EvalRunRecord(
+                    run_id=run_id,
+                    eval_type=eval_type,
+                    eval_data=run_data,
+                    eval_input=eval_input,
+                    agent_id=agent_id,
+                    model_id=model_id,
+                    model_provider=model_provider,
+                    name=name,
+                    evaluated_component_name=evaluated_component_name,
+                    team_id=team_id,
+                    workflow_id=workflow_id,
+                )
+            )
+        else:
+            db.create_eval_run(
+                EvalRunRecord(
+                    run_id=run_id,
+                    eval_type=eval_type,
+                    eval_data=run_data,
+                    eval_input=eval_input,
+                    agent_id=agent_id,
+                    model_id=model_id,
+                    model_provider=model_provider,
+                    name=name,
+                    evaluated_component_name=evaluated_component_name,
+                    team_id=team_id,
+                    workflow_id=workflow_id,
+                )
+            )
+    except Exception as e:
+        log_debug(f"Could not create agent event: {e}")
+
+
+def store_result_in_file(
+    file_path: str,
+    result: Union["AccuracyResult", "PerformanceResult", "ReliabilityResult"],
+    eval_id: Optional[str] = None,
+    name: Optional[str] = None,
+):
+    """Store the given result in the given file path"""
+    try:
+        import json
+
+        fn_path = Path(file_path.format(name=name, eval_id=eval_id))
+        if not fn_path.parent.exists():
+            fn_path.parent.mkdir(parents=True, exist_ok=True)
+        fn_path.write_text(json.dumps(asdict(result), indent=4))
+    except Exception as e:
+        logger.warning(f"Failed to save result to file: {e}")

agno/exceptions.py

@@ -0,0 +1,161 @@
+from enum import Enum
+from typing import Any, Dict, List, Optional, Union
+
+from agno.models.message import Message
+
+
+class AgentRunException(Exception):
+    def __init__(
+        self,
+        exc,
+        user_message: Optional[Union[str, Message]] = None,
+        agent_message: Optional[Union[str, Message]] = None,
+        messages: Optional[List[Union[dict, Message]]] = None,
+        stop_execution: bool = False,
+    ):
+        super().__init__(exc)
+        self.user_message = user_message
+        self.agent_message = agent_message
+        self.messages = messages
+        self.stop_execution = stop_execution
+        self.type = "agent_run_error"
+        self.error_id = "agent_run_error"
+
+
+class RetryAgentRun(AgentRunException):
+    """Exception raised when a tool call should be retried."""
+
+    def __init__(
+        self,
+        exc,
+        user_message: Optional[Union[str, Message]] = None,
+        agent_message: Optional[Union[str, Message]] = None,
+        messages: Optional[List[Union[dict, Message]]] = None,
+    ):
+        super().__init__(
+            exc, user_message=user_message, agent_message=agent_message, messages=messages, stop_execution=False
+        )
+        self.error_id = "retry_agent_run_error"
+
+
+class StopAgentRun(AgentRunException):
+    """Exception raised when an agent should stop executing entirely."""
+
+    def __init__(
+        self,
+        exc,
+        user_message: Optional[Union[str, Message]] = None,
+        agent_message: Optional[Union[str, Message]] = None,
+        messages: Optional[List[Union[dict, Message]]] = None,
+    ):
+        super().__init__(
+            exc, user_message=user_message, agent_message=agent_message, messages=messages, stop_execution=True
+        )
+        self.error_id = "stop_agent_run_error"
+
+
+class RunCancelledException(Exception):
+    """Exception raised when a run is cancelled."""
+
+    def __init__(self, message: str = "Operation cancelled by user"):
+        super().__init__(message)
+        self.type = "run_cancelled_error"
+        self.error_id = "run_cancelled_error"
+
+
+class AgnoError(Exception):
+    """Exception raised when an internal error occurs."""
+
+    def __init__(self, message: str, status_code: int = 500):
+        super().__init__(message)
+        self.message = message
+        self.status_code = status_code
+        self.type = "agno_error"
+        self.error_id = "agno_error"
+
+    def __str__(self) -> str:
+        return str(self.message)
+
+
+class ModelProviderError(AgnoError):
+    """Exception raised when a model provider returns an error."""
+
+    def __init__(
+        self, message: str, status_code: int = 502, model_name: Optional[str] = None, model_id: Optional[str] = None
+    ):
+        super().__init__(message, status_code)
+        self.model_name = model_name
+        self.model_id = model_id
+
+        self.type = "model_provider_error"
+        self.error_id = "model_provider_error"
+
+
+class ModelRateLimitError(ModelProviderError):
+    """Exception raised when a model provider returns a rate limit error."""
+
+    def __init__(
+        self, message: str, status_code: int = 429, model_name: Optional[str] = None, model_id: Optional[str] = None
+    ):
+        super().__init__(message, status_code, model_name, model_id)
+        self.error_id = "model_rate_limit_error"
+
+
+class EvalError(Exception):
+    """Exception raised when an evaluation fails."""
+
+    pass
+
+
+class CheckTrigger(Enum):
+    """Enum for guardrail triggers."""
+
+    OFF_TOPIC = "off_topic"
+    INPUT_NOT_ALLOWED = "input_not_allowed"
+    OUTPUT_NOT_ALLOWED = "output_not_allowed"
+    VALIDATION_FAILED = "validation_failed"
+
+    PROMPT_INJECTION = "prompt_injection"
+    PII_DETECTED = "pii_detected"
+
+
+class InputCheckError(Exception):
+    """Exception raised when an input check fails."""
+
+    def __init__(
+        self,
+        message: str,
+        check_trigger: CheckTrigger = CheckTrigger.INPUT_NOT_ALLOWED,
+        additional_data: Optional[Dict[str, Any]] = None,
+    ):
+        super().__init__(message)
+        self.type = "input_check_error"
+        if isinstance(check_trigger, CheckTrigger):
+            self.error_id = check_trigger.value
+        else:
+            self.error_id = str(check_trigger)
+
+        self.message = message
+        self.check_trigger = check_trigger
+        self.additional_data = additional_data
+
+
+class OutputCheckError(Exception):
+    """Exception raised when an output check fails."""
+
+    def __init__(
+        self,
+        message: str,
+        check_trigger: CheckTrigger = CheckTrigger.OUTPUT_NOT_ALLOWED,
+        additional_data: Optional[Dict[str, Any]] = None,
+    ):
+        super().__init__(message)
+        self.type = "output_check_error"
+        if isinstance(check_trigger, CheckTrigger):
+            self.error_id = check_trigger.value
+        else:
+            self.error_id = str(check_trigger)
+
+        self.message = message
+        self.check_trigger = check_trigger
+        self.additional_data = additional_data