agno 1.8.1__py3-none-any.whl → 2.0.0__py3-none-any.whl

agno/eval/accuracy.py

@@ -1,14 +1,15 @@
 from dataclasses import asdict, dataclass, field
 from os import getenv
 from textwrap import dedent
-from typing import TYPE_CHECKING, Callable, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
 from uuid import uuid4
 
 from pydantic import BaseModel, Field
 
 from agno.agent import Agent
-from agno.api.schemas.evals import EvalType
-from agno.eval.utils import async_log_eval_run, log_eval_run, store_result_in_file
+from agno.db.base import BaseDb
+from agno.db.schemas.evals import EvalType
+from agno.eval.utils import async_log_eval, log_eval_run, store_result_in_file
 from agno.exceptions import EvalError
 from agno.models.base import Model
 from agno.team.team import Team
@@ -96,11 +97,18 @@ class AccuracyResult:
             title_justify="center",
         )
         summary_table.add_row("Number of Runs", f"{len(self.results)}")
-        summary_table.add_row("Average Score", f"{self.avg_score:.2f}")
-        summary_table.add_row("Mean Score", f"{self.mean_score:.2f}")
-        summary_table.add_row("Minimum Score", f"{self.min_score:.2f}")
-        summary_table.add_row("Maximum Score", f"{self.max_score:.2f}")
-        summary_table.add_row("Standard Deviation", f"{self.std_dev_score:.2f}")
+
+        if self.avg_score is not None:
+            summary_table.add_row("Average Score", f"{self.avg_score:.2f}")
+        if self.mean_score is not None:
+            summary_table.add_row("Mean Score", f"{self.mean_score:.2f}")
+        if self.min_score is not None:
+            summary_table.add_row("Minimum Score", f"{self.min_score:.2f}")
+        if self.max_score is not None:
+            summary_table.add_row("Maximum Score", f"{self.max_score:.2f}")
+        if self.std_dev_score is not None:
+            summary_table.add_row("Standard Deviation", f"{self.std_dev_score:.2f}")
+
         console.print(summary_table)
 
     def print_results(self, console: Optional["Console"] = None):
@@ -167,8 +175,13 @@ class AccuracyEval:
     file_path_to_save_results: Optional[str] = None
     # Enable debug logs
     debug_mode: bool = getenv("AGNO_DEBUG", "false").lower() == "true"
-    # Log the results to the Agno platform. On by default.
-    monitoring: bool = getenv("AGNO_MONITOR", "true").lower() == "true"
+    # The database to store Evaluation results
+    db: Optional[BaseDb] = None
+
+    # Telemetry settings
+    # telemetry=True logs minimal telemetry for analytics
+    # This helps us improve our Evals and provide better support
+    telemetry: bool = True
 
     def get_evaluator_agent(self) -> Agent:
         """Return the evaluator agent. If not provided, build it based on the evaluator fields and default instructions."""
@@ -235,7 +248,7 @@ You are an expert judge tasked with comparing the quality of an AI Agent’s out
 {additional_guidelines}{additional_context}
 Remember: You must only compare the agent_output to the expected_output. The expected_output is correct as it was provided by the user.
 """,
-            response_model=AccuracyAgentResponse,
+            output_schema=AccuracyAgentResponse,
             structured_outputs=True,
         )
 
@@ -344,9 +357,9 @@ Remember: You must only compare the agent_output to the expected_output. The exp
                 live_log.update(status)
 
                 if self.agent is not None:
-                    output = self.agent.run(message=eval_input).content
+                    output = self.agent.run(input=eval_input).content
                 elif self.team is not None:
-                    output = self.team.run(message=eval_input).content
+                    output = self.team.run(input=eval_input).content
 
                 if not output:
                     logger.error(f"Failed to generate a valid answer on iteration {i + 1}: {output}")
@@ -398,22 +411,31 @@ Remember: You must only compare the agent_output to the expected_output. The exp
         if self.print_summary or print_summary:
             self.result.print_summary(console)
 
-        # Log results to the Agno platform if requested
+        # Log results to the Agno DB if requested
         if self.agent is not None:
-            agent_id = self.agent.agent_id
+            agent_id = self.agent.id
             team_id = None
             model_id = self.agent.model.id if self.agent.model is not None else None
             model_provider = self.agent.model.provider if self.agent.model is not None else None
-            evaluated_entity_name = self.agent.name
+            evaluated_component_name = self.agent.name
         elif self.team is not None:
             agent_id = None
-            team_id = self.team.team_id
+            team_id = self.team.id
             model_id = self.team.model.id if self.team.model is not None else None
             model_provider = self.team.model.provider if self.team.model is not None else None
-            evaluated_entity_name = self.team.name
+            evaluated_component_name = self.team.name
+
+        if self.db:
+            log_eval_input = {
+                "additional_guidelines": self.additional_guidelines,
+                "additional_context": self.additional_context,
+                "num_iterations": self.num_iterations,
+                "expected_output": self.expected_output,
+                "input": self.input,
+            }
 
-        if self.monitoring:
             log_eval_run(
+                db=self.db,
                 run_id=self.eval_id,  # type: ignore
                 run_data=asdict(self.result),
                 eval_type=EvalType.ACCURACY,
@@ -422,7 +444,19 @@ Remember: You must only compare the agent_output to the expected_output. The exp
                 model_id=model_id,
                 model_provider=model_provider,
                 name=self.name if self.name is not None else None,
-                evaluated_entity_name=evaluated_entity_name,
+                evaluated_component_name=evaluated_component_name,
+                eval_input=log_eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, create_eval_run_telemetry
+
+            create_eval_run_telemetry(
+                eval_run=EvalRunCreate(
+                    run_id=self.eval_id,
+                    eval_type=EvalType.ACCURACY,
+                    data=self._get_telemetry_data(),
+                ),
             )
 
         logger.debug(f"*********** Evaluation {self.eval_id} Finished ***********")
@@ -464,10 +498,10 @@ Remember: You must only compare the agent_output to the expected_output. The exp
                 live_log.update(status)
 
                 if self.agent is not None:
-                    response = await self.agent.arun(message=eval_input)
+                    response = await self.agent.arun(input=eval_input)
                     output = response.content
                 elif self.team is not None:
-                    response = await self.team.arun(message=eval_input)
+                    response = await self.team.arun(input=eval_input)  # type: ignore
                     output = response.content
 
                 if not output:
@@ -520,19 +554,48 @@ Remember: You must only compare the agent_output to the expected_output. The exp
         if self.print_summary or print_summary:
             self.result.print_summary(console)
 
-        # Log results to the Agno platform if requested
-        if self.monitoring:
-            await async_log_eval_run(
+        if self.agent is not None:
+            agent_id = self.agent.id
+            team_id = None
+            model_id = self.agent.model.id if self.agent.model is not None else None
+            model_provider = self.agent.model.provider if self.agent.model is not None else None
+            evaluated_component_name = self.agent.name
+        elif self.team is not None:
+            agent_id = None
+            team_id = self.team.id
+            model_id = self.team.model.id if self.team.model is not None else None
+            model_provider = self.team.model.provider if self.team.model is not None else None
+            evaluated_component_name = self.team.name
+
+        # Log results to the Agno DB if requested
+        if self.db:
+            log_eval_input = {
+                "additional_guidelines": self.additional_guidelines,
+                "additional_context": self.additional_context,
+                "num_iterations": self.num_iterations,
+                "expected_output": self.expected_output,
+                "input": self.input,
+            }
+            await async_log_eval(
+                db=self.db,
                 run_id=self.eval_id,  # type: ignore
                 run_data=asdict(self.result),
                 eval_type=EvalType.ACCURACY,
-                agent_id=self.agent.agent_id if self.agent is not None else None,
-                model_id=self.agent.model.id if self.agent is not None and self.agent.model is not None else None,
-                model_provider=self.agent.model.provider
-                if self.agent is not None and self.agent.model is not None
-                else None,
+                agent_id=agent_id,
+                model_id=model_id,
+                model_provider=model_provider,
                 name=self.name if self.name is not None else None,
-                evaluated_entity_name=self.agent.name if self.agent is not None else None,
+                evaluated_component_name=evaluated_component_name,
+                team_id=team_id,
+                workflow_id=None,
+                eval_input=log_eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, async_create_eval_run_telemetry
+
+            await async_create_eval_run_telemetry(
+                eval_run=EvalRunCreate(run_id=self.eval_id, eval_type=EvalType.ACCURACY),
             )
 
         logger.debug(f"*********** Evaluation {self.eval_id} Finished ***********")
@@ -596,28 +659,37 @@ Remember: You must only compare the agent_output to the expected_output. The exp
                     eval_id=self.eval_id,
                     result=self.result,
                 )
-        # Log results to the Agno platform if requested
-        if self.monitoring:
+        # Log results to the Agno DB if requested
+        if self.db:
             if self.agent is not None:
-                agent_id = self.agent.agent_id
+                agent_id = self.agent.id
                 team_id = None
                 model_id = self.agent.model.id if self.agent.model is not None else None
                 model_provider = self.agent.model.provider if self.agent.model is not None else None
-                evaluated_entity_name = self.agent.name
+                evaluated_component_name = self.agent.name
             elif self.team is not None:
                 agent_id = None
-                team_id = self.team.team_id
+                team_id = self.team.id
                 model_id = self.team.model.id if self.team.model is not None else None
                 model_provider = self.team.model.provider if self.team.model is not None else None
-                evaluated_entity_name = self.team.name
+                evaluated_component_name = self.team.name
             else:
                 agent_id = None
                 team_id = None
                 model_id = None
                 model_provider = None
-                evaluated_entity_name = None
+                evaluated_component_name = None
+
+            log_eval_input = {
+                "additional_guidelines": self.additional_guidelines,
+                "additional_context": self.additional_context,
+                "num_iterations": self.num_iterations,
+                "expected_output": self.expected_output,
+                "input": self.input,
+            }
 
             log_eval_run(
+                db=self.db,
                 run_id=self.eval_id,  # type: ignore
                 run_data=asdict(self.result),
                 eval_type=EvalType.ACCURACY,
@@ -626,7 +698,20 @@ Remember: You must only compare the agent_output to the expected_output. The exp
                 team_id=team_id,
                 model_id=model_id,
                 model_provider=model_provider,
-                evaluated_entity_name=evaluated_entity_name,
+                evaluated_component_name=evaluated_component_name,
+                workflow_id=None,
+                eval_input=log_eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, create_eval_run_telemetry
+
+            create_eval_run_telemetry(
+                eval_run=EvalRunCreate(
+                    run_id=self.eval_id,
+                    eval_type=EvalType.ACCURACY,
+                    data=self._get_telemetry_data(),
+                ),
             )
 
         logger.debug(f"*********** Evaluation End: {self.eval_id} ***********")
@@ -690,22 +775,31 @@ Remember: You must only compare the agent_output to the expected_output. The exp
                     eval_id=self.eval_id,
                     result=self.result,
                 )
-        # Log results to the Agno platform if requested
-        if self.monitoring:
+        # Log results to the Agno DB if requested
+        if self.db:
             if self.agent is not None:
-                agent_id = self.agent.agent_id
+                agent_id = self.agent.id
                 team_id = None
                 model_id = self.agent.model.id if self.agent.model is not None else None
                 model_provider = self.agent.model.provider if self.agent.model is not None else None
-                evaluated_entity_name = self.agent.name
+                evaluated_component_name = self.agent.name
             elif self.team is not None:
                 agent_id = None
-                team_id = self.team.team_id
+                team_id = self.team.id
                 model_id = self.team.model.id if self.team.model is not None else None
                 model_provider = self.team.model.provider if self.team.model is not None else None
-                evaluated_entity_name = self.team.name
-
-            await async_log_eval_run(
+                evaluated_component_name = self.team.name
+
+            log_eval_input = {
+                "additional_guidelines": self.additional_guidelines,
+                "additional_context": self.additional_context,
+                "num_iterations": self.num_iterations,
+                "expected_output": self.expected_output,
+                "input": self.input,
+            }
+
+            await async_log_eval(
+                db=self.db,
                 run_id=self.eval_id,  # type: ignore
                 run_data=asdict(self.result),
                 eval_type=EvalType.ACCURACY,
@@ -714,8 +808,20 @@ Remember: You must only compare the agent_output to the expected_output. The exp
                 team_id=team_id,
                 model_id=model_id,
                 model_provider=model_provider,
-                evaluated_entity_name=evaluated_entity_name,
+                evaluated_component_name=evaluated_component_name,
+                workflow_id=None,
+                eval_input=log_eval_input,
             )
 
         logger.debug(f"*********** Evaluation End: {self.eval_id} ***********")
         return self.result
+
+    def _get_telemetry_data(self) -> Dict[str, Any]:
+        """Get the telemetry data for the evaluation"""
+        return {
+            "agent_id": self.agent.id if self.agent else None,
+            "team_id": self.team.id if self.team else None,
+            "model_id": self.agent.model.id if self.agent and self.agent.model else None,
+            "model_provider": self.agent.model.provider if self.agent and self.agent.model else None,
+            "num_iterations": self.num_iterations,
+        }

agno/eval/performance.py

@@ -3,11 +3,12 @@ import gc
 import tracemalloc
 from dataclasses import dataclass, field
 from os import getenv
-from typing import TYPE_CHECKING, Callable, List, Optional
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
 from uuid import uuid4
 
-from agno.api.schemas.evals import EvalType
-from agno.eval.utils import async_log_eval_run, log_eval_run, store_result_in_file
+from agno.db.base import BaseDb
+from agno.db.schemas.evals import EvalType
+from agno.eval.utils import async_log_eval, log_eval_run, store_result_in_file
 from agno.utils.log import log_debug, set_log_level_to_debug, set_log_level_to_info
 from agno.utils.timer import Timer
 
@@ -195,7 +196,7 @@ class PerformanceEval:
     # Evaluation UUID
     eval_id: str = field(default_factory=lambda: str(uuid4()))
     # Number of warm-up runs (not included in final stats)
-    warmup_runs: int = 10
+    warmup_runs: Optional[int] = 10
     # Number of measured iterations
     num_iterations: int = 50
     # Result of the evaluation
@@ -210,12 +211,23 @@ class PerformanceEval:
     # Number of memory allocations to track
     top_n_memory_allocations: int = 5
 
+    # Agent and Team information
+    agent_id: Optional[str] = None
+    team_id: Optional[str] = None
+    model_id: Optional[str] = None
+    model_provider: Optional[str] = None
+
     # If set, results will be saved in the given file path
     file_path_to_save_results: Optional[str] = None
     # Enable debug logs
     debug_mode: bool = getenv("AGNO_DEBUG", "false").lower() == "true"
-    # Log the results to the Agno platform. On by default.
-    monitoring: bool = getenv("AGNO_MONITOR", "true").lower() == "true"
+    # The database to store Evaluation results
+    db: Optional[BaseDb] = None
+
+    # Telemetry settings
+    # telemetry=True logs minimal telemetry for analytics
+    # This helps us improve our Evals and provide better support
+    telemetry: bool = True
 
     def _measure_time(self) -> float:
         """Measure execution time for a single run."""
@@ -495,12 +507,13 @@ class PerformanceEval:
         console = Console()
         with Live(console=console, transient=True) as live_log:
             # 1. Do optional warm-up runs.
-            for i in range(self.warmup_runs):
-                status = Status(f"Warm-up run {i + 1}/{self.warmup_runs}...", spinner="dots", speed=1.0)
-                live_log.update(status)
-                self.func()  # Simply run the function without measuring
-                self._set_log_level()  # Set log level incase function changed it
-                status.stop()
+            if self.warmup_runs is not None:
+                for i in range(self.warmup_runs):
+                    status = Status(f"Warm-up run {i + 1}/{self.warmup_runs}...", spinner="dots", speed=1.0)
+                    live_log.update(status)
+                    self.func()  # Simply run the function without measuring
+                    self._set_log_level()  # Set log level incase function changed it
+                    status.stop()
 
             # 2. Measure runtime
             if self.measure_runtime:
@@ -570,13 +583,33 @@ class PerformanceEval:
             self.result.print_summary(console, measure_memory=self.measure_memory, measure_runtime=self.measure_runtime)
 
         # 7. Log results to the Agno platform if requested
-        if self.monitoring:
+        if self.db:
+            eval_input = {
+                "num_iterations": self.num_iterations,
+                "warmup_runs": self.warmup_runs,
+            }
+
             log_eval_run(
+                db=self.db,
                 run_id=self.eval_id,  # type: ignore
                 run_data=self._parse_eval_run_data(),
                 eval_type=EvalType.PERFORMANCE,
                 name=self.name if self.name is not None else None,
-                evaluated_entity_name=self.func.__name__,
+                evaluated_component_name=self.func.__name__,
+                agent_id=self.agent_id,
+                team_id=self.team_id,
+                model_id=self.model_id,
+                model_provider=self.model_provider,
+                eval_input=eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, create_eval_run_telemetry
+
+            create_eval_run_telemetry(
+                eval_run=EvalRunCreate(
+                    run_id=self.eval_id, eval_type=EvalType.PERFORMANCE, data=self._get_telemetry_data()
+                ),
             )
 
         log_debug(f"*********** Evaluation End: {self.eval_id} ***********")
@@ -617,12 +650,13 @@ class PerformanceEval:
         console = Console()
         with Live(console=console, transient=True) as live_log:
             # 1. Do optional warm-up runs.
-            for i in range(self.warmup_runs):
-                status = Status(f"Warm-up run {i + 1}/{self.warmup_runs}...", spinner="dots", speed=1.0)
-                live_log.update(status)
-                await self.func()  # Simply run the function without measuring
-                self._set_log_level()  # Set log level incase function changed it
-                status.stop()
+            if self.warmup_runs is not None:
+                for i in range(self.warmup_runs):
+                    status = Status(f"Warm-up run {i + 1}/{self.warmup_runs}...", spinner="dots", speed=1.0)
+                    live_log.update(status)
+                    await self.func()  # Simply run the function without measuring
+                    self._set_log_level()  # Set log level incase function changed it
+                    status.stop()
 
             # 2. Measure runtime
             if self.measure_runtime:
@@ -692,14 +726,45 @@ class PerformanceEval:
             self.result.print_summary(console, measure_memory=self.measure_memory, measure_runtime=self.measure_runtime)
 
         # 7. Log results to the Agno platform if requested
-        if self.monitoring:
-            await async_log_eval_run(
+        if self.db:
+            eval_input = {
+                "num_iterations": self.num_iterations,
+                "warmup_runs": self.warmup_runs,
+            }
+
+            await async_log_eval(
+                db=self.db,
                 run_id=self.eval_id,  # type: ignore
                 run_data=self._parse_eval_run_data(),
                 eval_type=EvalType.PERFORMANCE,
                 name=self.name if self.name is not None else None,
-                evaluated_entity_name=self.func.__name__,
+                evaluated_component_name=self.func.__name__,
+                agent_id=self.agent_id,
+                team_id=self.team_id,
+                model_id=self.model_id,
+                model_provider=self.model_provider,
+                eval_input=eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, async_create_eval_run_telemetry
+
+            await async_create_eval_run_telemetry(
+                eval_run=EvalRunCreate(
+                    run_id=self.eval_id, eval_type=EvalType.PERFORMANCE, data=self._get_telemetry_data()
+                ),
             )
 
         log_debug(f"*********** Evaluation End: {self.eval_id} ***********")
         return self.result
+
+    def _get_telemetry_data(self) -> Dict[str, Any]:
+        """Get the telemetry data for the evaluation"""
+        return {
+            "model_id": self.model_id,
+            "model_provider": self.model_provider,
+            "num_iterations": self.num_iterations,
+            "warmup_runs": self.warmup_runs,
+            "measure_memory": self.measure_memory,
+            "measure_runtime": self.measure_runtime,
+        }