agno 0.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

agno/eval/accuracy.py

@@ -1,14 +1,19 @@
 from dataclasses import asdict, dataclass, field
 from os import getenv
-from pathlib import Path
-from typing import TYPE_CHECKING, Callable, List, Optional, Union
+from textwrap import dedent
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
 from uuid import uuid4
 
 from pydantic import BaseModel, Field
 
-from agno.agent import Agent, RunResponse
+from agno.agent import Agent
+from agno.db.base import AsyncBaseDb, BaseDb
+from agno.db.schemas.evals import EvalType
+from agno.eval.utils import async_log_eval, log_eval_run, store_result_in_file
+from agno.exceptions import EvalError
 from agno.models.base import Model
-from agno.utils.log import logger, set_log_level_to_debug, set_log_level_to_info
+from agno.team.team import Team
+from agno.utils.log import log_error, logger, set_log_level_to_debug, set_log_level_to_info
 
 if TYPE_CHECKING:
     from rich.console import Console
@@ -21,9 +26,9 @@ class AccuracyAgentResponse(BaseModel):
 
 @dataclass
 class AccuracyEvaluation:
-    question: str
-    answer: str
-    expected_answer: str
+    input: str
+    output: str
+    expected_output: str
     score: int
     reason: str
 
@@ -44,9 +49,9 @@ class AccuracyEvaluation:
             title_style="bold sky_blue1",
             title_justify="center",
         )
-        results_table.add_row("Question", self.question)
-        results_table.add_row("Answer", self.answer)
-        results_table.add_row("Expected Answer", self.expected_answer)
+        results_table.add_row("Input", self.input)
+        results_table.add_row("Output", self.output)
+        results_table.add_row("Expected Output", self.expected_output)
         results_table.add_row("Accuracy Score", f"{str(self.score)}/10")
         results_table.add_row("Accuracy Reason", Markdown(self.reason))
         console.print(results_table)
@@ -92,11 +97,18 @@ class AccuracyResult:
             title_justify="center",
         )
         summary_table.add_row("Number of Runs", f"{len(self.results)}")
-        summary_table.add_row("Average Score", f"{self.avg_score:.2f}")
-        summary_table.add_row("Mean Score", f"{self.mean_score:.2f}")
-        summary_table.add_row("Minimum Score", f"{self.min_score:.2f}")
-        summary_table.add_row("Maximum Score", f"{self.max_score:.2f}")
-        summary_table.add_row("Standard Deviation", f"{self.std_dev_score:.2f}")
+
+        if self.avg_score is not None:
+            summary_table.add_row("Average Score", f"{self.avg_score:.2f}")
+        if self.mean_score is not None:
+            summary_table.add_row("Mean Score", f"{self.mean_score:.2f}")
+        if self.min_score is not None:
+            summary_table.add_row("Minimum Score", f"{self.min_score:.2f}")
+        if self.max_score is not None:
+            summary_table.add_row("Maximum Score", f"{self.max_score:.2f}")
+        if self.std_dev_score is not None:
+            summary_table.add_row("Standard Deviation", f"{self.std_dev_score:.2f}")
+
         console.print(summary_table)
 
     def print_results(self, console: Optional["Console"] = None):
@@ -116,9 +128,9 @@ class AccuracyResult:
             title_justify="center",
         )
         for result in self.results:
-            results_table.add_row("Question", result.question)
-            results_table.add_row("Answer", result.answer)
-            results_table.add_row("Expected Answer", result.expected_answer)
+            results_table.add_row("Input", result.input)
+            results_table.add_row("Output", result.output)
+            results_table.add_row("Expected Output", result.expected_output)
             results_table.add_row("Accuracy Score", f"{str(result.score)}/10")
             results_table.add_row("Accuracy Reason", result.reason)
         console.print(results_table)
@@ -126,61 +138,53 @@ class AccuracyResult:
 
 @dataclass
 class AccuracyEval:
-    """Evaluate the accuracy of an agent's answer."""
+    """Interface to evaluate the accuracy of an Agent or Team, given a prompt and expected answer"""
+
+    # Input to evaluate
+    input: Union[str, Callable]
+    # Expected answer to the input
+    expected_output: Union[str, Callable]
+    # Agent to evaluate
+    agent: Optional[Agent] = None
+    # Team to evaluate
+    team: Optional[Team] = None
 
     # Evaluation name
     name: Optional[str] = None
-    # Evaluation UUID (autogenerated if not set)
-    eval_id: Optional[str] = None
+    # Evaluation UUID
+    eval_id: str = field(default_factory=lambda: str(uuid4()))
+    # Number of iterations to run
+    num_iterations: int = 1
+    # Result of the evaluation
+    result: Optional[AccuracyResult] = None
 
-    # Model used to evaluate the answer
+    # Model for the evaluator agent
     model: Optional[Model] = None
-
-    # Evaluate an Agent
-    agent: Optional[Agent] = None
-    # Question to evaluate (can also be provided with the run method)
-    question: Optional[Union[str, Callable]] = None
-    # Answer to evaluate (can also be provided with the run method)
-    answer: Optional[Union[str, Callable]] = None
-    # Expected Answer for the question (can also be provided with the run method)
-    expected_answer: Optional[Union[str, Callable]] = None
-
+    # Agent used to evaluate the answer
     evaluator_agent: Optional[Agent] = None
     # Guidelines for the evaluator agent
-    evaluator_guidelines: Optional[List[str]] = None
+    additional_guidelines: Optional[Union[str, List[str]]] = None
     # Additional context to the evaluator agent
-    evaluator_context: Optional[str] = None
-
-    # Number of iterations to run
-    num_iterations: int = 3
-    # Result of the evaluation
-    result: Optional[AccuracyResult] = None
+    additional_context: Optional[str] = None
 
     # Print summary of results
     print_summary: bool = False
     # Print detailed results
     print_results: bool = False
-    # Save the result to a file
-    save_result_to_file: Optional[str] = None
-
-    # debug_mode=True enables debug logs
-    debug_mode: bool = False
-
-    def set_eval_id(self) -> str:
-        if self.eval_id is None:
-            self.eval_id = str(uuid4())
-        logger.debug(f"*********** Evaluation ID: {self.eval_id} ***********")
-        return self.eval_id
-
-    def set_debug_mode(self) -> None:
-        if self.debug_mode or getenv("AGNO_DEBUG", "false").lower() == "true":
-            self.debug_mode = True
-            set_log_level_to_debug()
-            logger.debug("Debug logs enabled")
-        else:
-            set_log_level_to_info()
-
-    def get_evaluator_agent(self, question: str, expected_answer: str) -> Agent:
+    # If set, results will be saved in the given file path
+    file_path_to_save_results: Optional[str] = None
+    # Enable debug logs
+    debug_mode: bool = getenv("AGNO_DEBUG", "false").lower() == "true"
+    # The database to store Evaluation results
+    db: Optional[Union[BaseDb, AsyncBaseDb]] = None
+
+    # Telemetry settings
+    # telemetry=True logs minimal telemetry for analytics
+    # This helps us improve our Evals and provide better support
+    telemetry: bool = True
+
+    def get_evaluator_agent(self) -> Agent:
+        """Return the evaluator agent. If not provided, build it based on the evaluator fields and default instructions."""
         if self.evaluator_agent is not None:
             return self.evaluator_agent
 
@@ -189,269 +193,655 @@ class AccuracyEval:
             try:
                 from agno.models.openai import OpenAIChat
 
-                model = OpenAIChat(id="gpt-4o-mini")
+                model = OpenAIChat(id="o4-mini")
             except (ModuleNotFoundError, ImportError) as e:
                 logger.exception(e)
-                logger.error(
+                raise EvalError(
                     "Agno uses `openai` as the default model provider. Please run `pip install openai` to use the default evaluator."
                 )
-                exit(1)
 
-        evaluator_guidelines = ""
-        if self.evaluator_guidelines is not None and len(self.evaluator_guidelines) > 0:
-            evaluator_guidelines = "\n## Guidelines for the Agent's answer:\n"
-            evaluator_guidelines += "\n- ".join(self.evaluator_guidelines)
-            evaluator_guidelines += "\n"
-
-        evaluator_context = ""
-        if self.evaluator_context is not None and len(self.evaluator_context) > 0:
-            evaluator_context = "## Additional Context:\n"
-            evaluator_context += self.evaluator_context
-            evaluator_context += "\n"
+        additional_guidelines = ""
+        if self.additional_guidelines is not None:
+            additional_guidelines = "\n## Additional Guidelines\n"
+            if isinstance(self.additional_guidelines, str):
+                additional_guidelines += self.additional_guidelines
+            else:
+                additional_guidelines += "\n- ".join(self.additional_guidelines)
+            additional_guidelines += "\n"
+
+        additional_context = ""
+        if self.additional_context is not None and len(self.additional_context) > 0:
+            additional_context = "\n## Additional Context\n"
+            additional_context += self.additional_context
+            additional_context += "\n"
 
         return Agent(
-            model=OpenAIChat(id="gpt-4o-mini"),
+            model=model,
             description=f"""\
-You are an Agent Evaluator tasked with assessing the accuracy of an AI Agent's answer compared to an expected answer for a given question.
-Your task is to provide a detailed analysis and assign a score on a scale of 1 to 10, where 10 indicates a perfect match to the expected answer.
-
-## Question:
-{question}
-
-## Expected Answer:
-{expected_answer}
-
-## Evaluation Criteria:
-1. Accuracy of information
-2. Completeness of the answer
-3. Relevance to the question
-4. Use of key concepts and ideas
-5. Overall structure and clarity of presentation
-{evaluator_guidelines}{evaluator_context}
-## Instructions:
-1. Carefully compare the AI Agent's answer to the expected answer.
-2. Provide a detailed analysis, highlighting:
-   - Specific similarities and differences
-   - Key points included or missed
-   - Any inaccuracies or misconceptions
-3. Explicitly reference the evaluation criteria and any provided guidelines in your reasoning.
-4. Assign a score from 1 to 10 (use only whole numbers) based on the following scale:
-   1-2: Completely incorrect or irrelevant
-   3-4: Major inaccuracies or missing crucial information
-   5-6: Partially correct, but with significant omissions or errors
+You are an expert judge tasked with comparing the quality of an AI Agent’s output to a user-provided expected output. You must assume the expected_output is correct - even if you personally disagree.
+
+## Evaluation Inputs
+- agent_input: The original task or query given to the Agent.
+- expected_output: The correct response to the task (provided by the user).
+    - NOTE: You must assume the expected_output is correct - even if you personally disagree.
+- agent_output: The response generated by the Agent.
+
+## Evaluation Criteria
+- Accuracy: How closely does the agent_output match the expected_output?
+- Completeness: Does the agent_output include all the key elements of the expected_output?
+
+## Instructions
+1. Compare the agent_output only to the expected_output, not what you think the expected_output should be.
+2. Do not judge the correctness of the expected_output itself. Your role is only to compare the two outputs, the user provided expected_output is correct.
+3. Follow the additional guidelines if provided.
+4. Provide a detailed analysis including:
+    - Specific similarities and differences
+    - Important points included or omitted
+    - Any inaccuracies, paraphrasing errors, or structural differences
+5. Reference the criteria explicitly in your reasoning.
+6. Assign a score from 1 to 10 (whole numbers only):
+   1-2: Completely incorrect or irrelevant.
+   3-4: Major inaccuracies or missing key information.
+   5-6: Partially correct, but with significant issues.
    7-8: Mostly accurate and complete, with minor issues
-   9-10: Highly accurate and complete, matching the expected answer closely
-
-Your evaluation should be objective, thorough, and well-reasoned. Provide specific examples from both answers to support your assessment.""",
-            response_model=AccuracyAgentResponse,
+   9-10: Highly accurate and complete, matching the expected answer and given guidelines closely.
+{additional_guidelines}{additional_context}
+Remember: You must only compare the agent_output to the expected_output. The expected_output is correct as it was provided by the user.
+""",
+            output_schema=AccuracyAgentResponse,
             structured_outputs=True,
         )
 
-    def get_question_to_evaluate(self, question: Optional[Union[str, Callable]] = None) -> Optional[str]:
-        """Get the question to evaluate."""
-        try:
-            # Get question from the run method
-            if question is not None:
-                if isinstance(question, str):
-                    return question
-                elif callable(question):
-                    _question = question()
-                    if isinstance(_question, str):
-                        return _question
-                    else:
-                        logger.error("Question is not a string")
-                else:
-                    logger.error("Question is not a string or callable")
-
-            # Get the question from the eval
-            if self.question is not None:
-                if isinstance(self.question, str):
-                    return self.question
-                elif callable(self.question):
-                    _question = self.question()
-                    if isinstance(_question, str):
-                        return _question
-                    else:
-                        logger.error("Question is not a string")
-                else:
-                    logger.error("Question is not a string or callable")
-        except Exception as e:
-            logger.error(f"Failed to get question to evaluate: {e}")
-        return None
-
-    def get_answer_to_evaluate(
-        self, question: str, answer: Optional[Union[str, Callable]] = None
-    ) -> Optional[RunResponse]:
-        """Get the answer to evaluate.
-
-        Priority:
-        1. Answer provided with the run method
-        2. Answer provided with the eval
-        3. Answer from the agent
-        """
+    def get_eval_expected_output(self) -> str:
+        """Return the eval expected answer. If it is a callable, call it and return the resulting string"""
+        if callable(self.expected_output):
+            _output = self.expected_output()
+            if isinstance(_output, str):
+                return _output
+            else:
+                raise EvalError(f"The expected output needs to be or return a string, but it returned: {type(_output)}")
+        return self.expected_output
+
+    def get_eval_input(self) -> str:
+        """Return the evaluation input. If it is a callable, call it and return the resulting string"""
+        if callable(self.input):
+            _input = self.input()
+            if isinstance(_input, str):
+                return _input
+            else:
+                raise EvalError(f"The eval input needs to be or return a string, but it returned: {type(_input)}")
+        return self.input
+
+    def evaluate_answer(
+        self,
+        input: str,
+        evaluator_agent: Agent,
+        evaluation_input: str,
+        evaluator_expected_output: str,
+        agent_output: str,
+    ) -> Optional[AccuracyEvaluation]:
+        """Orchestrate the evaluation process."""
         try:
-            # Get answer from the run method
-            if answer is not None:
-                if isinstance(answer, str):
-                    return RunResponse(content=answer)
-                elif callable(answer):
-                    _answer = answer()
-                    if isinstance(_answer, str):
-                        return RunResponse(content=_answer)
-                    else:
-                        logger.error("Answer is not a string")
-                else:
-                    logger.error("Answer is not a string or callable")
-
-            # Get answer from the eval
-            if self.answer is not None:
-                if isinstance(self.answer, str):
-                    return RunResponse(content=self.answer)
-                elif callable(self.answer):
-                    _answer = self.answer()
-                    if isinstance(_answer, str):
-                        return RunResponse(content=_answer)
-                    else:
-                        logger.error("Answer is not a string")
-                else:
-                    logger.error("Answer is not a string or callable")
-
-            # Get answer from the agent
-            if self.agent is not None and question is not None:
-                logger.debug("Getting answer from agent")
-                return self.agent.run(question)
+            response = evaluator_agent.run(evaluation_input, stream=False)
+            accuracy_agent_response = response.content
+            if accuracy_agent_response is None or not isinstance(accuracy_agent_response, AccuracyAgentResponse):
+                raise EvalError(f"Evaluator Agent returned an invalid response: {accuracy_agent_response}")
+            return AccuracyEvaluation(
+                input=input,
+                output=agent_output,
+                expected_output=evaluator_expected_output,
+                score=accuracy_agent_response.accuracy_score,
+                reason=accuracy_agent_response.accuracy_reason,
+            )
         except Exception as e:
-            logger.error(f"Failed to get answer to evaluate: {e}")
-        return None
+            logger.exception(f"Failed to evaluate accuracy: {e}")
+            return None
 
-    def get_expected_answer_to_evaluate(self, expected_answer: Optional[Union[str, Callable]] = None) -> Optional[str]:
-        """Get the expected answer to evaluate."""
+    async def aevaluate_answer(
+        self,
+        input: str,
+        evaluator_agent: Agent,
+        evaluation_input: str,
+        evaluator_expected_output: str,
+        agent_output: str,
+    ) -> Optional[AccuracyEvaluation]:
+        """Orchestrate the evaluation process asynchronously."""
         try:
-            # Get expected_answer from the run method
-            if expected_answer is not None:
-                if isinstance(expected_answer, str):
-                    return expected_answer
-                elif callable(expected_answer):
-                    _expected_answer = expected_answer()
-                    if isinstance(_expected_answer, str):
-                        return _expected_answer
-                    else:
-                        logger.error("Expected Answer is not a string")
-                else:
-                    logger.error("Expected Answer is not a string or callable")
-
-            # Get the expected_answer from the eval
-            if self.expected_answer is not None:
-                if isinstance(self.expected_answer, str):
-                    return self.expected_answer
-                elif callable(self.expected_answer):
-                    _expected_answer = self.expected_answer()
-                    if isinstance(_expected_answer, str):
-                        return _expected_answer
-                    else:
-                        logger.error("Expected Answer is not a string")
-                else:
-                    logger.error("Expected Answer is not a string or callable")
+            response = await evaluator_agent.arun(evaluation_input, stream=False)
+            accuracy_agent_response = response.content
+            if accuracy_agent_response is None or not isinstance(accuracy_agent_response, AccuracyAgentResponse):
+                raise EvalError(f"Evaluator Agent returned an invalid response: {accuracy_agent_response}")
+            return AccuracyEvaluation(
+                input=input,
+                output=agent_output,
+                expected_output=evaluator_expected_output,
+                score=accuracy_agent_response.accuracy_score,
+                reason=accuracy_agent_response.accuracy_reason,
+            )
         except Exception as e:
-            logger.error(f"Failed to get expected answer to evaluate: {e}")
-        return None
+            logger.exception(f"Failed to evaluate accuracy asynchronously: {e}")
+            return None
 
     def run(
         self,
         *,
-        question: Optional[Union[str, Callable]] = None,
-        expected_answer: Optional[Union[str, Callable]] = None,
-        answer: Optional[Union[str, Callable]] = None,
         print_summary: bool = True,
         print_results: bool = True,
     ) -> Optional[AccuracyResult]:
+        if isinstance(self.db, AsyncBaseDb):
+            raise ValueError("run() is not supported with an async DB. Please use arun() instead.")
+
+        if self.agent is None and self.team is None:
+            logger.error("You need to provide one of 'agent' or 'team' to run the evaluation.")
+            return None
+
+        if self.agent is not None and self.team is not None:
+            logger.error("Provide only one of 'agent' or 'team' to run the evaluation.")
+            return None
+
         from rich.console import Console
         from rich.live import Live
         from rich.status import Status
 
-        self.set_eval_id()
-        self.set_debug_mode()
+        set_log_level_to_debug() if self.debug_mode else set_log_level_to_info()
+
         self.result = AccuracyResult()
-        self.print_results = print_results
-        self.print_summary = print_summary
 
-        question_to_evaluate: Optional[str] = self.get_question_to_evaluate(question=question)
-        if question_to_evaluate is None:
-            logger.error("No Question to evaluate.")
+        logger.debug(f"************ Evaluation Start: {self.eval_id} ************")
+
+        # Add a spinner while running the evaluations
+        console = Console()
+        with Live(console=console, transient=True) as live_log:
+            evaluator_agent = self.get_evaluator_agent()
+            eval_input = self.get_eval_input()
+            eval_expected_output = self.get_eval_expected_output()
+
+            for i in range(self.num_iterations):
+                status = Status(f"Running evaluation {i + 1}...", spinner="dots", speed=1.0, refresh_per_second=10)
+                live_log.update(status)
+
+                agent_session_id = f"eval_{self.eval_id}_{i + 1}"
+
+                if self.agent is not None:
+                    agent_response = self.agent.run(input=eval_input, session_id=agent_session_id, stream=False)
+                    output = agent_response.content
+                elif self.team is not None:
+                    team_response = self.team.run(input=eval_input, session_id=agent_session_id, stream=False)
+                    output = team_response.content
+
+                if not output:
+                    logger.error(f"Failed to generate a valid answer on iteration {i + 1}: {output}")
+                    continue
+
+                evaluation_input = dedent(f"""\
+                    <agent_input>
+                    {eval_input}
+                    </agent_input>
+
+                    <expected_output>
+                    {eval_expected_output}
+                    </expected_output>
+
+                    <agent_output>
+                    {output}
+                    </agent_output>\
+                    """)
+                logger.debug(f"Agent output #{i + 1}: {output}")
+                result = self.evaluate_answer(
+                    input=eval_input,
+                    evaluator_agent=evaluator_agent,
+                    evaluation_input=evaluation_input,
+                    evaluator_expected_output=eval_expected_output,
+                    agent_output=output,
+                )
+                if result is None:
+                    logger.error(f"Failed to evaluate accuracy on iteration {i + 1}")
+                    continue
+
+                self.result.results.append(result)
+                self.result.compute_stats()
+                status.update(f"Eval iteration {i + 1} finished")
+
+            status.stop()
+
+        # Save result to file if requested
+        if self.file_path_to_save_results is not None and self.result is not None:
+            store_result_in_file(
+                file_path=self.file_path_to_save_results,
+                name=self.name,
+                eval_id=self.eval_id,
+                result=self.result,
+            )
+
+        # Print results if requested
+        if self.print_results or print_results:
+            self.result.print_results(console)
+        if self.print_summary or print_summary:
+            self.result.print_summary(console)
+
+        # Log results to the Agno DB if requested
+        if self.agent is not None:
+            agent_id = self.agent.id
+            team_id = None
+            model_id = self.agent.model.id if self.agent.model is not None else None
+            model_provider = self.agent.model.provider if self.agent.model is not None else None
+            evaluated_component_name = self.agent.name
+        elif self.team is not None:
+            agent_id = None
+            team_id = self.team.id
+            model_id = self.team.model.id if self.team.model is not None else None
+            model_provider = self.team.model.provider if self.team.model is not None else None
+            evaluated_component_name = self.team.name
+
+        if self.db:
+            log_eval_input = {
+                "additional_guidelines": self.additional_guidelines,
+                "additional_context": self.additional_context,
+                "num_iterations": self.num_iterations,
+                "expected_output": self.expected_output,
+                "input": self.input,
+            }
+
+            log_eval_run(
+                db=self.db,
+                run_id=self.eval_id,  # type: ignore
+                run_data=asdict(self.result),
+                eval_type=EvalType.ACCURACY,
+                agent_id=agent_id,
+                team_id=team_id,
+                model_id=model_id,
+                model_provider=model_provider,
+                name=self.name if self.name is not None else None,
+                evaluated_component_name=evaluated_component_name,
+                eval_input=log_eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, create_eval_run_telemetry
+
+            create_eval_run_telemetry(
+                eval_run=EvalRunCreate(
+                    run_id=self.eval_id,
+                    eval_type=EvalType.ACCURACY,
+                    data=self._get_telemetry_data(),
+                ),
+            )
+
+        logger.debug(f"*********** Evaluation {self.eval_id} Finished ***********")
+        return self.result
+
+    async def arun(
+        self,
+        *,
+        print_summary: bool = True,
+        print_results: bool = True,
+    ) -> Optional[AccuracyResult]:
+        if self.agent is None and self.team is None:
+            logger.error("You need to provide one of 'agent' or 'team' to run the evaluation.")
             return None
 
-        expected_answer_to_evaluate: Optional[str] = self.get_expected_answer_to_evaluate(
-            expected_answer=expected_answer
-        )
-        if expected_answer_to_evaluate is None:
-            logger.error("No Expected Answer to evaluate.")
+        if self.agent is not None and self.team is not None:
+            logger.error("Provide only one of 'agent' or 'team' to run the evaluation.")
             return None
 
-        logger.debug(f"************ Evaluation Start: {self.eval_id} ************")
-        logger.debug(f"Question: {question_to_evaluate}")
-        logger.debug(f"Expected Answer: {expected_answer_to_evaluate}")
-        logger.debug("***********************************************************")
+        from rich.console import Console
+        from rich.live import Live
+        from rich.status import Status
 
-        evaluator_agent: Agent = self.get_evaluator_agent(
-            question=question_to_evaluate, expected_answer=expected_answer_to_evaluate
-        )
+        set_log_level_to_debug() if self.debug_mode else set_log_level_to_info()
+
+        self.result = AccuracyResult()
+
+        logger.debug(f"************ Evaluation Start: {self.eval_id} ************")
 
         # Add a spinner while running the evaluations
         console = Console()
         with Live(console=console, transient=True) as live_log:
+            evaluator_agent = self.get_evaluator_agent()
+            eval_input = self.get_eval_input()
+            eval_expected_output = self.get_eval_expected_output()
+
             for i in range(self.num_iterations):
                 status = Status(f"Running evaluation {i + 1}...", spinner="dots", speed=1.0, refresh_per_second=10)
                 live_log.update(status)
 
-                answer_to_evaluate: Optional[RunResponse] = self.get_answer_to_evaluate(
-                    question=question_to_evaluate, answer=answer
+                agent_session_id = f"eval_{self.eval_id}_{i + 1}"
+
+                if self.agent is not None:
+                    agent_response = await self.agent.arun(input=eval_input, session_id=agent_session_id, stream=False)
+                    output = agent_response.content
+                elif self.team is not None:
+                    team_response = await self.team.arun(input=eval_input, session_id=agent_session_id, stream=False)
+                    output = team_response.content
+
+                if not output:
+                    logger.error(f"Failed to generate a valid answer on iteration {i + 1}: {output}")
+                    continue
+
+                evaluation_input = dedent(f"""\
+                    <agent_input>
+                    {eval_input}
+                    </agent_input>
+
+                    <expected_output>
+                    {eval_expected_output}
+                    </expected_output>
+
+                    <agent_output>
+                    {output}
+                    </agent_output>\
+                    """)
+                logger.debug(f"Agent output #{i + 1}: {output}")
+                result = await self.aevaluate_answer(
+                    input=eval_input,
+                    evaluator_agent=evaluator_agent,
+                    evaluation_input=evaluation_input,
+                    evaluator_expected_output=eval_expected_output,
+                    agent_output=output,
                 )
-                if answer_to_evaluate is None:
-                    logger.error("No Answer to evaluate.")
+                if result is None:
+                    logger.error(f"Failed to evaluate accuracy on iteration {i + 1}")
                     continue
 
-                try:
-                    logger.debug(f"Answer #{i + 1}: {answer_to_evaluate.content}")
-                    accuracy_agent_response = evaluator_agent.run(answer_to_evaluate.content).content
-                    if accuracy_agent_response is None or not isinstance(
-                        accuracy_agent_response, AccuracyAgentResponse
-                    ):
-                        logger.error("Evaluator Agent returned an invalid response")
-                        continue
-
-                    accuracy_evaluation = AccuracyEvaluation(
-                        question=question_to_evaluate,
-                        answer=answer_to_evaluate.content,  # type: ignore
-                        expected_answer=expected_answer_to_evaluate,
-                        score=accuracy_agent_response.accuracy_score,
-                        reason=accuracy_agent_response.accuracy_reason,
-                    )
-                    if self.print_results:
-                        accuracy_evaluation.print_eval(console)
-                    self.result.results.append(accuracy_evaluation)
-                    self.result.compute_stats()
-                    status.update(f"Running evaluation {i + 1}... Done")
-                except Exception as e:
-                    logger.exception(f"Failed to evaluate accuracy, run #{i + 1}: {e}")
-                    return None
-
-                status.stop()
-
-        # -*- Save result to file if save_result_to_file is set
-        if self.save_result_to_file is not None and self.result is not None:
-            try:
-                import json
+                self.result.results.append(result)
+                self.result.compute_stats()
+                status.update(f"Eval iteration {i + 1} finished")
+
+            status.stop()
+
+        # Save result to file if requested
+        if self.file_path_to_save_results is not None and self.result is not None:
+            store_result_in_file(
+                file_path=self.file_path_to_save_results,
+                name=self.name,
+                eval_id=self.eval_id,
+                result=self.result,
+            )
+
+        # Print results if requested
+        if self.print_results or print_results:
+            self.result.print_results(console)
+        if self.print_summary or print_summary:
+            self.result.print_summary(console)
 
-                fn_path = Path(self.save_result_to_file.format(name=self.name, eval_id=self.eval_id))
-                if not fn_path.parent.exists():
-                    fn_path.parent.mkdir(parents=True, exist_ok=True)
-                fn_path.write_text(json.dumps(asdict(self.result), indent=4))
-            except Exception as e:
-                logger.warning(f"Failed to save result to file: {e}")
+        if self.agent is not None:
+            agent_id = self.agent.id
+            team_id = None
+            model_id = self.agent.model.id if self.agent.model is not None else None
+            model_provider = self.agent.model.provider if self.agent.model is not None else None
+            evaluated_component_name = self.agent.name
+        elif self.team is not None:
+            agent_id = None
+            team_id = self.team.id
+            model_id = self.team.model.id if self.team.model is not None else None
+            model_provider = self.team.model.provider if self.team.model is not None else None
+            evaluated_component_name = self.team.name
+
+        # Log results to the Agno DB if requested
+        if self.db:
+            log_eval_input = {
+                "additional_guidelines": self.additional_guidelines,
+                "additional_context": self.additional_context,
+                "num_iterations": self.num_iterations,
+                "expected_output": self.expected_output,
+                "input": self.input,
+            }
+            await async_log_eval(
+                db=self.db,
+                run_id=self.eval_id,  # type: ignore
+                run_data=asdict(self.result),
+                eval_type=EvalType.ACCURACY,
+                agent_id=agent_id,
+                model_id=model_id,
+                model_provider=model_provider,
+                name=self.name if self.name is not None else None,
+                evaluated_component_name=evaluated_component_name,
+                team_id=team_id,
+                workflow_id=None,
+                eval_input=log_eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, async_create_eval_run_telemetry
+
+            await async_create_eval_run_telemetry(
+                eval_run=EvalRunCreate(run_id=self.eval_id, eval_type=EvalType.ACCURACY),
+            )
+
+        logger.debug(f"*********** Evaluation {self.eval_id} Finished ***********")
+        return self.result
 
-        # Show results
-        if self.print_summary or self.print_results:
-            self.result.print_summary(console)
+    def run_with_output(
+        self,
+        *,
+        output: str,
+        print_summary: bool = True,
+        print_results: bool = True,
+    ) -> Optional[AccuracyResult]:
+        """Run the evaluation logic against the given answer, instead of generating an answer with the Agent"""
+        # Generate unique run_id for this execution (don't modify self.eval_id due to concurrency)
+        run_id = str(uuid4())
+
+        set_log_level_to_debug() if self.debug_mode else set_log_level_to_info()
+
+        self.result = AccuracyResult()
+
+        logger.debug(f"************ Evaluation Start: {run_id} ************")
+
+        evaluator_agent = self.get_evaluator_agent()
+        eval_input = self.get_eval_input()
+        eval_expected_output = self.get_eval_expected_output()
+
+        evaluation_input = dedent(f"""\
+            <agent_input>
+            {eval_input}
+            </agent_input>
 
-        logger.debug(f"*********** Evaluation End: {self.eval_id} ***********")
+            <expected_output>
+            {eval_expected_output}
+            </expected_output>
+
+            <agent_output>
+            {output}
+            </agent_output>\
+            """)
+
+        result = self.evaluate_answer(
+            input=eval_input,
+            evaluator_agent=evaluator_agent,
+            evaluation_input=evaluation_input,
+            evaluator_expected_output=eval_expected_output,
+            agent_output=output,
+        )
+
+        if result is not None:
+            self.result.results.append(result)
+            self.result.compute_stats()
+
+            # Print results if requested
+            if self.print_results or print_results:
+                self.result.print_results()
+            if self.print_summary or print_summary:
+                self.result.print_summary()
+
+            # Save result to file if requested
+            if self.file_path_to_save_results is not None:
+                store_result_in_file(
+                    file_path=self.file_path_to_save_results,
+                    name=self.name,
+                    eval_id=self.eval_id,
+                    result=self.result,
+                )
+        # Log results to the Agno DB if requested
+        if self.db:
+            if isinstance(self.db, AsyncBaseDb):
+                log_error("You are using an async DB in a non-async method. The evaluation won't be stored in the DB.")
+
+            else:
+                if self.agent is not None:
+                    agent_id = self.agent.id
+                    team_id = None
+                    model_id = self.agent.model.id if self.agent.model is not None else None
+                    model_provider = self.agent.model.provider if self.agent.model is not None else None
+                    evaluated_component_name = self.agent.name
+                elif self.team is not None:
+                    agent_id = None
+                    team_id = self.team.id
+                    model_id = self.team.model.id if self.team.model is not None else None
+                    model_provider = self.team.model.provider if self.team.model is not None else None
+                    evaluated_component_name = self.team.name
+                else:
+                    agent_id = None
+                    team_id = None
+                    model_id = None
+                    model_provider = None
+                    evaluated_component_name = None
+
+                log_eval_input = {
+                    "additional_guidelines": self.additional_guidelines,
+                    "additional_context": self.additional_context,
+                    "num_iterations": self.num_iterations,
+                    "expected_output": self.expected_output,
+                    "input": self.input,
+                }
+
+                log_eval_run(
+                    db=self.db,
+                    run_id=self.eval_id,  # type: ignore
+                    run_data=asdict(self.result),
+                    eval_type=EvalType.ACCURACY,
+                    name=self.name if self.name is not None else None,
+                    agent_id=agent_id,
+                    team_id=team_id,
+                    model_id=model_id,
+                    model_provider=model_provider,
+                    evaluated_component_name=evaluated_component_name,
+                    workflow_id=None,
+                    eval_input=log_eval_input,
+                )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, create_eval_run_telemetry
+
+            create_eval_run_telemetry(
+                eval_run=EvalRunCreate(
+                    run_id=self.eval_id,
+                    eval_type=EvalType.ACCURACY,
+                    data=self._get_telemetry_data(),
+                ),
+            )
+
+        logger.debug(f"*********** Evaluation End: {run_id} ***********")
+        return self.result
+
+    async def arun_with_output(
+        self,
+        *,
+        output: str,
+        print_summary: bool = True,
+        print_results: bool = True,
+    ) -> Optional[AccuracyResult]:
+        """Run the evaluation logic against the given answer, instead of generating an answer with the Agent"""
+        # Generate unique run_id for this execution (don't modify self.eval_id due to concurrency)
+        run_id = str(uuid4())
+
+        set_log_level_to_debug() if self.debug_mode else set_log_level_to_info()
+
+        self.result = AccuracyResult()
+
+        logger.debug(f"************ Evaluation Start: {run_id} ************")
+
+        evaluator_agent = self.get_evaluator_agent()
+        eval_input = self.get_eval_input()
+        eval_expected_output = self.get_eval_expected_output()
+
+        evaluation_input = dedent(f"""\
+            <agent_input>
+            {eval_input}
+            </agent_input>
+
+            <expected_output>
+            {eval_expected_output}
+            </expected_output>
+
+            <agent_output>
+            {output}
+            </agent_output>\
+            """)
+
+        result = await self.aevaluate_answer(
+            input=eval_input,
+            evaluator_agent=evaluator_agent,
+            evaluation_input=evaluation_input,
+            evaluator_expected_output=eval_expected_output,
+            agent_output=output,
+        )
+
+        if result is not None:
+            self.result.results.append(result)
+            self.result.compute_stats()
+
+            # Print results if requested
+            if self.print_results or print_results:
+                self.result.print_results()
+            if self.print_summary or print_summary:
+                self.result.print_summary()
+
+            # Save result to file if requested
+            if self.file_path_to_save_results is not None:
+                store_result_in_file(
+                    file_path=self.file_path_to_save_results,
+                    name=self.name,
+                    eval_id=self.eval_id,
+                    result=self.result,
+                )
+        # Log results to the Agno DB if requested
+        if self.db:
+            if self.agent is not None:
+                agent_id = self.agent.id
+                team_id = None
+                model_id = self.agent.model.id if self.agent.model is not None else None
+                model_provider = self.agent.model.provider if self.agent.model is not None else None
+                evaluated_component_name = self.agent.name
+            elif self.team is not None:
+                agent_id = None
+                team_id = self.team.id
+                model_id = self.team.model.id if self.team.model is not None else None
+                model_provider = self.team.model.provider if self.team.model is not None else None
+                evaluated_component_name = self.team.name
+
+            log_eval_input = {
+                "additional_guidelines": self.additional_guidelines,
+                "additional_context": self.additional_context,
+                "num_iterations": self.num_iterations,
+                "expected_output": self.expected_output,
+                "input": self.input,
+            }
+
+            await async_log_eval(
+                db=self.db,
+                run_id=self.eval_id,  # type: ignore
+                run_data=asdict(self.result),
+                eval_type=EvalType.ACCURACY,
+                name=self.name if self.name is not None else None,
+                agent_id=agent_id,
+                team_id=team_id,
+                model_id=model_id,
+                model_provider=model_provider,
+                evaluated_component_name=evaluated_component_name,
+                workflow_id=None,
+                eval_input=log_eval_input,
+            )
+
+        logger.debug(f"*********** Evaluation End: {run_id} ***********")
         return self.result
+
+    def _get_telemetry_data(self) -> Dict[str, Any]:
+        """Get the telemetry data for the evaluation"""
+        return {
+            "agent_id": self.agent.id if self.agent else None,
+            "team_id": self.team.id if self.team else None,
+            "model_id": self.agent.model.id if self.agent and self.agent.model else None,
+            "model_provider": self.agent.model.provider if self.agent and self.agent.model else None,
+            "num_iterations": self.num_iterations,
+        }