agno 2.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

agno/db/surrealdb/utils.py

@@ -0,0 +1,147 @@
+import dataclasses
+from typing import Any, Dict, Optional, Sequence, TypeVar, Union, cast
+
+from surrealdb import BlockingHttpSurrealConnection, BlockingWsSurrealConnection, Surreal
+
+from agno.db.schemas.culture import CulturalKnowledge
+from agno.utils.log import logger
+
+RecordType = TypeVar("RecordType")
+
+
+def build_client(
+    url: str, creds: dict[str, str], ns: str, db: str
+) -> Union[BlockingWsSurrealConnection, BlockingHttpSurrealConnection]:
+    client = Surreal(url=url)
+    client.signin(creds)
+    client.use(namespace=ns, database=db)
+    return client
+
+
+def _query_aux(
+    client: Union[BlockingWsSurrealConnection, BlockingHttpSurrealConnection],
+    query: str,
+    vars: dict[str, Any],
+) -> Union[list, dict, str, int]:
+    try:
+        response = client.query(query, vars)
+    except Exception as e:
+        msg = f"!! Query execution error: {query} with {vars}, Error: {e}"
+        logger.error(msg)
+        raise RuntimeError(msg)
+    return response
+
+
+def query(
+    client: Union[BlockingWsSurrealConnection, BlockingHttpSurrealConnection],
+    query: str,
+    vars: dict[str, Any],
+    record_type: type[RecordType],
+) -> Sequence[RecordType]:
+    response = _query_aux(client, query, vars)
+    if isinstance(response, list):
+        if dataclasses.is_dataclass(record_type) and hasattr(record_type, "from_dict"):
+            return [getattr(record_type, "from_dict").__call__(x) for x in response]
+        else:
+            result: list[RecordType] = []
+            for x in response:
+                if isinstance(x, dict):
+                    result.append(record_type(**x))
+                else:
+                    result.append(record_type.__call__(x))
+            return result
+    else:
+        raise ValueError(f"Unexpected response type: {type(response)}")
+
+
+def query_one(
+    client: Union[BlockingWsSurrealConnection, BlockingHttpSurrealConnection],
+    query: str,
+    vars: dict[str, Any],
+    record_type: type[RecordType],
+) -> Optional[RecordType]:
+    response = _query_aux(client, query, vars)
+    if response is None:
+        return None
+    elif not isinstance(response, list):
+        if dataclasses.is_dataclass(record_type) and hasattr(record_type, "from_dict"):
+            return getattr(record_type, "from_dict").__call__(response)
+        elif isinstance(response, dict):
+            return record_type(**response)
+        else:
+            return record_type.__call__(response)
+    elif isinstance(response, list):
+        # Handle list responses - SurrealDB might return a list with a single element
+        if len(response) == 1 and isinstance(response[0], dict):
+            result = response[0]
+            if dataclasses.is_dataclass(record_type) and hasattr(record_type, "from_dict"):
+                return getattr(record_type, "from_dict").__call__(result)
+            elif record_type is dict:
+                return cast(RecordType, result)
+            else:
+                return record_type(**result)
+        elif len(response) == 0:
+            return None
+        else:
+            raise ValueError(f"Expected single record, got {len(response)} records: {response}")
+    else:
+        raise ValueError(f"Unexpected response type: {type(response)}")
+
+
+# -- Cultural Knowledge util methods --
+
+
+def serialize_cultural_knowledge_for_db(cultural_knowledge: CulturalKnowledge) -> Dict[str, Any]:
+    """Serialize a CulturalKnowledge object for database storage.
+
+    Converts the model's separate content, categories, and notes fields
+    into a single dict for the database content field.
+
+    Args:
+        cultural_knowledge (CulturalKnowledge): The cultural knowledge object to serialize.
+
+    Returns:
+        Dict[str, Any]: A dictionary with content, categories, and notes.
+    """
+    content_dict: Dict[str, Any] = {}
+    if cultural_knowledge.content is not None:
+        content_dict["content"] = cultural_knowledge.content
+    if cultural_knowledge.categories is not None:
+        content_dict["categories"] = cultural_knowledge.categories
+    if cultural_knowledge.notes is not None:
+        content_dict["notes"] = cultural_knowledge.notes
+
+    return content_dict if content_dict else {}
+
+
+def deserialize_cultural_knowledge_from_db(db_row: Dict[str, Any]) -> CulturalKnowledge:
+    """Deserialize a database row to a CulturalKnowledge object.
+
+    The database stores content as a dict containing content, categories, and notes.
+    This method extracts those fields and converts them back to the model format.
+
+    Args:
+        db_row (Dict[str, Any]): The database row as a dictionary.
+
+    Returns:
+        CulturalKnowledge: The cultural knowledge object.
+    """
+    # Extract content, categories, and notes from the content field
+    content_json = db_row.get("content", {}) or {}
+
+    return CulturalKnowledge.from_dict(
+        {
+            "id": db_row.get("id"),
+            "name": db_row.get("name"),
+            "summary": db_row.get("summary"),
+            "content": content_json.get("content"),
+            "categories": content_json.get("categories"),
+            "notes": content_json.get("notes"),
+            "metadata": db_row.get("metadata"),
+            "input": db_row.get("input"),
+            "created_at": db_row.get("created_at"),
+            "updated_at": db_row.get("updated_at"),
+            "agent_id": db_row.get("agent_id"),
+            "team_id": db_row.get("team_id"),
+        }
+    )

agno/db/utils.py

@@ -20,6 +20,8 @@ class CustomJSONEncoder(json.JSONEncoder):
             return obj.to_dict()
         elif isinstance(obj, Metrics):
             return obj.to_dict()
+        elif isinstance(obj, type):
+            return str(obj)
 
         return super().default(obj)
 

agno/eval/__init__.py

@@ -1,4 +1,10 @@
 from agno.eval.accuracy import AccuracyAgentResponse, AccuracyEval, AccuracyEvaluation, AccuracyResult
+from agno.eval.agent_as_judge import (
+    AgentAsJudgeEval,
+    AgentAsJudgeEvaluation,
+    AgentAsJudgeResult,
+)
+from agno.eval.base import BaseEval
 from agno.eval.performance import PerformanceEval, PerformanceResult
 from agno.eval.reliability import ReliabilityEval, ReliabilityResult
 
@@ -7,6 +13,10 @@ __all__ = [
     "AccuracyEvaluation",
     "AccuracyResult",
     "AccuracyEval",
+    "AgentAsJudgeEval",
+    "AgentAsJudgeEvaluation",
+    "AgentAsJudgeResult",
+    "BaseEval",
     "PerformanceEval",
     "PerformanceResult",
     "ReliabilityEval",

agno/eval/accuracy.py

@@ -7,13 +7,13 @@ from uuid import uuid4
 from pydantic import BaseModel, Field
 
 from agno.agent import Agent
-from agno.db.base import BaseDb
+from agno.db.base import AsyncBaseDb, BaseDb
 from agno.db.schemas.evals import EvalType
 from agno.eval.utils import async_log_eval, log_eval_run, store_result_in_file
 from agno.exceptions import EvalError
 from agno.models.base import Model
 from agno.team.team import Team
-from agno.utils.log import logger, set_log_level_to_debug, set_log_level_to_info
+from agno.utils.log import log_error, logger, set_log_level_to_debug, set_log_level_to_info
 
 if TYPE_CHECKING:
     from rich.console import Console
@@ -176,7 +176,7 @@ class AccuracyEval:
     # Enable debug logs
     debug_mode: bool = getenv("AGNO_DEBUG", "false").lower() == "true"
     # The database to store Evaluation results
-    db: Optional[BaseDb] = None
+    db: Optional[Union[BaseDb, AsyncBaseDb]] = None
 
     # Telemetry settings
     # telemetry=True logs minimal telemetry for analytics
@@ -282,7 +282,8 @@ Remember: You must only compare the agent_output to the expected_output. The exp
     ) -> Optional[AccuracyEvaluation]:
         """Orchestrate the evaluation process."""
         try:
-            accuracy_agent_response = evaluator_agent.run(evaluation_input).content
+            response = evaluator_agent.run(evaluation_input, stream=False)
+            accuracy_agent_response = response.content
             if accuracy_agent_response is None or not isinstance(accuracy_agent_response, AccuracyAgentResponse):
                 raise EvalError(f"Evaluator Agent returned an invalid response: {accuracy_agent_response}")
             return AccuracyEvaluation(
@@ -306,7 +307,7 @@ Remember: You must only compare the agent_output to the expected_output. The exp
     ) -> Optional[AccuracyEvaluation]:
         """Orchestrate the evaluation process asynchronously."""
         try:
-            response = await evaluator_agent.arun(evaluation_input)
+            response = await evaluator_agent.arun(evaluation_input, stream=False)
             accuracy_agent_response = response.content
             if accuracy_agent_response is None or not isinstance(accuracy_agent_response, AccuracyAgentResponse):
                 raise EvalError(f"Evaluator Agent returned an invalid response: {accuracy_agent_response}")
@@ -327,6 +328,9 @@ Remember: You must only compare the agent_output to the expected_output. The exp
         print_summary: bool = True,
         print_results: bool = True,
     ) -> Optional[AccuracyResult]:
+        if isinstance(self.db, AsyncBaseDb):
+            raise ValueError("run() is not supported with an async DB. Please use arun() instead.")
+
         if self.agent is None and self.team is None:
             logger.error("You need to provide one of 'agent' or 'team' to run the evaluation.")
             return None
@@ -356,10 +360,14 @@ Remember: You must only compare the agent_output to the expected_output. The exp
                 status = Status(f"Running evaluation {i + 1}...", spinner="dots", speed=1.0, refresh_per_second=10)
                 live_log.update(status)
 
+                agent_session_id = f"eval_{self.eval_id}_{i + 1}"
+
                 if self.agent is not None:
-                    output = self.agent.run(input=eval_input).content
+                    agent_response = self.agent.run(input=eval_input, session_id=agent_session_id, stream=False)
+                    output = agent_response.content
                 elif self.team is not None:
-                    output = self.team.run(input=eval_input).content
+                    team_response = self.team.run(input=eval_input, session_id=agent_session_id, stream=False)
+                    output = team_response.content
 
                 if not output:
                     logger.error(f"Failed to generate a valid answer on iteration {i + 1}: {output}")
@@ -497,12 +505,14 @@ Remember: You must only compare the agent_output to the expected_output. The exp
                 status = Status(f"Running evaluation {i + 1}...", spinner="dots", speed=1.0, refresh_per_second=10)
                 live_log.update(status)
 
+                agent_session_id = f"eval_{self.eval_id}_{i + 1}"
+
                 if self.agent is not None:
-                    response = await self.agent.arun(input=eval_input)
-                    output = response.content
+                    agent_response = await self.agent.arun(input=eval_input, session_id=agent_session_id, stream=False)
+                    output = agent_response.content
                 elif self.team is not None:
-                    response = await self.team.arun(input=eval_input)  # type: ignore
-                    output = response.content
+                    team_response = await self.team.arun(input=eval_input, session_id=agent_session_id, stream=False)
+                    output = team_response.content
 
                 if not output:
                     logger.error(f"Failed to generate a valid answer on iteration {i + 1}: {output}")
@@ -609,11 +619,14 @@ Remember: You must only compare the agent_output to the expected_output. The exp
         print_results: bool = True,
     ) -> Optional[AccuracyResult]:
         """Run the evaluation logic against the given answer, instead of generating an answer with the Agent"""
+        # Generate unique run_id for this execution (don't modify self.eval_id due to concurrency)
+        run_id = str(uuid4())
+
         set_log_level_to_debug() if self.debug_mode else set_log_level_to_info()
 
         self.result = AccuracyResult()
 
-        logger.debug(f"************ Evaluation Start: {self.eval_id} ************")
+        logger.debug(f"************ Evaluation Start: {run_id} ************")
 
         evaluator_agent = self.get_evaluator_agent()
         eval_input = self.get_eval_input()
@@ -661,47 +674,51 @@ Remember: You must only compare the agent_output to the expected_output. The exp
                 )
         # Log results to the Agno DB if requested
         if self.db:
-            if self.agent is not None:
-                agent_id = self.agent.id
-                team_id = None
-                model_id = self.agent.model.id if self.agent.model is not None else None
-                model_provider = self.agent.model.provider if self.agent.model is not None else None
-                evaluated_component_name = self.agent.name
-            elif self.team is not None:
-                agent_id = None
-                team_id = self.team.id
-                model_id = self.team.model.id if self.team.model is not None else None
-                model_provider = self.team.model.provider if self.team.model is not None else None
-                evaluated_component_name = self.team.name
-            else:
-                agent_id = None
-                team_id = None
-                model_id = None
-                model_provider = None
-                evaluated_component_name = None
-
-            log_eval_input = {
-                "additional_guidelines": self.additional_guidelines,
-                "additional_context": self.additional_context,
-                "num_iterations": self.num_iterations,
-                "expected_output": self.expected_output,
-                "input": self.input,
-            }
+            if isinstance(self.db, AsyncBaseDb):
+                log_error("You are using an async DB in a non-async method. The evaluation won't be stored in the DB.")
 
-            log_eval_run(
-                db=self.db,
-                run_id=self.eval_id,  # type: ignore
-                run_data=asdict(self.result),
-                eval_type=EvalType.ACCURACY,
-                name=self.name if self.name is not None else None,
-                agent_id=agent_id,
-                team_id=team_id,
-                model_id=model_id,
-                model_provider=model_provider,
-                evaluated_component_name=evaluated_component_name,
-                workflow_id=None,
-                eval_input=log_eval_input,
-            )
+            else:
+                if self.agent is not None:
+                    agent_id = self.agent.id
+                    team_id = None
+                    model_id = self.agent.model.id if self.agent.model is not None else None
+                    model_provider = self.agent.model.provider if self.agent.model is not None else None
+                    evaluated_component_name = self.agent.name
+                elif self.team is not None:
+                    agent_id = None
+                    team_id = self.team.id
+                    model_id = self.team.model.id if self.team.model is not None else None
+                    model_provider = self.team.model.provider if self.team.model is not None else None
+                    evaluated_component_name = self.team.name
+                else:
+                    agent_id = None
+                    team_id = None
+                    model_id = None
+                    model_provider = None
+                    evaluated_component_name = None
+
+                log_eval_input = {
+                    "additional_guidelines": self.additional_guidelines,
+                    "additional_context": self.additional_context,
+                    "num_iterations": self.num_iterations,
+                    "expected_output": self.expected_output,
+                    "input": self.input,
+                }
+
+                log_eval_run(
+                    db=self.db,
+                    run_id=self.eval_id,  # type: ignore
+                    run_data=asdict(self.result),
+                    eval_type=EvalType.ACCURACY,
+                    name=self.name if self.name is not None else None,
+                    agent_id=agent_id,
+                    team_id=team_id,
+                    model_id=model_id,
+                    model_provider=model_provider,
+                    evaluated_component_name=evaluated_component_name,
+                    workflow_id=None,
+                    eval_input=log_eval_input,
+                )
 
         if self.telemetry:
             from agno.api.evals import EvalRunCreate, create_eval_run_telemetry
@@ -714,7 +731,7 @@ Remember: You must only compare the agent_output to the expected_output. The exp
                 ),
             )
 
-        logger.debug(f"*********** Evaluation End: {self.eval_id} ***********")
+        logger.debug(f"*********** Evaluation End: {run_id} ***********")
         return self.result
 
     async def arun_with_output(
@@ -725,11 +742,14 @@ Remember: You must only compare the agent_output to the expected_output. The exp
         print_results: bool = True,
     ) -> Optional[AccuracyResult]:
         """Run the evaluation logic against the given answer, instead of generating an answer with the Agent"""
+        # Generate unique run_id for this execution (don't modify self.eval_id due to concurrency)
+        run_id = str(uuid4())
+
         set_log_level_to_debug() if self.debug_mode else set_log_level_to_info()
 
         self.result = AccuracyResult()
 
-        logger.debug(f"************ Evaluation Start: {self.eval_id} ************")
+        logger.debug(f"************ Evaluation Start: {run_id} ************")
 
         evaluator_agent = self.get_evaluator_agent()
         eval_input = self.get_eval_input()
@@ -813,7 +833,7 @@ Remember: You must only compare the agent_output to the expected_output. The exp
                 eval_input=log_eval_input,
             )
 
-        logger.debug(f"*********** Evaluation End: {self.eval_id} ***********")
+        logger.debug(f"*********** Evaluation End: {run_id} ***********")
         return self.result
 
     def _get_telemetry_data(self) -> Dict[str, Any]: