agno 1.8.1__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

agno/eval/performance.py

@@ -3,11 +3,12 @@ import gc
 import tracemalloc
 from dataclasses import dataclass, field
 from os import getenv
-from typing import TYPE_CHECKING, Callable, List, Optional
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
 from uuid import uuid4
 
-from agno.api.schemas.evals import EvalType
-from agno.eval.utils import async_log_eval_run, log_eval_run, store_result_in_file
+from agno.db.base import BaseDb
+from agno.db.schemas.evals import EvalType
+from agno.eval.utils import async_log_eval, log_eval_run, store_result_in_file
 from agno.utils.log import log_debug, set_log_level_to_debug, set_log_level_to_info
 from agno.utils.timer import Timer
 
@@ -195,7 +196,7 @@ class PerformanceEval:
     # Evaluation UUID
     eval_id: str = field(default_factory=lambda: str(uuid4()))
     # Number of warm-up runs (not included in final stats)
-    warmup_runs: int = 10
+    warmup_runs: Optional[int] = 10
     # Number of measured iterations
     num_iterations: int = 50
     # Result of the evaluation
@@ -210,12 +211,23 @@ class PerformanceEval:
     # Number of memory allocations to track
     top_n_memory_allocations: int = 5
 
+    # Agent and Team information
+    agent_id: Optional[str] = None
+    team_id: Optional[str] = None
+    model_id: Optional[str] = None
+    model_provider: Optional[str] = None
+
     # If set, results will be saved in the given file path
     file_path_to_save_results: Optional[str] = None
     # Enable debug logs
     debug_mode: bool = getenv("AGNO_DEBUG", "false").lower() == "true"
-    # Log the results to the Agno platform. On by default.
-    monitoring: bool = getenv("AGNO_MONITOR", "true").lower() == "true"
+    # The database to store Evaluation results
+    db: Optional[BaseDb] = None
+
+    # Telemetry settings
+    # telemetry=True logs minimal telemetry for analytics
+    # This helps us improve our Evals and provide better support
+    telemetry: bool = True
 
     def _measure_time(self) -> float:
         """Measure execution time for a single run."""
@@ -495,12 +507,13 @@ class PerformanceEval:
         console = Console()
         with Live(console=console, transient=True) as live_log:
             # 1. Do optional warm-up runs.
-            for i in range(self.warmup_runs):
-                status = Status(f"Warm-up run {i + 1}/{self.warmup_runs}...", spinner="dots", speed=1.0)
-                live_log.update(status)
-                self.func()  # Simply run the function without measuring
-                self._set_log_level()  # Set log level incase function changed it
-                status.stop()
+            if self.warmup_runs is not None:
+                for i in range(self.warmup_runs):
+                    status = Status(f"Warm-up run {i + 1}/{self.warmup_runs}...", spinner="dots", speed=1.0)
+                    live_log.update(status)
+                    self.func()  # Simply run the function without measuring
+                    self._set_log_level()  # Set log level incase function changed it
+                    status.stop()
 
             # 2. Measure runtime
             if self.measure_runtime:
@@ -570,13 +583,33 @@ class PerformanceEval:
             self.result.print_summary(console, measure_memory=self.measure_memory, measure_runtime=self.measure_runtime)
 
         # 7. Log results to the Agno platform if requested
-        if self.monitoring:
+        if self.db:
+            eval_input = {
+                "num_iterations": self.num_iterations,
+                "warmup_runs": self.warmup_runs,
+            }
+
             log_eval_run(
+                db=self.db,
                 run_id=self.eval_id,  # type: ignore
                 run_data=self._parse_eval_run_data(),
                 eval_type=EvalType.PERFORMANCE,
                 name=self.name if self.name is not None else None,
-                evaluated_entity_name=self.func.__name__,
+                evaluated_component_name=self.func.__name__,
+                agent_id=self.agent_id,
+                team_id=self.team_id,
+                model_id=self.model_id,
+                model_provider=self.model_provider,
+                eval_input=eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, create_eval_run_telemetry
+
+            create_eval_run_telemetry(
+                eval_run=EvalRunCreate(
+                    run_id=self.eval_id, eval_type=EvalType.PERFORMANCE, data=self._get_telemetry_data()
+                ),
             )
 
         log_debug(f"*********** Evaluation End: {self.eval_id} ***********")
@@ -617,12 +650,13 @@ class PerformanceEval:
         console = Console()
         with Live(console=console, transient=True) as live_log:
             # 1. Do optional warm-up runs.
-            for i in range(self.warmup_runs):
-                status = Status(f"Warm-up run {i + 1}/{self.warmup_runs}...", spinner="dots", speed=1.0)
-                live_log.update(status)
-                await self.func()  # Simply run the function without measuring
-                self._set_log_level()  # Set log level incase function changed it
-                status.stop()
+            if self.warmup_runs is not None:
+                for i in range(self.warmup_runs):
+                    status = Status(f"Warm-up run {i + 1}/{self.warmup_runs}...", spinner="dots", speed=1.0)
+                    live_log.update(status)
+                    await self.func()  # Simply run the function without measuring
+                    self._set_log_level()  # Set log level incase function changed it
+                    status.stop()
 
             # 2. Measure runtime
             if self.measure_runtime:
@@ -692,14 +726,45 @@ class PerformanceEval:
             self.result.print_summary(console, measure_memory=self.measure_memory, measure_runtime=self.measure_runtime)
 
         # 7. Log results to the Agno platform if requested
-        if self.monitoring:
-            await async_log_eval_run(
+        if self.db:
+            eval_input = {
+                "num_iterations": self.num_iterations,
+                "warmup_runs": self.warmup_runs,
+            }
+
+            await async_log_eval(
+                db=self.db,
                 run_id=self.eval_id,  # type: ignore
                 run_data=self._parse_eval_run_data(),
                 eval_type=EvalType.PERFORMANCE,
                 name=self.name if self.name is not None else None,
-                evaluated_entity_name=self.func.__name__,
+                evaluated_component_name=self.func.__name__,
+                agent_id=self.agent_id,
+                team_id=self.team_id,
+                model_id=self.model_id,
+                model_provider=self.model_provider,
+                eval_input=eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, async_create_eval_run_telemetry
+
+            await async_create_eval_run_telemetry(
+                eval_run=EvalRunCreate(
+                    run_id=self.eval_id, eval_type=EvalType.PERFORMANCE, data=self._get_telemetry_data()
+                ),
             )
 
         log_debug(f"*********** Evaluation End: {self.eval_id} ***********")
         return self.result
+
+    def _get_telemetry_data(self) -> Dict[str, Any]:
+        """Get the telemetry data for the evaluation"""
+        return {
+            "model_id": self.model_id,
+            "model_provider": self.model_provider,
+            "num_iterations": self.num_iterations,
+            "warmup_runs": self.warmup_runs,
+            "measure_memory": self.measure_memory,
+            "measure_runtime": self.measure_runtime,
+        }

agno/eval/reliability.py

@@ -1,15 +1,17 @@
 from dataclasses import asdict, dataclass, field
 from os import getenv
-from typing import TYPE_CHECKING, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 from uuid import uuid4
 
+from agno.db.base import BaseDb
+
 if TYPE_CHECKING:
     from rich.console import Console
 
-from agno.agent import RunResponse
-from agno.api.schemas.evals import EvalType
-from agno.eval.utils import async_log_eval_run, log_eval_run, store_result_in_file
-from agno.run.team import TeamRunResponse
+from agno.agent import RunOutput
+from agno.db.schemas.evals import EvalType
+from agno.eval.utils import async_log_eval, log_eval_run, store_result_in_file
+from agno.run.team import TeamRunOutput
 from agno.utils.log import logger
 
 
@@ -46,9 +48,9 @@ class ReliabilityEval:
     eval_id: str = field(default_factory=lambda: str(uuid4()))
 
     # Agent response
-    agent_response: Optional[RunResponse] = None
+    agent_response: Optional[RunOutput] = None
     # Team response
-    team_response: Optional[TeamRunResponse] = None
+    team_response: Optional[TeamRunOutput] = None
     # Expected tool calls
     expected_tool_calls: Optional[List[str]] = None
     # Result of the evaluation
@@ -60,8 +62,13 @@ class ReliabilityEval:
     file_path_to_save_results: Optional[str] = None
     # Enable debug logs
     debug_mode: bool = getenv("AGNO_DEBUG", "false").lower() == "true"
-    # Log the results to the Agno platform. On by default.
-    monitoring: bool = getenv("AGNO_MONITOR", "true").lower() == "true"
+    # The database to store Evaluation results
+    db: Optional[BaseDb] = None
+
+    # Telemetry settings
+    # telemetry=True logs minimal telemetry for analytics
+    # This helps us improve our Evals and provide better support
+    telemetry: bool = True
 
     def run(self, *, print_results: bool = False) -> Optional[ReliabilityResult]:
         if self.agent_response is None and self.team_response is None:
@@ -100,15 +107,18 @@ class ReliabilityEval:
 
             failed_tool_calls = []
             passed_tool_calls = []
-            for tool_call in actual_tool_calls:  # type: ignore
-                tool_name = tool_call.get("function", {}).get("name")
-                if not tool_name:
-                    continue
-                else:
-                    if tool_name not in self.expected_tool_calls:  # type: ignore
-                        failed_tool_calls.append(tool_call.get("function", {}).get("name"))
+            if not actual_tool_calls:
+                failed_tool_calls = self.expected_tool_calls or []
+            else:
+                for tool_call in actual_tool_calls:  # type: ignore
+                    tool_name = tool_call.get("function", {}).get("name")
+                    if not tool_name:
+                        continue
                     else:
-                        passed_tool_calls.append(tool_call.get("function", {}).get("name"))
+                        if tool_name not in self.expected_tool_calls:  # type: ignore
+                            failed_tool_calls.append(tool_call.get("function", {}).get("name"))
+                        else:
+                            passed_tool_calls.append(tool_call.get("function", {}).get("name"))
 
             self.result = ReliabilityResult(
                 eval_status="PASSED" if len(failed_tool_calls) == 0 else "FAILED",
@@ -130,7 +140,7 @@ class ReliabilityEval:
             self.result.print_eval(console)
 
         # Log results to the Agno platform if requested
-        if self.monitoring:
+        if self.db:
             if self.agent_response is not None:
                 agent_id = self.agent_response.agent_id
                 team_id = None
@@ -142,7 +152,12 @@ class ReliabilityEval:
                 model_id = self.team_response.model
                 model_provider = self.team_response.model_provider
 
+            eval_input = {
+                "expected_tool_calls": self.expected_tool_calls,
+            }
+
             log_eval_run(
+                db=self.db,
                 run_id=self.eval_id,  # type: ignore
                 run_data=asdict(self.result),
                 eval_type=EvalType.RELIABILITY,
@@ -151,6 +166,18 @@ class ReliabilityEval:
                 team_id=team_id,
                 model_id=model_id,
                 model_provider=model_provider,
+                eval_input=eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, create_eval_run_telemetry
+
+            create_eval_run_telemetry(
+                eval_run=EvalRunCreate(
+                    run_id=self.eval_id,
+                    eval_type=EvalType.RELIABILITY,
+                    data=self._get_telemetry_data(),
+                ),
             )
 
         logger.debug(f"*********** Evaluation End: {self.eval_id} ***********")
@@ -223,7 +250,7 @@ class ReliabilityEval:
             self.result.print_eval(console)
 
         # Log results to the Agno platform if requested
-        if self.monitoring:
+        if self.db:
             if self.agent_response is not None:
                 agent_id = self.agent_response.agent_id
                 team_id = None
@@ -235,7 +262,12 @@ class ReliabilityEval:
                 model_id = self.team_response.model
                 model_provider = self.team_response.model_provider
 
-            await async_log_eval_run(
+            eval_input = {
+                "expected_tool_calls": self.expected_tool_calls,
+            }
+
+            await async_log_eval(
+                db=self.db,
                 run_id=self.eval_id,  # type: ignore
                 run_data=asdict(self.result),
                 eval_type=EvalType.RELIABILITY,
@@ -244,7 +276,28 @@ class ReliabilityEval:
                 team_id=team_id,
                 model_id=model_id,
                 model_provider=model_provider,
+                eval_input=eval_input,
+            )
+
+        if self.telemetry:
+            from agno.api.evals import EvalRunCreate, async_create_eval_run_telemetry
+
+            await async_create_eval_run_telemetry(
+                eval_run=EvalRunCreate(
+                    run_id=self.eval_id,
+                    eval_type=EvalType.RELIABILITY,
+                    data=self._get_telemetry_data(),
+                ),
             )
 
         logger.debug(f"*********** Evaluation End: {self.eval_id} ***********")
         return self.result
+
+    def _get_telemetry_data(self) -> Dict[str, Any]:
+        """Get the telemetry data for the evaluation"""
+        return {
+            "team_id": self.team_response.team_id if self.team_response else None,
+            "agent_id": self.agent_response.agent_id if self.agent_response else None,
+            "model_id": self.agent_response.model if self.agent_response else None,
+            "model_provider": self.agent_response.model_provider if self.agent_response else None,
+        }

agno/eval/utils.py

@@ -2,8 +2,8 @@ from dataclasses import asdict
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional, Union
 
-from agno.api.evals import async_create_eval_run, create_eval_run
-from agno.api.schemas.evals import EvalRunCreate, EvalType
+from agno.db.base import BaseDb
+from agno.db.schemas.evals import EvalRunRecord, EvalType
 from agno.utils.log import log_debug, logger
 
 if TYPE_CHECKING:
@@ -13,61 +13,71 @@ if TYPE_CHECKING:
 
 
 def log_eval_run(
+    db: BaseDb,
     run_id: str,
     run_data: dict,
     eval_type: EvalType,
+    eval_input: dict,
     agent_id: Optional[str] = None,
     model_id: Optional[str] = None,
     model_provider: Optional[str] = None,
     name: Optional[str] = None,
-    evaluated_entity_name: Optional[str] = None,
+    evaluated_component_name: Optional[str] = None,
     team_id: Optional[str] = None,
+    workflow_id: Optional[str] = None,
 ) -> None:
     """Call the API to create an evaluation run."""
 
     try:
-        create_eval_run(
-            eval_run=EvalRunCreate(
+        db.create_eval_run(
+            EvalRunRecord(
                 run_id=run_id,
                 eval_type=eval_type,
                 eval_data=run_data,
+                eval_input=eval_input,
                 agent_id=agent_id,
                 model_id=model_id,
                 model_provider=model_provider,
                 name=name,
-                evaluated_entity_name=evaluated_entity_name,
+                evaluated_component_name=evaluated_component_name,
                 team_id=team_id,
+                workflow_id=workflow_id,
             )
         )
     except Exception as e:
         log_debug(f"Could not create agent event: {e}")
 
 
-async def async_log_eval_run(
+async def async_log_eval(
+    db: BaseDb,
     run_id: str,
     run_data: dict,
     eval_type: EvalType,
+    eval_input: dict,
     agent_id: Optional[str] = None,
     model_id: Optional[str] = None,
     model_provider: Optional[str] = None,
     name: Optional[str] = None,
-    evaluated_entity_name: Optional[str] = None,
+    evaluated_component_name: Optional[str] = None,
     team_id: Optional[str] = None,
+    workflow_id: Optional[str] = None,
 ) -> None:
-    """Asycn call to the API to create an evaluation run."""
+    """Call the API to create an evaluation run."""
 
     try:
-        await async_create_eval_run(
-            eval_run=EvalRunCreate(
+        db.create_eval_run(
+            EvalRunRecord(
                 run_id=run_id,
                 eval_type=eval_type,
                 eval_data=run_data,
+                eval_input=eval_input,
                 agent_id=agent_id,
                 model_id=model_id,
                 model_provider=model_provider,
-                team_id=team_id,
                 name=name,
-                evaluated_entity_name=evaluated_entity_name,
+                evaluated_component_name=evaluated_component_name,
+                team_id=team_id,
+                workflow_id=workflow_id,
             )
         )
     except Exception as e:

agno/integrations/discord/__init__.py

@@ -0,0 +1,3 @@
+from agno.integrations.discord.client import DiscordClient
+
+__all__ = ["DiscordClient"]

agno/{app → integrations}/discord/client.py

@@ -4,9 +4,9 @@ from typing import Optional, Union
 
 import requests
 
-from agno.agent.agent import Agent, RunResponse
+from agno.agent.agent import Agent, RunOutput
 from agno.media import Audio, File, Image, Video
-from agno.team.team import Team, TeamRunResponse
+from agno.team.team import Team, TeamRunOutput
 from agno.utils.log import log_info, log_warning
 
 try:
@@ -115,8 +115,8 @@ class DiscordClient:
                     """)
                 if self.agent:
                     self.agent.additional_context = additional_context
-                    agent_response: RunResponse = await self.agent.arun(
-                        message_text,
+                    agent_response: RunOutput = await self.agent.arun(
+                        input=message_text,
                         user_id=message_user_id,
                         session_id=str(thread.id),
                         images=[Image(url=message_image)] if message_image else None,
@@ -127,8 +127,8 @@ class DiscordClient:
                     await self._handle_response_in_thread(agent_response, thread)
                 elif self.team:
                     self.team.additional_context = additional_context
-                    team_response: TeamRunResponse = await self.team.arun(
-                        message_text,
+                    team_response: TeamRunOutput = await self.team.arun(
+                        input=message_text,
                         user_id=message_user_id,
                         session_id=str(thread.id),
                         images=[Image(url=message_image)] if message_image else None,
@@ -139,8 +139,8 @@ class DiscordClient:
                     await self._handle_response_in_thread(team_response, thread)
 
     async def handle_hitl(
-        self, run_response: RunResponse, thread: Union[discord.Thread, discord.TextChannel]
-    ) -> RunResponse:
+        self, run_response: RunOutput, thread: Union[discord.Thread, discord.TextChannel]
+    ) -> RunOutput:
         """Handles optional Human-In-The-Loop interaction."""
         if run_response.is_paused:
             for tool in run_response.tools_requiring_confirmation:
@@ -157,9 +157,9 @@ class DiscordClient:
         return run_response
 
     async def _handle_response_in_thread(
-        self, response: Union[RunResponse, TeamRunResponse], thread: Union[discord.TextChannel, discord.Thread]
+        self, response: Union[RunOutput, TeamRunOutput], thread: Union[discord.TextChannel, discord.Thread]
     ):
-        if isinstance(response, RunResponse):
+        if isinstance(response, RunOutput):
             response = await self.handle_hitl(response, thread)
 
         if response.reasoning_content:

agno/knowledge/__init__.py

@@ -1,5 +1,5 @@
-from agno.knowledge.agent import AgentKnowledge
+from agno.knowledge.knowledge import Knowledge
 
 __all__ = [
-    "AgentKnowledge",
+    "Knowledge",
 ]

agno/{document → knowledge}/chunking/agentic.py

@@ -1,7 +1,7 @@
 from typing import List, Optional
 
-from agno.document.base import Document
-from agno.document.chunking.strategy import ChunkingStrategy
+from agno.knowledge.chunking.strategy import ChunkingStrategy
+from agno.knowledge.document.base import Document
 from agno.models.base import Model
 from agno.models.defaults import DEFAULT_OPENAI_MODEL_ID
 from agno.models.message import Message

agno/{document → knowledge}/chunking/document.py

@@ -1,7 +1,7 @@
 from typing import List
 
-from agno.document.base import Document
-from agno.document.chunking.strategy import ChunkingStrategy
+from agno.knowledge.chunking.strategy import ChunkingStrategy
+from agno.knowledge.document.base import Document
 
 
 class DocumentChunking(ChunkingStrategy):

agno/{document → knowledge}/chunking/fixed.py

@@ -1,13 +1,13 @@
 from typing import List
 
-from agno.document.base import Document
-from agno.document.chunking.strategy import ChunkingStrategy
+from agno.knowledge.chunking.strategy import ChunkingStrategy
+from agno.knowledge.document.base import Document
 
 
 class FixedSizeChunking(ChunkingStrategy):
     """Chunking strategy that splits text into fixed-size chunks with optional overlap"""
 
-    def __init__(self, chunk_size: int = 5000, overlap: int = 0):
+    def __init__(self, chunk_size: int = 100, overlap: int = 0):
         # overlap must be less than chunk size
         if overlap >= chunk_size:
             raise ValueError(f"Invalid parameters: overlap ({overlap}) must be less than chunk size ({chunk_size}).")

agno/{document → knowledge}/chunking/markdown.py

@@ -8,8 +8,8 @@ try:
 except ImportError:
     raise ImportError("`unstructured` not installed. Please install it using `pip install unstructured markdown`")
 
-from agno.document.base import Document
-from agno.document.chunking.strategy import ChunkingStrategy
+from agno.knowledge.chunking.strategy import ChunkingStrategy
+from agno.knowledge.document.base import Document
 
 
 class MarkdownChunking(ChunkingStrategy):

agno/{document → knowledge}/chunking/recursive.py

@@ -1,8 +1,8 @@
 import warnings
 from typing import List
 
-from agno.document.base import Document
-from agno.document.chunking.strategy import ChunkingStrategy
+from agno.knowledge.chunking.strategy import ChunkingStrategy
+from agno.knowledge.document.base import Document
 
 
 class RecursiveChunking(ChunkingStrategy):

agno/{document → knowledge}/chunking/row.py

@@ -1,7 +1,7 @@
 from typing import List
 
-from agno.document.base import Document
-from agno.document.chunking.strategy import ChunkingStrategy
+from agno.knowledge.chunking.strategy import ChunkingStrategy
+from agno.knowledge.document.base import Document
 
 
 class RowChunking(ChunkingStrategy):

agno/knowledge/chunking/semantic.py

@@ -0,0 +1,59 @@
+from typing import List, Optional
+
+from agno.knowledge.chunking.strategy import ChunkingStrategy
+from agno.knowledge.document.base import Document
+from agno.knowledge.embedder.base import Embedder
+from agno.knowledge.embedder.openai import OpenAIEmbedder
+
+
+class SemanticChunking(ChunkingStrategy):
+    """Chunking strategy that splits text into semantic chunks using chonkie"""
+
+    def __init__(self, embedder: Optional[Embedder] = None, chunk_size: int = 5000, similarity_threshold: float = 0.5):
+        self.embedder = embedder or OpenAIEmbedder(id="text-embedding-3-small")  # type: ignore
+        self.chunk_size = chunk_size
+        self.similarity_threshold = similarity_threshold
+        self.chunker = None  # Will be initialized lazily when needed
+
+    def _initialize_chunker(self):
+        """Lazily initialize the chunker with chonkie dependency."""
+        if self.chunker is None:
+            try:
+                from chonkie import SemanticChunker
+            except ImportError:
+                raise ImportError(
+                    "`chonkie` is required for semantic chunking. "
+                    "Please install it using `pip install chonkie` to use SemanticChunking."
+                )
+
+            self.chunker = SemanticChunker(
+                embedding_model=self.embedder.id,  # type: ignore
+                chunk_size=self.chunk_size,
+                threshold=self.similarity_threshold,
+            )
+
+    def chunk(self, document: Document) -> List[Document]:
+        """Split document into semantic chunks using chonkie"""
+        if not document.content:
+            return [document]
+
+        # Ensure chunker is initialized (will raise ImportError if chonkie is missing)
+        self._initialize_chunker()
+
+        # Use chonkie to split into semantic chunks
+        if self.chunker is None:
+            raise RuntimeError("Chunker failed to initialize")
+
+        chunks = self.chunker.chunk(self.clean_text(document.content))
+
+        # Convert chunks to Documents
+        chunked_documents: List[Document] = []
+        for i, chunk in enumerate(chunks, 1):
+            meta_data = document.meta_data.copy()
+            meta_data["chunk"] = i
+            chunk_id = f"{document.id}_{i}" if document.id else None
+            meta_data["chunk_size"] = len(chunk.text)
+
+            chunked_documents.append(Document(id=chunk_id, name=document.name, meta_data=meta_data, content=chunk.text))
+
+        return chunked_documents