agno 2.3.24__py3-none-any.whl → 2.3.25__py3-none-any.whl

agno/knowledge/chunking/code.py

@@ -0,0 +1,90 @@
+from typing import Any, Dict, List, Literal, Optional, Union
+
+try:
+    from chonkie import CodeChunker
+    from chonkie.tokenizer import TokenizerProtocol
+except ImportError:
+    raise ImportError(
+        "`chonkie` is required for code chunking. "
+        "Please install it using `pip install chonkie[code]` to use CodeChunking."
+    )
+
+from agno.knowledge.chunking.strategy import ChunkingStrategy
+from agno.knowledge.document.base import Document
+
+
+class CodeChunking(ChunkingStrategy):
+    """Splits code into chunks based on its structure, leveraging Abstract Syntax Trees (ASTs) to create contextually relevant segments using Chonkie.
+
+    Args:
+        tokenizer: The tokenizer to use. Can be a string name or a TokenizerProtocol instance.
+        chunk_size: The size of the chunks to create.
+        language: The language to parse. Use "auto" for detection or specify a tree-sitter-language-pack language.
+        include_nodes: Whether to include AST nodes (Note: Chonkie's base Chunk type does not store node information).
+        chunker_params: Additional parameters to pass to Chonkie's CodeChunker.
+    """
+
+    def __init__(
+        self,
+        tokenizer: Union[str, TokenizerProtocol] = "character",
+        chunk_size: int = 2048,
+        language: Union[Literal["auto"], Any] = "auto",
+        include_nodes: bool = False,
+        chunker_params: Optional[Dict[str, Any]] = None,
+    ):
+        self.tokenizer = tokenizer
+        self.chunk_size = chunk_size
+        self.language = language
+        self.include_nodes = include_nodes
+        self.chunker_params = chunker_params
+        self.chunker: Optional[CodeChunker] = None
+
+    def _initialize_chunker(self):
+        """Lazily initialize the chunker with Chonkie dependency."""
+        if self.chunker is not None:
+            return
+
+        _chunker_params: Dict[str, Any] = {
+            "tokenizer": self.tokenizer,
+            "chunk_size": self.chunk_size,
+            "language": self.language,
+            "include_nodes": self.include_nodes,
+        }
+        if self.chunker_params:
+            _chunker_params.update(self.chunker_params)
+
+        try:
+            self.chunker = CodeChunker(**_chunker_params)
+        except ValueError as e:
+            if "Tokenizer not found" in str(e):
+                raise ImportError(
+                    f"Missing dependencies for tokenizer `{self.tokenizer}`. "
+                    f"Please install using `pip install tiktoken`, `pip install transformers`, or `pip install tokenizers`"
+                ) from e
+            raise
+
+    def chunk(self, document: Document) -> List[Document]:
+        """Split document into code chunks using Chonkie."""
+        if not document.content:
+            return [document]
+
+        # Ensure chunker is initialized (will raise ImportError if Chonkie is missing)
+        self._initialize_chunker()
+
+        # Use Chonkie to split into code chunks
+        if self.chunker is None:
+            raise RuntimeError("Chunker failed to initialize")
+
+        chunks = self.chunker.chunk(document.content)
+
+        # Convert chunks to Documents
+        chunked_documents: List[Document] = []
+        for i, chunk in enumerate(chunks, 1):
+            meta_data = document.meta_data.copy()
+            meta_data["chunk"] = i
+            chunk_id = f"{document.id}_{i}" if document.id else None
+            meta_data["chunk_size"] = len(chunk.text)
+
+            chunked_documents.append(Document(id=chunk_id, name=document.name, meta_data=meta_data, content=chunk.text))
+
+        return chunked_documents

agno/knowledge/chunking/document.py

@@ -20,16 +20,75 @@ class DocumentChunking(ChunkingStrategy):
         raw_paragraphs = document.content.split("\n\n")
         paragraphs = [self.clean_text(para) for para in raw_paragraphs]
         chunks: List[Document] = []
-        current_chunk = []
+        current_chunk: List[str] = []
         current_size = 0
         chunk_meta_data = document.meta_data
         chunk_number = 1
 
         for para in paragraphs:
             para = para.strip()
+            if not para:
+                continue
+
             para_size = len(para)
 
-            if current_size + para_size <= self.chunk_size:
+            # If paragraph itself is larger than chunk_size, split it by sentences
+            if para_size > self.chunk_size:
+                # Save current chunk first
+                if current_chunk:
+                    meta_data = chunk_meta_data.copy()
+                    meta_data["chunk"] = chunk_number
+                    chunk_id = None
+                    if document.id:
+                        chunk_id = f"{document.id}_{chunk_number}"
+                    elif document.name:
+                        chunk_id = f"{document.name}_{chunk_number}"
+                    meta_data["chunk_size"] = len("\n\n".join(current_chunk))
+                    chunks.append(
+                        Document(
+                            id=chunk_id, name=document.name, meta_data=meta_data, content="\n\n".join(current_chunk)
+                        )
+                    )
+                    chunk_number += 1
+                    current_chunk = []
+                    current_size = 0
+
+                # Split oversized paragraph by sentences
+                import re
+
+                sentences = re.split(r"(?<=[.!?])\s+", para)
+                for sentence in sentences:
+                    sentence = sentence.strip()
+                    if not sentence:
+                        continue
+                    sentence_size = len(sentence)
+
+                    if current_size + sentence_size <= self.chunk_size:
+                        current_chunk.append(sentence)
+                        current_size += sentence_size
+                    else:
+                        if current_chunk:
+                            meta_data = chunk_meta_data.copy()
+                            meta_data["chunk"] = chunk_number
+                            chunk_id = None
+                            if document.id:
+                                chunk_id = f"{document.id}_{chunk_number}"
+                            elif document.name:
+                                chunk_id = f"{document.name}_{chunk_number}"
+                            meta_data["chunk_size"] = len(" ".join(current_chunk))
+                            chunks.append(
+                                Document(
+                                    id=chunk_id,
+                                    name=document.name,
+                                    meta_data=meta_data,
+                                    content=" ".join(current_chunk),
+                                )
+                            )
+                            chunk_number += 1
+                        current_chunk = [sentence]
+                        current_size = sentence_size
+
+            elif current_size + para_size <= self.chunk_size:
                 current_chunk.append(para)
                 current_size += para_size
             else:
@@ -47,6 +106,7 @@ class DocumentChunking(ChunkingStrategy):
                             id=chunk_id, name=document.name, meta_data=meta_data, content="\n\n".join(current_chunk)
                         )
                     )
+                    chunk_number += 1
                 current_chunk = [para]
                 current_size = para_size
 

agno/knowledge/chunking/strategy.py

@@ -36,6 +36,7 @@ class ChunkingStrategyType(str, Enum):
     """Enumeration of available chunking strategies."""
 
     AGENTIC_CHUNKER = "AgenticChunker"
+    CODE_CHUNKER = "CodeChunker"
     DOCUMENT_CHUNKER = "DocumentChunker"
     RECURSIVE_CHUNKER = "RecursiveChunker"
     SEMANTIC_CHUNKER = "SemanticChunker"
@@ -70,6 +71,7 @@ class ChunkingStrategyFactory:
         """Create an instance of the chunking strategy with the given parameters."""
         strategy_map = {
             ChunkingStrategyType.AGENTIC_CHUNKER: cls._create_agentic_chunking,
+            ChunkingStrategyType.CODE_CHUNKER: cls._create_code_chunking,
             ChunkingStrategyType.DOCUMENT_CHUNKER: cls._create_document_chunking,
             ChunkingStrategyType.RECURSIVE_CHUNKER: cls._create_recursive_chunking,
             ChunkingStrategyType.SEMANTIC_CHUNKER: cls._create_semantic_chunking,
@@ -91,6 +93,18 @@ class ChunkingStrategyFactory:
         # Remove overlap since AgenticChunking doesn't support it
         return AgenticChunking(**kwargs)
 
+    @classmethod
+    def _create_code_chunking(
+        cls, chunk_size: Optional[int] = None, overlap: Optional[int] = None, **kwargs
+    ) -> ChunkingStrategy:
+        from agno.knowledge.chunking.code import CodeChunking
+
+        # CodeChunking accepts chunk_size but not overlap
+        if chunk_size is not None:
+            kwargs["chunk_size"] = chunk_size
+        # Remove overlap since CodeChunking doesn't support it
+        return CodeChunking(**kwargs)
+
     @classmethod
     def _create_document_chunking(
         cls, chunk_size: Optional[int] = None, overlap: Optional[int] = None, **kwargs

agno/knowledge/knowledge.py

@@ -2034,7 +2034,13 @@ class Knowledge:
                 self.vector_db.update_metadata(content_id=content.id, metadata=content.metadata or {})
 
             return content_row.to_dict()
-        return None
+
+        else:
+            if self.name:
+                log_debug(f"Contents DB not found for knowledge base: {self.name}")
+            else:
+                log_debug("Contents DB not found for knowledge base")
+            return None
 
     async def _aupdate_content(self, content: Content) -> Optional[Dict[str, Any]]:
         if self.contents_db:

agno/knowledge/reader/arxiv_reader.py

@@ -20,6 +20,7 @@ class ArxivReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for Arxiv readers."""
         return [
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.FIXED_SIZE_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,
             ChunkingStrategyType.DOCUMENT_CHUNKER,

agno/knowledge/reader/csv_reader.py

@@ -29,6 +29,7 @@ class CSVReader(Reader):
         """Get the list of supported chunking strategies for CSV readers."""
         return [
             ChunkingStrategyType.ROW_CHUNKER,
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.FIXED_SIZE_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,
             ChunkingStrategyType.DOCUMENT_CHUNKER,

agno/knowledge/reader/docx_reader.py

@@ -27,6 +27,7 @@ class DocxReader(Reader):
         """Get the list of supported chunking strategies for DOCX readers."""
         return [
             ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.FIXED_SIZE_CHUNKER,
             ChunkingStrategyType.SEMANTIC_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,

agno/knowledge/reader/firecrawl_reader.py

@@ -46,6 +46,7 @@ class FirecrawlReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for Firecrawl readers."""
         return [
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.SEMANTIC_CHUNKER,
             ChunkingStrategyType.FIXED_SIZE_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,

agno/knowledge/reader/json_reader.py

@@ -24,6 +24,7 @@ class JSONReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for JSON readers."""
         return [
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.FIXED_SIZE_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,
             ChunkingStrategyType.DOCUMENT_CHUNKER,

agno/knowledge/reader/markdown_reader.py

@@ -31,6 +31,7 @@ class MarkdownReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for Markdown readers."""
         strategies = [
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.DOCUMENT_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,
             ChunkingStrategyType.RECURSIVE_CHUNKER,

agno/knowledge/reader/pdf_reader.py

@@ -204,6 +204,7 @@ class BasePDFReader(Reader):
         """Get the list of supported chunking strategies for PDF readers."""
         return [
             ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.FIXED_SIZE_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,
             ChunkingStrategyType.SEMANTIC_CHUNKER,

agno/knowledge/reader/pptx_reader.py

@@ -27,6 +27,7 @@ class PPTXReader(Reader):
         """Get the list of supported chunking strategies for PPTX readers."""
         return [
             ChunkingStrategyType.DOCUMENT_CHUNKER,
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.FIXED_SIZE_CHUNKER,
             ChunkingStrategyType.SEMANTIC_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,

agno/knowledge/reader/s3_reader.py

@@ -38,6 +38,7 @@ class S3Reader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for S3 readers."""
         return [
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.FIXED_SIZE_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,
             ChunkingStrategyType.DOCUMENT_CHUNKER,

agno/knowledge/reader/tavily_reader.py

@@ -65,6 +65,7 @@ class TavilyReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for Tavily readers."""
         return [
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.SEMANTIC_CHUNKER,
             ChunkingStrategyType.FIXED_SIZE_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,

agno/knowledge/reader/text_reader.py

@@ -21,6 +21,7 @@ class TextReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for Text readers."""
         return [
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.FIXED_SIZE_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,
             ChunkingStrategyType.DOCUMENT_CHUNKER,

agno/knowledge/reader/web_search_reader.py

@@ -56,6 +56,7 @@ class WebSearchReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for Web Search readers."""
         return [
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,
             ChunkingStrategyType.DOCUMENT_CHUNKER,
             ChunkingStrategyType.RECURSIVE_CHUNKER,

agno/knowledge/reader/website_reader.py

@@ -52,6 +52,7 @@ class WebsiteReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for Website readers."""
         return [
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,
             ChunkingStrategyType.DOCUMENT_CHUNKER,
             ChunkingStrategyType.RECURSIVE_CHUNKER,

agno/knowledge/reader/wikipedia_reader.py

@@ -27,6 +27,7 @@ class WikipediaReader(Reader):
     def get_supported_chunking_strategies(self) -> List[ChunkingStrategyType]:
         """Get the list of supported chunking strategies for Wikipedia readers."""
         return [
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.FIXED_SIZE_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,
             ChunkingStrategyType.DOCUMENT_CHUNKER,

agno/knowledge/reader/youtube_reader.py

@@ -27,6 +27,7 @@ class YouTubeReader(Reader):
         """Get the list of supported chunking strategies for YouTube readers."""
         return [
             ChunkingStrategyType.RECURSIVE_CHUNKER,
+            ChunkingStrategyType.CODE_CHUNKER,
             ChunkingStrategyType.AGENTIC_CHUNKER,
             ChunkingStrategyType.DOCUMENT_CHUNKER,
             ChunkingStrategyType.SEMANTIC_CHUNKER,

agno/knowledge/utils.py

@@ -15,6 +15,7 @@ def _get_chunker_class(strategy_type):
         ChunkingStrategyType.AGENTIC_CHUNKER: lambda: _import_class(
             "agno.knowledge.chunking.agentic", "AgenticChunking"
         ),
+        ChunkingStrategyType.CODE_CHUNKER: lambda: _import_class("agno.knowledge.chunking.code", "CodeChunking"),
         ChunkingStrategyType.DOCUMENT_CHUNKER: lambda: _import_class(
             "agno.knowledge.chunking.document", "DocumentChunking"
         ),

agno/learn/__init__.py

@@ -0,0 +1,65 @@
+"""
+Agno Learning Module
+====================
+Gives agents the ability to learn and remember.
+
+Main Components:
+- LearningMachine: Unified learning system
+- Config: Configuration for learning types
+- Schemas: Data structures for learning types
+- Stores: Storage backends for learning types
+"""
+
+from agno.learn.config import (
+    EntityMemoryConfig,
+    LearnedKnowledgeConfig,
+    LearningMode,
+    MemoriesConfig,
+    SessionContextConfig,
+    UserMemoryConfig,
+    UserProfileConfig,
+)
+from agno.learn.machine import LearningMachine
+from agno.learn.schemas import (
+    EntityMemory,
+    LearnedKnowledge,
+    Memories,
+    SessionContext,
+    UserProfile,
+)
+from agno.learn.stores import (
+    EntityMemoryStore,
+    LearnedKnowledgeStore,
+    LearningStore,
+    MemoriesStore,
+    SessionContextStore,
+    UserMemoryStore,
+    UserProfileStore,
+)
+
+__all__ = [
+    # Main class
+    "LearningMachine",
+    # Configs
+    "LearningMode",
+    "UserProfileConfig",
+    "UserMemoryConfig",
+    "MemoriesConfig",  # Backwards compatibility alias
+    "EntityMemoryConfig",
+    "SessionContextConfig",
+    "LearnedKnowledgeConfig",
+    # Schemas
+    "UserProfile",
+    "Memories",
+    "EntityMemory",
+    "SessionContext",
+    "LearnedKnowledge",
+    # Stores
+    "LearningStore",
+    "UserProfileStore",
+    "UserMemoryStore",
+    "MemoriesStore",  # Backwards compatibility alias
+    "SessionContextStore",
+    "LearnedKnowledgeStore",
+    "EntityMemoryStore",
+]