agno 0.1.2__py3-none-any.whl → 2.3.13__py3-none-any.whl

agno/knowledge/chunking/row.py

@@ -0,0 +1,39 @@
+from typing import List
+
+from agno.knowledge.chunking.strategy import ChunkingStrategy
+from agno.knowledge.document.base import Document
+
+
+class RowChunking(ChunkingStrategy):
+    def __init__(self, skip_header: bool = False, clean_rows: bool = True):
+        self.skip_header = skip_header
+        self.clean_rows = clean_rows
+
+    def chunk(self, document: Document) -> List[Document]:
+        if not document or not document.content:
+            return []
+
+        if not isinstance(document.content, str):
+            raise ValueError("Document content must be a string")
+
+        rows = document.content.splitlines()
+
+        if self.skip_header and rows:
+            rows = rows[1:]
+            start_index = 2
+        else:
+            start_index = 1
+
+        chunks = []
+        for i, row in enumerate(rows):
+            if self.clean_rows:
+                chunk_content = " ".join(row.split())  # Normalize internal whitespace
+            else:
+                chunk_content = row.strip()
+
+            if chunk_content:  # Skip empty rows
+                meta_data = document.meta_data.copy()
+                meta_data["row_number"] = start_index + i  # Preserve logical row numbering
+                chunk_id = f"{document.id}_row_{start_index + i}" if document.id else None
+                chunks.append(Document(id=chunk_id, name=document.name, meta_data=meta_data, content=chunk_content))
+        return chunks

agno/knowledge/chunking/semantic.py

@@ -0,0 +1,91 @@
+import inspect
+from typing import Any, Dict, List, Optional
+
+from agno.knowledge.chunking.strategy import ChunkingStrategy
+from agno.knowledge.document.base import Document
+from agno.knowledge.embedder.base import Embedder
+from agno.utils.log import log_info
+
+
+class SemanticChunking(ChunkingStrategy):
+    """Chunking strategy that splits text into semantic chunks using chonkie"""
+
+    def __init__(self, embedder: Optional[Embedder] = None, chunk_size: int = 5000, similarity_threshold: float = 0.5):
+        if embedder is None:
+            from agno.knowledge.embedder.openai import OpenAIEmbedder
+
+            embedder = OpenAIEmbedder()  # type: ignore
+            log_info("Embedder not provided, using OpenAIEmbedder as default.")
+        self.embedder = embedder
+        self.chunk_size = chunk_size
+        self.similarity_threshold = similarity_threshold
+        self.chunker = None  # Will be initialized lazily when needed
+
+    def _initialize_chunker(self):
+        """Lazily initialize the chunker with chonkie dependency."""
+        if self.chunker is None:
+            try:
+                from chonkie import SemanticChunker
+            except ImportError:
+                raise ImportError(
+                    "`chonkie` is required for semantic chunking. "
+                    "Please install it using `pip install chonkie` to use SemanticChunking."
+                )
+
+            # Build arguments dynamically based on chonkie's supported signature
+            params: Dict[str, Any] = {
+                "chunk_size": self.chunk_size,
+                "threshold": self.similarity_threshold,
+            }
+
+            try:
+                sig = inspect.signature(SemanticChunker)
+                param_names = set(sig.parameters.keys())
+
+                # Prefer passing a callable to avoid Chonkie initializing its own client
+                if "embedding_fn" in param_names:
+                    params["embedding_fn"] = self.embedder.get_embedding  # type: ignore[attr-defined]
+                    # If chonkie allows specifying dimensions, provide them
+                    if "embedding_dimensions" in param_names and getattr(self.embedder, "dimensions", None):
+                        params["embedding_dimensions"] = self.embedder.dimensions  # type: ignore[attr-defined]
+                elif "embedder" in param_names:
+                    # Some versions may accept an embedder object directly
+                    params["embedder"] = self.embedder
+                else:
+                    # Fallback to model id
+                    params["embedding_model"] = getattr(self.embedder, "id", None) or "text-embedding-3-small"
+
+                self.chunker = SemanticChunker(**params)  # type: ignore
+            except Exception:
+                # As a final fallback, use the original behavior
+                self.chunker = SemanticChunker(  # type: ignore
+                    embedding_model=getattr(self.embedder, "id", None) or "text-embedding-3-small",
+                    chunk_size=self.chunk_size,
+                    threshold=self.similarity_threshold,
+                )
+
+    def chunk(self, document: Document) -> List[Document]:
+        """Split document into semantic chunks using chonkie"""
+        if not document.content:
+            return [document]
+
+        # Ensure chunker is initialized (will raise ImportError if chonkie is missing)
+        self._initialize_chunker()
+
+        # Use chonkie to split into semantic chunks
+        if self.chunker is None:
+            raise RuntimeError("Chunker failed to initialize")
+
+        chunks = self.chunker.chunk(self.clean_text(document.content))
+
+        # Convert chunks to Documents
+        chunked_documents: List[Document] = []
+        for i, chunk in enumerate(chunks, 1):
+            meta_data = document.meta_data.copy()
+            meta_data["chunk"] = i
+            chunk_id = f"{document.id}_{i}" if document.id else None
+            meta_data["chunk_size"] = len(chunk.text)
+
+            chunked_documents.append(Document(id=chunk_id, name=document.name, meta_data=meta_data, content=chunk.text))
+
+        return chunked_documents

agno/knowledge/chunking/strategy.py

@@ -0,0 +1,165 @@
+from abc import ABC, abstractmethod
+from enum import Enum
+from typing import List, Optional
+
+from agno.knowledge.document.base import Document
+
+
+class ChunkingStrategy(ABC):
+    """Base class for chunking strategies"""
+
+    @abstractmethod
+    def chunk(self, document: Document) -> List[Document]:
+        raise NotImplementedError
+
+    def clean_text(self, text: str) -> str:
+        """Clean the text by replacing multiple newlines with a single newline"""
+        import re
+
+        # Replace multiple newlines with a single newline
+        cleaned_text = re.sub(r"\n+", "\n", text)
+        # Replace multiple spaces with a single space
+        cleaned_text = re.sub(r"\s+", " ", cleaned_text)
+        # Replace multiple tabs with a single tab
+        cleaned_text = re.sub(r"\t+", "\t", cleaned_text)
+        # Replace multiple carriage returns with a single carriage return
+        cleaned_text = re.sub(r"\r+", "\r", cleaned_text)
+        # Replace multiple form feeds with a single form feed
+        cleaned_text = re.sub(r"\f+", "\f", cleaned_text)
+        # Replace multiple vertical tabs with a single vertical tab
+        cleaned_text = re.sub(r"\v+", "\v", cleaned_text)
+
+        return cleaned_text
+
+
+class ChunkingStrategyType(str, Enum):
+    """Enumeration of available chunking strategies."""
+
+    AGENTIC_CHUNKER = "AgenticChunker"
+    DOCUMENT_CHUNKER = "DocumentChunker"
+    RECURSIVE_CHUNKER = "RecursiveChunker"
+    SEMANTIC_CHUNKER = "SemanticChunker"
+    FIXED_SIZE_CHUNKER = "FixedSizeChunker"
+    ROW_CHUNKER = "RowChunker"
+    MARKDOWN_CHUNKER = "MarkdownChunker"
+
+    @classmethod
+    def from_string(cls, strategy_name: str) -> "ChunkingStrategyType":
+        """Convert a string to a ChunkingStrategyType."""
+        strategy_name_clean = strategy_name.strip()
+
+        # Try exact enum value match first
+        for enum_member in cls:
+            if enum_member.value == strategy_name_clean:
+                return enum_member
+
+        raise ValueError(f"Unsupported chunking strategy: {strategy_name}. Valid options: {[e.value for e in cls]}")
+
+
+class ChunkingStrategyFactory:
+    """Factory for creating chunking strategy instances."""
+
+    @classmethod
+    def create_strategy(
+        cls,
+        strategy_type: ChunkingStrategyType,
+        chunk_size: Optional[int] = None,
+        overlap: Optional[int] = None,
+        **kwargs,
+    ) -> ChunkingStrategy:
+        """Create an instance of the chunking strategy with the given parameters."""
+        strategy_map = {
+            ChunkingStrategyType.AGENTIC_CHUNKER: cls._create_agentic_chunking,
+            ChunkingStrategyType.DOCUMENT_CHUNKER: cls._create_document_chunking,
+            ChunkingStrategyType.RECURSIVE_CHUNKER: cls._create_recursive_chunking,
+            ChunkingStrategyType.SEMANTIC_CHUNKER: cls._create_semantic_chunking,
+            ChunkingStrategyType.FIXED_SIZE_CHUNKER: cls._create_fixed_chunking,
+            ChunkingStrategyType.ROW_CHUNKER: cls._create_row_chunking,
+            ChunkingStrategyType.MARKDOWN_CHUNKER: cls._create_markdown_chunking,
+        }
+        return strategy_map[strategy_type](chunk_size=chunk_size, overlap=overlap, **kwargs)
+
+    @classmethod
+    def _create_agentic_chunking(
+        cls, chunk_size: Optional[int] = None, overlap: Optional[int] = None, **kwargs
+    ) -> ChunkingStrategy:
+        from agno.knowledge.chunking.agentic import AgenticChunking
+
+        # AgenticChunking accepts max_chunk_size (not chunk_size) and no overlap
+        if chunk_size is not None:
+            kwargs["max_chunk_size"] = chunk_size
+        # Remove overlap since AgenticChunking doesn't support it
+        return AgenticChunking(**kwargs)
+
+    @classmethod
+    def _create_document_chunking(
+        cls, chunk_size: Optional[int] = None, overlap: Optional[int] = None, **kwargs
+    ) -> ChunkingStrategy:
+        from agno.knowledge.chunking.document import DocumentChunking
+
+        # DocumentChunking accepts both chunk_size and overlap
+        if chunk_size is not None:
+            kwargs["chunk_size"] = chunk_size
+        if overlap is not None:
+            kwargs["overlap"] = overlap
+        return DocumentChunking(**kwargs)
+
+    @classmethod
+    def _create_recursive_chunking(
+        cls, chunk_size: Optional[int] = None, overlap: Optional[int] = None, **kwargs
+    ) -> ChunkingStrategy:
+        from agno.knowledge.chunking.recursive import RecursiveChunking
+
+        # RecursiveChunking accepts both chunk_size and overlap
+        if chunk_size is not None:
+            kwargs["chunk_size"] = chunk_size
+        if overlap is not None:
+            kwargs["overlap"] = overlap
+        return RecursiveChunking(**kwargs)
+
+    @classmethod
+    def _create_semantic_chunking(
+        cls, chunk_size: Optional[int] = None, overlap: Optional[int] = None, **kwargs
+    ) -> ChunkingStrategy:
+        from agno.knowledge.chunking.semantic import SemanticChunking
+
+        # SemanticChunking accepts chunk_size but not overlap
+        if chunk_size is not None:
+            kwargs["chunk_size"] = chunk_size
+        # Remove overlap since SemanticChunking doesn't support it
+        return SemanticChunking(**kwargs)
+
+    @classmethod
+    def _create_fixed_chunking(
+        cls, chunk_size: Optional[int] = None, overlap: Optional[int] = None, **kwargs
+    ) -> ChunkingStrategy:
+        from agno.knowledge.chunking.fixed import FixedSizeChunking
+
+        # FixedSizeChunking accepts both chunk_size and overlap
+        if chunk_size is not None:
+            kwargs["chunk_size"] = chunk_size
+        if overlap is not None:
+            kwargs["overlap"] = overlap
+        return FixedSizeChunking(**kwargs)
+
+    @classmethod
+    def _create_row_chunking(
+        cls, chunk_size: Optional[int] = None, overlap: Optional[int] = None, **kwargs
+    ) -> ChunkingStrategy:
+        from agno.knowledge.chunking.row import RowChunking
+
+        # RowChunking doesn't accept chunk_size or overlap, only skip_header and clean_rows
+        return RowChunking(**kwargs)
+
+    @classmethod
+    def _create_markdown_chunking(
+        cls, chunk_size: Optional[int] = None, overlap: Optional[int] = None, **kwargs
+    ) -> ChunkingStrategy:
+        from agno.knowledge.chunking.markdown import MarkdownChunking
+
+        # MarkdownChunking accepts both chunk_size and overlap
+        if chunk_size is not None:
+            kwargs["chunk_size"] = chunk_size
+        if overlap is not None:
+            kwargs["overlap"] = overlap
+        return MarkdownChunking(**kwargs)

agno/knowledge/content.py

@@ -0,0 +1,74 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import Any, Dict, List, Optional, Union
+
+from agno.knowledge.reader import Reader
+from agno.knowledge.remote_content.remote_content import RemoteContent
+
+
+class ContentStatus(str, Enum):
+    """Enumeration of possible content processing statuses."""
+
+    PROCESSING = "processing"
+    COMPLETED = "completed"
+    FAILED = "failed"
+
+
+@dataclass
+class FileData:
+    content: Optional[Union[str, bytes]] = None
+    type: Optional[str] = None
+    filename: Optional[str] = None
+    size: Optional[int] = None
+
+
+@dataclass
+class ContentAuth:
+    password: Optional[str] = None
+
+
+@dataclass
+class Content:
+    id: Optional[str] = None
+    name: Optional[str] = None
+    description: Optional[str] = None
+    path: Optional[str] = None
+    url: Optional[str] = None
+    auth: Optional[ContentAuth] = None
+    file_data: Optional[FileData] = None
+    metadata: Optional[Dict[str, Any]] = None
+    topics: Optional[List[str]] = None
+    remote_content: Optional[RemoteContent] = None
+    reader: Optional[Reader] = None
+    size: Optional[int] = None
+    file_type: Optional[str] = None
+    content_hash: Optional[str] = None
+    status: Optional[ContentStatus] = None
+    status_message: Optional[str] = None
+    created_at: Optional[int] = None
+    updated_at: Optional[int] = None
+    external_id: Optional[str] = None
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "Content":
+        return cls(
+            id=data.get("id"),
+            name=data.get("name"),
+            description=data.get("description"),
+            path=data.get("path"),
+            url=data.get("url"),
+            auth=data.get("auth"),
+            file_data=data.get("file_data"),
+            metadata=data.get("metadata"),
+            topics=data.get("topics"),
+            remote_content=data.get("remote_content"),
+            reader=data.get("reader"),
+            size=data.get("size"),
+            file_type=data.get("file_type"),
+            content_hash=data.get("content_hash"),
+            status=data.get("status"),
+            status_message=data.get("status_message"),
+            created_at=data.get("created_at"),
+            updated_at=data.get("updated_at"),
+            external_id=data.get("external_id"),
+        )

agno/knowledge/document/__init__.py

@@ -0,0 +1,5 @@
+from agno.knowledge.document.base import Document
+
+__all__ = [
+    "Document",
+]

agno/{document → knowledge/document}/base.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass, field
 from typing import Any, Dict, List, Optional
 
-from agno.embedder import Embedder
+from agno.knowledge.embedder import Embedder
 
 
 @dataclass
@@ -12,10 +12,13 @@ class Document:
     id: Optional[str] = None
     name: Optional[str] = None
     meta_data: Dict[str, Any] = field(default_factory=dict)
-    embedder: Optional[Embedder] = None
+    embedder: Optional["Embedder"] = None
     embedding: Optional[List[float]] = None
     usage: Optional[Dict[str, Any]] = None
     reranking_score: Optional[float] = None
+    content_id: Optional[str] = None
+    content_origin: Optional[str] = None
+    size: Optional[int] = None
 
     def embed(self, embedder: Optional[Embedder] = None) -> None:
         """Embed the document using the provided embedder"""
@@ -26,6 +29,13 @@ class Document:
 
         self.embedding, self.usage = _embedder.get_embedding_and_usage(self.content)
 
+    async def async_embed(self, embedder: Optional[Embedder] = None) -> None:
+        """Embed the document using the provided embedder"""
+        _embedder = embedder or self.embedder
+        if _embedder is None:
+            raise ValueError("No embedder provided")
+        self.embedding, self.usage = await _embedder.async_get_embedding_and_usage(self.content)
+
     def to_dict(self) -> Dict[str, Any]:
         """Returns a dictionary representation of the document"""
         fields = {"name", "meta_data", "content"}

agno/knowledge/embedder/__init__.py

@@ -0,0 +1,5 @@
+from agno.knowledge.embedder.base import Embedder
+
+__all__ = [
+    "Embedder",
+]