agno 2.2.13__py3-none-any.whl → 2.4.3__py3-none-any.whl

agno/knowledge/chunking/markdown.py

@@ -1,6 +1,7 @@
 import os
+import re
 import tempfile
-from typing import List
+from typing import List, Union
 
 try:
     from unstructured.chunking.title import chunk_by_title  # type: ignore
@@ -13,17 +14,83 @@ from agno.knowledge.document.base import Document
 
 
 class MarkdownChunking(ChunkingStrategy):
-    """A chunking strategy that splits markdown based on structure like headers, paragraphs and sections"""
-
-    def __init__(self, chunk_size: int = 5000, overlap: int = 0):
+    """A chunking strategy that splits markdown based on structure like headers, paragraphs and sections
+
+    Args:
+        chunk_size: Maximum size of each chunk in characters
+        overlap: Number of characters to overlap between chunks
+        split_on_headings: Controls heading-based splitting behavior:
+            - False: Use size-based chunking (default)
+            - True: Split on all headings (H1-H6)
+            - int: Split on headings at or above this level (1-6)
+                  e.g., 2 splits on H1 and H2, keeping H3-H6 content together
+    """
+
+    def __init__(self, chunk_size: int = 5000, overlap: int = 0, split_on_headings: Union[bool, int] = False):
         self.chunk_size = chunk_size
         self.overlap = overlap
+        self.split_on_headings = split_on_headings
+
+        # Validate split_on_headings parameter
+        # Note: In Python, isinstance(False, int) is True, so we exclude booleans explicitly
+        if isinstance(split_on_headings, int) and not isinstance(split_on_headings, bool):
+            if not (1 <= split_on_headings <= 6):
+                raise ValueError("split_on_headings must be between 1 and 6 when using integer value")
+
+    def _split_by_headings(self, content: str) -> List[str]:
+        """
+        Split markdown content by headings, keeping each heading with its content.
+        Returns a list of sections where each section starts with a heading.
+
+        When split_on_headings is an int, only splits on headings at or above that level.
+        For example, split_on_headings=2 splits on H1 and H2, keeping H3-H6 content together.
+        """
+        # Determine which heading levels to split on
+        if isinstance(self.split_on_headings, int) and not isinstance(self.split_on_headings, bool):
+            # Split on headings at or above this level (1 to split_on_headings)
+            max_heading_level = self.split_on_headings
+            heading_pattern = rf"^#{{{1},{max_heading_level}}}\s+.+$"
+        else:
+            # split_on_headings is True: split on all headings (# to ######)
+            heading_pattern = r"^#{1,6}\s+.+$"
+
+        # Split content while keeping the delimiter (heading)
+        # Use non-capturing group for the pattern to avoid extra capture groups
+        parts = re.split(f"({heading_pattern})", content, flags=re.MULTILINE)
+
+        sections = []
+        current_section = ""
+
+        for part in parts:
+            if not part or not part.strip():
+                continue
+
+            # Check if this part is a heading
+            if re.match(heading_pattern, part.strip(), re.MULTILINE):
+                # Save previous section if exists
+                if current_section.strip():
+                    sections.append(current_section.strip())
+                # Start new section with this heading
+                current_section = part
+            else:
+                # Add content to current section
+                current_section += "\n\n" + part if current_section else part
+
+        # Don't forget the last section
+        if current_section.strip():
+            sections.append(current_section.strip())
+
+        return sections if sections else [content]
 
     def _partition_markdown_content(self, content: str) -> List[str]:
         """
         Partition markdown content and return a list of text chunks.
         Falls back to paragraph splitting if the markdown chunking fails.
         """
+        # When split_on_headings is True or an int, use regex-based splitting to preserve headings
+        if self.split_on_headings:
+            return self._split_by_headings(content)
+
         try:
             # Create a temporary file with the markdown content.
             # This is the recommended usage of the unstructured library.
@@ -35,9 +102,9 @@ class MarkdownChunking(ChunkingStrategy):
                 elements = partition_md(filename=temp_file_path)
 
                 if not elements:
-                    return self.clean_text(content).split("\n\n")
+                    raw_paragraphs = content.split("\n\n")
+                    return [self.clean_text(para) for para in raw_paragraphs]
 
-                # Chunk by title with some default values
                 chunked_elements = chunk_by_title(
                     elements=elements,
                     max_characters=self.chunk_size,
@@ -57,7 +124,10 @@ class MarkdownChunking(ChunkingStrategy):
                     if chunk_text.strip():
                         text_chunks.append(chunk_text.strip())
 
-                return text_chunks if text_chunks else self.clean_text(content).split("\n\n")
+                if text_chunks:
+                    return text_chunks
+                raw_paragraphs = content.split("\n\n")
+                return [self.clean_text(para) for para in raw_paragraphs]
 
             # Always clean up the temporary file
             finally:
@@ -65,11 +135,18 @@ class MarkdownChunking(ChunkingStrategy):
 
         # Fallback to simple paragraph splitting if the markdown chunking fails
         except Exception:
-            return self.clean_text(content).split("\n\n")
+            raw_paragraphs = content.split("\n\n")
+            return [self.clean_text(para) for para in raw_paragraphs]
 
     def chunk(self, document: Document) -> List[Document]:
         """Split markdown document into chunks based on markdown structure"""
-        if not document.content or len(document.content) <= self.chunk_size:
+        # If content is empty, return as-is
+        if not document.content:
+            return [document]
+
+        # When split_on_headings is enabled, always split by headings regardless of size
+        # Only skip chunking for small content when using size-based chunking
+        if not self.split_on_headings and len(document.content) <= self.chunk_size:
             return [document]
 
         # Split using markdown chunking logic, or fallback to paragraphs
@@ -85,7 +162,20 @@ class MarkdownChunking(ChunkingStrategy):
             section = section.strip()
             section_size = len(section)
 
-            if current_size + section_size <= self.chunk_size:
+            # When split_on_headings is True or an int, each section becomes its own chunk
+            if self.split_on_headings:
+                meta_data = chunk_meta_data.copy()
+                meta_data["chunk"] = chunk_number
+                chunk_id = None
+                if document.id:
+                    chunk_id = f"{document.id}_{chunk_number}"
+                elif document.name:
+                    chunk_id = f"{document.name}_{chunk_number}"
+                meta_data["chunk_size"] = section_size
+
+                chunks.append(Document(id=chunk_id, name=document.name, meta_data=meta_data, content=section))
+                chunk_number += 1
+            elif current_size + section_size <= self.chunk_size:
                 current_chunk.append(section)
                 current_size += section_size
             else:
@@ -109,7 +199,8 @@ class MarkdownChunking(ChunkingStrategy):
                 current_chunk = [section]
                 current_size = section_size
 
-        if current_chunk:
+        # Handle remaining content (only when not split_on_headings)
+        if current_chunk and not self.split_on_headings:
             meta_data = chunk_meta_data.copy()
             meta_data["chunk"] = chunk_number
             chunk_id = None

agno/knowledge/chunking/recursive.py

@@ -31,7 +31,7 @@ class RecursiveChunking(ChunkingStrategy):
         start = 0
         chunk_meta_data = document.meta_data
         chunk_number = 1
-        content = self.clean_text(document.content)
+        content = document.content
 
         while start < len(content):
             end = min(start + self.chunk_size, len(content))
@@ -43,7 +43,7 @@ class RecursiveChunking(ChunkingStrategy):
                         end = start + last_sep + 1
                         break
 
-            chunk = content[start:end]
+            chunk = self.clean_text(content[start:end])
             meta_data = chunk_meta_data.copy()
             meta_data["chunk"] = chunk_number
             chunk_id = None

agno/knowledge/chunking/semantic.py

@@ -1,63 +1,145 @@
-import inspect
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Literal, Optional, Union
+
+try:
+    import numpy as np
+except ImportError:
+    raise ImportError("`numpy` not installed. Please install using `pip install numpy`")
+
+try:
+    from chonkie import SemanticChunker
+    from chonkie.embeddings.base import BaseEmbeddings
+except ImportError:
+    raise ImportError(
+        "`chonkie` is required for semantic chunking. "
+        'Please install it using `pip install "chonkie[semantic]"` to use SemanticChunking.'
+    )
 
 from agno.knowledge.chunking.strategy import ChunkingStrategy
 from agno.knowledge.document.base import Document
 from agno.knowledge.embedder.base import Embedder
-from agno.knowledge.embedder.openai import OpenAIEmbedder
+from agno.utils.log import log_debug
 
 
-class SemanticChunking(ChunkingStrategy):
-    """Chunking strategy that splits text into semantic chunks using chonkie"""
+def _get_chonkie_embedder_wrapper(embedder: Embedder):
+    """Create a wrapper that adapts Agno Embedder to chonkie's BaseEmbeddings interface."""
+
+    class _ChonkieEmbedderWrapper(BaseEmbeddings):
+        """Wrapper to make Agno Embedders compatible with chonkie."""
+
+        def __init__(self, agno_embedder: Embedder):
+            super().__init__()
+            self._embedder = agno_embedder
+
+        def embed(self, text: str):
+            embedding = self._embedder.get_embedding(text)  # type: ignore[attr-defined]
+            return np.array(embedding, dtype=np.float32)
+
+        def get_tokenizer(self):
+            """Return a simple token counter function."""
+            return lambda text: len(text.split())
 
-    def __init__(self, embedder: Optional[Embedder] = None, chunk_size: int = 5000, similarity_threshold: float = 0.5):
-        self.embedder = embedder or OpenAIEmbedder(id="text-embedding-3-small")  # type: ignore
+        @property
+        def dimension(self) -> int:
+            return getattr(self._embedder, "dimensions")
+
+    return _ChonkieEmbedderWrapper(embedder)
+
+
+class SemanticChunking(ChunkingStrategy):
+    """Chunking strategy that splits text into semantic chunks using chonkie.
+
+    Args:
+        embedder: The embedder to use for generating embeddings. Can be:
+            - A string model identifier (e.g., "minishlab/potion-base-32M") for chonkie's built-in models
+            - A chonkie BaseEmbeddings instance (used directly)
+            - An Agno Embedder (wrapped for chonkie compatibility)
+        chunk_size: Maximum tokens allowed per chunk.
+        similarity_threshold: Threshold for semantic similarity (0-1).
+        similarity_window: Number of sentences to consider for similarity calculation.
+        min_sentences_per_chunk: Minimum number of sentences per chunk.
+        min_characters_per_sentence: Minimum number of characters per sentence.
+        delimiters: Delimiters to use for sentence splitting.
+        include_delimiters: Whether to include delimiter in prev/next sentence or None.
+        skip_window: Number of groups to skip when merging (0=disabled).
+        filter_window: Window length for the Savitzky-Golay filter.
+        filter_polyorder: Polynomial order for the Savitzky-Golay filter.
+        filter_tolerance: Tolerance for the Savitzky-Golay filter.
+        chunker_params: Additional parameters to pass to chonkie's SemanticChunker.
+    """
+
+    def __init__(
+        self,
+        embedder: Optional[Union[str, Embedder, BaseEmbeddings]] = None,
+        chunk_size: int = 5000,
+        similarity_threshold: float = 0.5,
+        similarity_window: int = 3,
+        min_sentences_per_chunk: int = 1,
+        min_characters_per_sentence: int = 24,
+        delimiters: Optional[List[str]] = None,
+        include_delimiters: Literal["prev", "next", None] = "prev",
+        skip_window: int = 0,
+        filter_window: int = 5,
+        filter_polyorder: int = 3,
+        filter_tolerance: float = 0.2,
+        chunker_params: Optional[Dict[str, Any]] = None,
+    ):
+        if embedder is None:
+            from agno.knowledge.embedder.openai import OpenAIEmbedder
+
+            embedder = OpenAIEmbedder()  # type: ignore
+            log_debug("Embedder not provided, using OpenAIEmbedder as default.")
+        self.embedder = embedder
         self.chunk_size = chunk_size
         self.similarity_threshold = similarity_threshold
-        self.chunker = None  # Will be initialized lazily when needed
+        self.similarity_window = similarity_window
+        self.min_sentences_per_chunk = min_sentences_per_chunk
+        self.min_characters_per_sentence = min_characters_per_sentence
+        self.delimiters = delimiters if delimiters is not None else [". ", "! ", "? ", "\n"]
+        self.include_delimiters = include_delimiters
+        self.skip_window = skip_window
+        self.filter_window = filter_window
+        self.filter_polyorder = filter_polyorder
+        self.filter_tolerance = filter_tolerance
+        self.chunker_params = chunker_params
+        self.chunker: Optional[SemanticChunker] = None
 
     def _initialize_chunker(self):
         """Lazily initialize the chunker with chonkie dependency."""
-        if self.chunker is None:
-            try:
-                from chonkie import SemanticChunker
-            except ImportError:
-                raise ImportError(
-                    "`chonkie` is required for semantic chunking. "
-                    "Please install it using `pip install chonkie` to use SemanticChunking."
-                )
-
-            # Build arguments dynamically based on chonkie's supported signature
-            params: Dict[str, Any] = {
-                "chunk_size": self.chunk_size,
-                "threshold": self.similarity_threshold,
-            }
-
-            try:
-                sig = inspect.signature(SemanticChunker)
-                param_names = set(sig.parameters.keys())
-
-                # Prefer passing a callable to avoid Chonkie initializing its own client
-                if "embedding_fn" in param_names:
-                    params["embedding_fn"] = self.embedder.get_embedding  # type: ignore[attr-defined]
-                    # If chonkie allows specifying dimensions, provide them
-                    if "embedding_dimensions" in param_names and getattr(self.embedder, "dimensions", None):
-                        params["embedding_dimensions"] = self.embedder.dimensions  # type: ignore[attr-defined]
-                elif "embedder" in param_names:
-                    # Some versions may accept an embedder object directly
-                    params["embedder"] = self.embedder
-                else:
-                    # Fallback to model id
-                    params["embedding_model"] = getattr(self.embedder, "id", None) or "text-embedding-3-small"
-
-                self.chunker = SemanticChunker(**params)
-            except Exception:
-                # As a final fallback, use the original behavior
-                self.chunker = SemanticChunker(
-                    embedding_model=getattr(self.embedder, "id", None) or "text-embedding-3-small",
-                    chunk_size=self.chunk_size,
-                    threshold=self.similarity_threshold,
-                )
+        if self.chunker is not None:
+            return
+
+        # Determine embedding model based on type:
+        # - str: pass directly to chonkie (uses chonkie's built-in models)
+        # - BaseEmbeddings: pass directly to chonkie
+        # - Agno Embedder: wrap for chonkie compatibility
+        embedding_model: Union[str, BaseEmbeddings]
+        if isinstance(self.embedder, str):
+            embedding_model = self.embedder
+        elif isinstance(self.embedder, BaseEmbeddings):
+            embedding_model = self.embedder
+        elif isinstance(self.embedder, Embedder):
+            embedding_model = _get_chonkie_embedder_wrapper(self.embedder)
+        else:
+            raise ValueError("Invalid embedder type. Must be a string, BaseEmbeddings, or Embedder instance.")
+
+        _chunker_params: Dict[str, Any] = {
+            "embedding_model": embedding_model,
+            "chunk_size": self.chunk_size,
+            "threshold": self.similarity_threshold,
+            "similarity_window": self.similarity_window,
+            "min_sentences_per_chunk": self.min_sentences_per_chunk,
+            "min_characters_per_sentence": self.min_characters_per_sentence,
+            "delim": self.delimiters,
+            "include_delim": self.include_delimiters,
+            "skip_window": self.skip_window,
+            "filter_window": self.filter_window,
+            "filter_polyorder": self.filter_polyorder,
+            "filter_tolerance": self.filter_tolerance,
+        }
+        if self.chunker_params:
+            _chunker_params.update(self.chunker_params)
+
+        self.chunker = SemanticChunker(**_chunker_params)
 
     def chunk(self, document: Document) -> List[Document]:
         """Split document into semantic chunks using chonkie"""

agno/knowledge/chunking/strategy.py

@@ -12,6 +12,10 @@ class ChunkingStrategy(ABC):
     def chunk(self, document: Document) -> List[Document]:
         raise NotImplementedError
 
+    async def achunk(self, document: Document) -> List[Document]:
+        """Async version of chunk. Override for truly async implementations."""
+        return self.chunk(document)
+
     def clean_text(self, text: str) -> str:
         """Clean the text by replacing multiple newlines with a single newline"""
         import re
@@ -36,6 +40,7 @@ class ChunkingStrategyType(str, Enum):
     """Enumeration of available chunking strategies."""
 
     AGENTIC_CHUNKER = "AgenticChunker"
+    CODE_CHUNKER = "CodeChunker"
     DOCUMENT_CHUNKER = "DocumentChunker"
     RECURSIVE_CHUNKER = "RecursiveChunker"
     SEMANTIC_CHUNKER = "SemanticChunker"
@@ -70,6 +75,7 @@ class ChunkingStrategyFactory:
         """Create an instance of the chunking strategy with the given parameters."""
         strategy_map = {
             ChunkingStrategyType.AGENTIC_CHUNKER: cls._create_agentic_chunking,
+            ChunkingStrategyType.CODE_CHUNKER: cls._create_code_chunking,
             ChunkingStrategyType.DOCUMENT_CHUNKER: cls._create_document_chunking,
             ChunkingStrategyType.RECURSIVE_CHUNKER: cls._create_recursive_chunking,
             ChunkingStrategyType.SEMANTIC_CHUNKER: cls._create_semantic_chunking,
@@ -91,6 +97,18 @@ class ChunkingStrategyFactory:
         # Remove overlap since AgenticChunking doesn't support it
         return AgenticChunking(**kwargs)
 
+    @classmethod
+    def _create_code_chunking(
+        cls, chunk_size: Optional[int] = None, overlap: Optional[int] = None, **kwargs
+    ) -> ChunkingStrategy:
+        from agno.knowledge.chunking.code import CodeChunking
+
+        # CodeChunking accepts chunk_size but not overlap
+        if chunk_size is not None:
+            kwargs["chunk_size"] = chunk_size
+        # Remove overlap since CodeChunking doesn't support it
+        return CodeChunking(**kwargs)
+
     @classmethod
     def _create_document_chunking(
         cls, chunk_size: Optional[int] = None, overlap: Optional[int] = None, **kwargs

agno/knowledge/embedder/azure_openai.py

@@ -18,7 +18,6 @@ except ImportError:
 @dataclass
 class AzureOpenAIEmbedder(Embedder):
     id: str = "text-embedding-3-small"  # This has to match the model that you deployed at the provided URL
-
     dimensions: int = 1536
     encoding_format: Literal["float", "base64"] = "float"
     user: Optional[str] = None

agno/knowledge/embedder/google.py

@@ -15,7 +15,7 @@ except ImportError:
 
 @dataclass
 class GeminiEmbedder(Embedder):
-    id: str = "gemini-embedding-exp-03-07"
+    id: str = "gemini-embedding-001"
     task_type: str = "RETRIEVAL_QUERY"
     title: Optional[str] = None
     dimensions: Optional[int] = 1536

agno/knowledge/embedder/mistral.py

@@ -37,7 +37,7 @@ class MistralEmbedder(Embedder):
             "api_key": self.api_key,
             "endpoint": self.endpoint,
             "max_retries": self.max_retries,
-            "timeout": self.timeout,
+            "timeout_ms": self.timeout * 1000 if self.timeout else None,
         }
         _client_params = {k: v for k, v in _client_params.items() if v is not None}
 

agno/knowledge/embedder/nebius.py

@@ -10,4 +10,4 @@ class NebiusEmbedder(OpenAIEmbedder):
     id: str = "BAAI/bge-en-icl"
     dimensions: int = 1024
     api_key: Optional[str] = getenv("NEBIUS_API_KEY")
-    base_url: str = "https://api.studio.nebius.com/v1/"
+    base_url: str = "https://api.tokenfactory.nebius.com/v1/"

agno/knowledge/embedder/openai.py

@@ -4,7 +4,7 @@ from typing import Any, Dict, List, Optional, Tuple
 from typing_extensions import Literal
 
 from agno.knowledge.embedder.base import Embedder
-from agno.utils.log import logger
+from agno.utils.log import log_info, log_warning
 
 try:
     from openai import AsyncOpenAI
@@ -71,7 +71,8 @@ class OpenAIEmbedder(Embedder):
         }
         if self.user is not None:
             _request_params["user"] = self.user
-        if self.id.startswith("text-embedding-3"):
+        # Pass dimensions for text-embedding-3 models or when using custom base_url (third-party APIs)
+        if self.id.startswith("text-embedding-3") or self.base_url is not None:
             _request_params["dimensions"] = self.dimensions
         if self.request_params:
             _request_params.update(self.request_params)
@@ -82,7 +83,7 @@ class OpenAIEmbedder(Embedder):
             response: CreateEmbeddingResponse = self.response(text=text)
             return response.data[0].embedding
         except Exception as e:
-            logger.warning(e)
+            log_warning(e)
             return []
 
     def get_embedding_and_usage(self, text: str) -> Tuple[List[float], Optional[Dict]]:
@@ -95,7 +96,7 @@ class OpenAIEmbedder(Embedder):
                 return embedding, usage.model_dump()
             return embedding, None
         except Exception as e:
-            logger.warning(e)
+            log_warning(e)
             return [], None
 
     async def async_get_embedding(self, text: str) -> List[float]:
@@ -106,7 +107,8 @@ class OpenAIEmbedder(Embedder):
         }
         if self.user is not None:
             req["user"] = self.user
-        if self.id.startswith("text-embedding-3"):
+        # Pass dimensions for text-embedding-3 models or when using custom base_url (third-party APIs)
+        if self.id.startswith("text-embedding-3") or self.base_url is not None:
             req["dimensions"] = self.dimensions
         if self.request_params:
             req.update(self.request_params)
@@ -115,7 +117,7 @@ class OpenAIEmbedder(Embedder):
             response: CreateEmbeddingResponse = await self.aclient.embeddings.create(**req)
             return response.data[0].embedding
         except Exception as e:
-            logger.warning(e)
+            log_warning(e)
             return []
 
     async def async_get_embedding_and_usage(self, text: str):
@@ -126,7 +128,8 @@ class OpenAIEmbedder(Embedder):
         }
         if self.user is not None:
             req["user"] = self.user
-        if self.id.startswith("text-embedding-3"):
+        # Pass dimensions for text-embedding-3 models or when using custom base_url (third-party APIs)
+        if self.id.startswith("text-embedding-3") or self.base_url is not None:
             req["dimensions"] = self.dimensions
         if self.request_params:
             req.update(self.request_params)
@@ -137,7 +140,7 @@ class OpenAIEmbedder(Embedder):
             usage = response.usage
             return embedding, usage.model_dump() if usage else None
         except Exception as e:
-            logger.warning(e)
+            log_warning(f"Error getting embedding: {e}")
             return [], None
 
     async def async_get_embeddings_batch_and_usage(
@@ -154,7 +157,7 @@ class OpenAIEmbedder(Embedder):
         """
         all_embeddings = []
         all_usage = []
-        logger.info(f"Getting embeddings and usage for {len(texts)} texts in batches of {self.batch_size} (async)")
+        log_info(f"Getting embeddings and usage for {len(texts)} texts in batches of {self.batch_size} (async)")
 
         for i in range(0, len(texts), self.batch_size):
             batch_texts = texts[i : i + self.batch_size]
@@ -166,7 +169,8 @@ class OpenAIEmbedder(Embedder):
             }
             if self.user is not None:
                 req["user"] = self.user
-            if self.id.startswith("text-embedding-3"):
+            # Pass dimensions for text-embedding-3 models or when using custom base_url (third-party APIs)
+            if self.id.startswith("text-embedding-3") or self.base_url is not None:
                 req["dimensions"] = self.dimensions
             if self.request_params:
                 req.update(self.request_params)
@@ -180,7 +184,7 @@ class OpenAIEmbedder(Embedder):
                 usage_dict = response.usage.model_dump() if response.usage else None
                 all_usage.extend([usage_dict] * len(batch_embeddings))
             except Exception as e:
-                logger.warning(f"Error in async batch embedding: {e}")
+                log_warning(f"Error in async batch embedding: {e}")
                 # Fallback to individual calls for this batch
                 for text in batch_texts:
                     try:
@@ -188,7 +192,7 @@ class OpenAIEmbedder(Embedder):
                         all_embeddings.append(embedding)
                         all_usage.append(usage)
                     except Exception as e2:
-                        logger.warning(f"Error in individual async embedding fallback: {e2}")
+                        log_warning(f"Error in individual async embedding fallback: {e2}")
                         all_embeddings.append([])
                         all_usage.append(None)