alita-sdk 0.3.351__py3-none-any.whl → 0.3.499__py3-none-any.whl

alita_sdk/tools/bitbucket/__init__.py

@@ -9,7 +9,7 @@ from pydantic import BaseModel, Field, ConfigDict, create_model
 
 from ..base.tool import BaseAction
 from ..elitea_base import filter_missconfigured_index_tools
-from ..utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length, check_connection_response
+from ..utils import clean_string, get_max_toolkit_length, check_connection_response
 from ...configurations.bitbucket import BitbucketConfiguration
 from ...configurations.pgvector import PgVectorConfiguration
 import requests
@@ -38,17 +38,15 @@ def get_tools(tool):
 
 class AlitaBitbucketToolkit(BaseToolkit):
     tools: List[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
         selected_tools = {x['name']: x['args_schema'].schema() for x in
                           BitbucketAPIWrapper.model_construct().get_available_tools()}
-        AlitaBitbucketToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
         m = create_model(
             name,
-            project=(str, Field(description="Project/Workspace", json_schema_extra={'configuration': True})),
-            repository=(str, Field(description="Repository", json_schema_extra={'max_toolkit_length': AlitaBitbucketToolkit.toolkit_max_length, 'configuration': True})),
+            project=(str, Field(description="Project/Workspace")),
+            repository=(str, Field(description="Repository")),
             branch=(str, Field(description="Main branch", default="main")),
             cloud=(Optional[bool], Field(description="Hosting Option", default=None)),
             bitbucket_configuration=(BitbucketConfiguration, Field(description="Bitbucket Configuration", json_schema_extra={'configuration_types': ['bitbucket']})),
@@ -99,16 +97,19 @@ class AlitaBitbucketToolkit(BaseToolkit):
         }
         bitbucket_api_wrapper = BitbucketAPIWrapper(**wrapper_payload)
         available_tools: List[Dict] = bitbucket_api_wrapper.get_available_tools()
-        prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         tools = []
         for tool in available_tools:
             if selected_tools:
                 if tool['name'] not in selected_tools:
                     continue
+            description = tool["description"] + f"\nrepo: {bitbucket_api_wrapper.repository}"
+            if toolkit_name:
+                description = f"{description}\nToolkit: {toolkit_name}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=bitbucket_api_wrapper,
-                name=prefix + tool["name"],
-                description=tool["description"] + f"\nrepo: {bitbucket_api_wrapper.repository}",
+                name=tool["name"],
+                description=description,
                 args_schema=tool["args_schema"]
             ))
         return cls(tools=tools)

alita_sdk/tools/bitbucket/api_wrapper.py

@@ -11,7 +11,9 @@ from .bitbucket_constants import create_pr_data
 from .cloud_api_wrapper import BitbucketCloudApi, BitbucketServerApi
 from pydantic.fields import PrivateAttr
 
-from ..elitea_base import BaseCodeToolApiWrapper
+from ..code_indexer_toolkit import CodeIndexerToolkit
+from ..utils.available_tools_decorator import extend_with_parent_available_tools
+from ..elitea_base import extend_with_file_operations
 
 logger = logging.getLogger(__name__)
 
@@ -117,7 +119,7 @@ CommentOnIssueModel = create_model(
 )
 
 
-class BitbucketAPIWrapper(BaseCodeToolApiWrapper):
+class BitbucketAPIWrapper(CodeIndexerToolkit):
     """Wrapper for Bitbucket API."""
 
     _bitbucket: Any = PrivateAttr()
@@ -167,7 +169,7 @@ class BitbucketAPIWrapper(BaseCodeToolApiWrapper):
             repository=values['repository']
         )
         cls._active_branch = values.get('branch')
-        return values
+        return super().validate_toolkit(values)
 
     def set_active_branch(self, branch_name: str) -> str:
         """Set the active branch for the bot."""
@@ -359,12 +361,15 @@ class BitbucketAPIWrapper(BaseCodeToolApiWrapper):
     #     except Exception as e:
     #         raise ToolException(f"Can't extract file commit hash (`{file_path}`) due to error:\n{str(e)}")
 
-    def _read_file(self, file_path: str, branch: str) -> str:
+    def _read_file(self, file_path: str, branch: str, **kwargs) -> str:
         """
-        Reads a file from the gitlab repo
+        Reads a file from the bitbucket repo with optional partial read support.
+        
         Parameters:
             file_path(str): the file path
             branch(str): branch name (by default: active_branch)
+            **kwargs: Additional parameters (offset, limit, head, tail) - currently ignored,
+                     partial read handled client-side by base class methods
         Returns:
             str: The file decoded as a string
         """
@@ -398,7 +403,46 @@ class BitbucketAPIWrapper(BaseCodeToolApiWrapper):
             return self._read_file(file_path, branch)
         except Exception as e:
             return f"Failed to read file {file_path}: {str(e)}"
+    
+    def _write_file(
+        self,
+        file_path: str,
+        content: str,
+        branch: str = None,
+        commit_message: str = None
+    ) -> str:
+        """
+        Write content to a file (create or update).
+        
+        Parameters:
+            file_path: Path to the file
+            content: New file content
+            branch: Branch name (uses active branch if None)
+            commit_message: Commit message (not used by Bitbucket API)
+            
+        Returns:
+            Success message
+        """
+        try:
+            branch = branch or self._active_branch
+            
+            # Check if file exists by attempting to read it
+            try:
+                self._read_file(file_path, branch)
+                # File exists, update it using OLD/NEW format
+                old_content = self._read_file(file_path, branch)
+                update_query = f"OLD <<<<\n{old_content}\n>>>> OLD\nNEW <<<<\n{content}\n>>>> NEW"
+                self._bitbucket.update_file(file_path=file_path, update_query=update_query, branch=branch)
+                return f"Updated file {file_path}"
+            except:
+                # File doesn't exist, create it
+                self._bitbucket.create_file(file_path=file_path, file_contents=content, branch=branch)
+                return f"Created file {file_path}"
+        except Exception as e:
+            raise ToolException(f"Unable to write file {file_path}: {str(e)}")
 
+    @extend_with_parent_available_tools
+    @extend_with_file_operations
     def get_available_tools(self):
         return [
             {
@@ -473,4 +517,4 @@ class BitbucketAPIWrapper(BaseCodeToolApiWrapper):
                 "description": self.add_pull_request_comment.__doc__ or "Add a comment to a pull request in the repository.",
                 "args_schema": AddPullRequestCommentModel,
             }
-        ] + self._get_vector_search_tools()
+        ]

alita_sdk/tools/browser/__init__.py

@@ -8,7 +8,7 @@ from langchain_community.utilities.wikipedia import WikipediaAPIWrapper
 from .google_search_rag import GoogleSearchResults
 from .crawler import SingleURLCrawler, MultiURLCrawler, GetHTMLContent, GetPDFContent
 from .wiki import WikipediaQueryRun
-from ..utils import get_max_toolkit_length, clean_string, TOOLKIT_SPLITTER
+from ..utils import get_max_toolkit_length, clean_string
 from ...configurations.browser import BrowserConfiguration
 from logging import getLogger
 
@@ -42,7 +42,6 @@ class BrowserToolkit(BaseToolkit):
             'google': GoogleSearchResults.__pydantic_fields__['args_schema'].default.schema(),
             'wiki': WikipediaQueryRun.__pydantic_fields__['args_schema'].default.schema()
         }
-        BrowserToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
 
         def validate_google_fields(cls, values):
             if 'google' in values.get('selected_tools', []):
@@ -90,7 +89,6 @@ class BrowserToolkit(BaseToolkit):
         }
 
         tools = []
-        prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         if not selected_tools:
             selected_tools = [
                 'single_url_crawler',
@@ -127,7 +125,9 @@ class BrowserToolkit(BaseToolkit):
 
             # Only add the tool if it was successfully created
             if tool_entry is not None:
-                tool_entry.name = f"{prefix}{tool_entry.name}"
+                if toolkit_name:
+                    tool_entry.description = f"{tool_entry.description}\nToolkit: {toolkit_name}"
+                    tool_entry.description = tool_entry.description[:1000]
                 tools.append(tool_entry)
         return cls(tools=tools)
 

alita_sdk/tools/carrier/__init__.py

@@ -7,7 +7,7 @@ from functools import lru_cache
 from .api_wrapper import CarrierAPIWrapper
 from .tools import __all__
 from ..elitea_base import filter_missconfigured_index_tools
-from ..utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ..utils import clean_string, get_max_toolkit_length
 from ...configurations.carrier import CarrierConfiguration
 
 logger = logging.getLogger(__name__)
@@ -17,7 +17,6 @@ name = 'carrier'
 
 class AlitaCarrierToolkit(BaseToolkit):
     tools: List[BaseTool] = []
-    toolkit_max_length: int = 100
 
     @classmethod
     @lru_cache(maxsize=32)
@@ -26,7 +25,6 @@ class AlitaCarrierToolkit(BaseToolkit):
         for t in __all__:
             default = t['tool'].__pydantic_fields__['args_schema'].default
             selected_tools[t['name']] = default.schema() if default else default
-        cls.toolkit_max_length = get_max_toolkit_length(selected_tools)
         return create_model(
             name,
             project_id=(Optional[str], Field(None, description="Optional project ID for scoped operations")),
@@ -70,15 +68,15 @@ class AlitaCarrierToolkit(BaseToolkit):
             logger.exception(f"[AlitaCarrierToolkit] Error initializing CarrierAPIWrapper: {e}")
             raise ValueError(f"CarrierAPIWrapper initialization error: {e}")
 
-        prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
-
         tools = []
         for tool_def in __all__:
             if selected_tools and tool_def['name'] not in selected_tools:
                 continue
             try:
                 tool_instance = tool_def['tool'](api_wrapper=carrier_api_wrapper)
-                tool_instance.name = prefix + tool_instance.name
+                if toolkit_name:
+                    tool_instance.description = f"{tool_instance.description}\nToolkit: {toolkit_name}"
+                    tool_instance.description = tool_instance.description[:1000]
                 tools.append(tool_instance)
                 logger.info(f"[AlitaCarrierToolkit] Successfully initialized tool '{tool_instance.name}'")
             except Exception as e:

alita_sdk/tools/chunkers/__init__.py

@@ -3,6 +3,7 @@ from .sematic.statistical_chunker import statistical_chunker
 from .sematic.markdown_chunker import markdown_chunker
 from .sematic.proposal_chunker import proposal_chunker
 from .sematic.json_chunker import json_chunker
+from .universal_chunker import universal_chunker, chunk_single_document, get_file_type
 from .models import StatisticalChunkerConfig, MarkdownChunkerConfig, ProposalChunkerConfig
 
 __all__ = {
@@ -10,7 +11,8 @@ __all__ = {
     'statistical': statistical_chunker,
     'markdown': markdown_chunker,
     'proposal': proposal_chunker,
-    'json': json_chunker
+    'json': json_chunker,
+    'universal': universal_chunker,
 }
 
 __confluence_chunkers__ = {

alita_sdk/tools/chunkers/sematic/json_chunker.py

@@ -17,6 +17,7 @@ def json_chunker(file_content_generator: Generator[Document, None, None], config
             for chunk in chunks:
                 metadata = doc.metadata.copy()
                 metadata['chunk_id'] = chunk_id
+                metadata['method_name'] = 'json'
                 chunk_id += 1
                 yield Document(page_content=json.dumps(chunk), metadata=metadata)
         except Exception as e:

alita_sdk/tools/chunkers/sematic/markdown_chunker.py

@@ -1,4 +1,4 @@
-from typing import Generator
+from typing import Generator, List
 from langchain_core.documents import Document
 from langchain_text_splitters import MarkdownHeaderTextSplitter, ExperimentalMarkdownSyntaxTextSplitter
 from langchain.text_splitter import TokenTextSplitter
@@ -7,34 +7,60 @@ from copy import deepcopy as copy
 
 
 def markdown_chunker(file_content_generator: Generator[Document, None, None], config: dict, *args, **kwargs) -> Generator[Document, None, None]:
+    """
+    Chunks markdown documents by headers, with support for:
+    - Minimum chunk size to avoid tiny fragments
+    - Maximum token limit with overflow splitting
+    - Header metadata preservation
+    
+    Config options:
+        strip_header (bool): Remove headers from content. Default: False
+        return_each_line (bool): Split on every line. Default: False
+        headers_to_split_on (list): Headers to split on, e.g. [('#', 'H1'), ('##', 'H2')]
+        max_tokens (int): Maximum tokens per chunk. Default: 512
+        token_overlap (int): Token overlap for large chunk splitting. Default: 10
+        min_chunk_chars (int): Minimum characters per chunk. Default: 100
+            Chunks smaller than this will be merged with the next chunk.
+    """
     strip_header = config.get("strip_header", False)
     return_each_line = config.get("return_each_line", False)
     headers_to_split_on = config.get("headers_to_split_on", [])
     max_tokens = config.get("max_tokens", 512)
     tokens_overlapping = config.get("token_overlap", 10)
+    min_chunk_chars = config.get("min_chunk_chars", 100)  # Minimum characters per chunk
+    
     headers_to_split_on = [tuple(header) for header in headers_to_split_on]
+    
     for doc in file_content_generator:
         doc_metadata = doc.metadata
         doc_content = doc.page_content
         chunk_id = 0
+        
         markdown_splitter = MarkdownHeaderTextSplitter(
             headers_to_split_on=headers_to_split_on, 
             strip_headers=strip_header,
             return_each_line=return_each_line
         )
         md_header_splits = markdown_splitter.split_text(doc_content)
-        for chunk in md_header_splits:
+        
+        # Merge small chunks with the next one
+        merged_chunks = _merge_small_chunks(md_header_splits, min_chunk_chars)
+        
+        for chunk in merged_chunks:
             if tiktoken_length(chunk.page_content) > max_tokens:
-                for subchunk in TokenTextSplitter(encoding_name="cl100k_base", 
-                                                  chunk_size=max_tokens, 
-                                                  chunk_overlap=tokens_overlapping
-                                                  ).split_text(chunk.page_content):
+                # Split large chunks into smaller ones
+                for subchunk in TokenTextSplitter(
+                    encoding_name="cl100k_base", 
+                    chunk_size=max_tokens, 
+                    chunk_overlap=tokens_overlapping
+                ).split_text(chunk.page_content):
                     chunk_id += 1
                     headers_meta = list(chunk.metadata.values())
                     docmeta = copy(doc_metadata)
                     docmeta.update({"headers": "; ".join(headers_meta)})
                     docmeta['chunk_id'] = chunk_id
                     docmeta['chunk_type'] = "document"
+                    docmeta['method_name'] = 'markdown'
                     yield Document(
                         page_content=subchunk,
                         metadata=docmeta
@@ -46,12 +72,77 @@ def markdown_chunker(file_content_generator: Generator[Document, None, None], co
                 docmeta.update({"headers": "; ".join(headers_meta)})
                 docmeta['chunk_id'] = chunk_id
                 docmeta['chunk_type'] = "document"
+                docmeta['method_name'] = 'text'
                 yield Document(
                     page_content=chunk.page_content,
                     metadata=docmeta
                 )
 
 
+def _merge_small_chunks(chunks: List[Document], min_chars: int) -> List[Document]:
+    """
+    Merge chunks that are smaller than min_chars with the next chunk.
+    
+    This prevents tiny fragments (like standalone headers or short notes)
+    from becoming separate chunks.
+    
+    Args:
+        chunks: List of Document chunks from markdown splitter
+        min_chars: Minimum character count for a chunk
+        
+    Returns:
+        List of merged Document chunks
+    """
+    if not chunks:
+        return chunks
+    
+    merged = []
+    pending_content = ""
+    pending_metadata = {}
+    
+    for i, chunk in enumerate(chunks):
+        content = chunk.page_content.strip()
+        
+        if pending_content:
+            # Merge pending content with current chunk
+            combined_content = pending_content + "\n\n" + content
+            # Use the pending metadata (from the header) but can be extended
+            combined_metadata = {**pending_metadata}
+            # Add any new header info from current chunk
+            for key, value in chunk.metadata.items():
+                if key not in combined_metadata or not combined_metadata[key]:
+                    combined_metadata[key] = value
+            
+            if len(combined_content) >= min_chars:
+                # Combined is big enough, emit it
+                merged.append(Document(
+                    page_content=combined_content,
+                    metadata=combined_metadata
+                ))
+                pending_content = ""
+                pending_metadata = {}
+            else:
+                # Still too small, keep accumulating
+                pending_content = combined_content
+                pending_metadata = combined_metadata
+        elif len(content) < min_chars:
+            # Current chunk is too small, start pending
+            pending_content = content
+            pending_metadata = dict(chunk.metadata)
+        else:
+            # Current chunk is big enough
+            merged.append(chunk)
+    
+    # Don't forget any remaining pending content
+    if pending_content:
+        merged.append(Document(
+            page_content=pending_content,
+            metadata=pending_metadata
+        ))
+    
+    return merged
+
+
 def markdown_by_headers_chunker(file_content_generator: Generator[Document, None, None], config: dict, *args, **kwargs) -> Generator[Document, None, None]:
     strip_header = config.get("strip_header", False)
     return_each_line = config.get("return_each_line", False)

alita_sdk/tools/chunkers/sematic/proposal_chunker.py

@@ -6,7 +6,7 @@ from langchain_core.prompts import ChatPromptTemplate
 from langchain.text_splitter import TokenTextSplitter
 
 from typing import Optional, List
-from langchain_core.pydantic_v1 import BaseModel
+from pydantic import BaseModel
 from ..utils import tiktoken_length
 
 logger = getLogger(__name__)

alita_sdk/tools/chunkers/universal_chunker.py

@@ -0,0 +1,270 @@
+"""
+Universal Chunker - Routes documents to appropriate chunkers based on file type.
+
+This module provides a universal chunking interface that automatically selects
+the appropriate chunking strategy based on the file extension:
+
+- .md, .markdown → Markdown chunker (header-based splitting)
+- .py, .js, .ts, .java, etc. → TreeSitter code chunker
+- .json → JSON chunker  
+- other → Default text chunker
+
+Usage:
+    from alita_sdk.tools.chunkers.universal_chunker import universal_chunker
+    
+    # Chunk documents from a loader
+    for chunk in universal_chunker(document_generator, config):
+        print(chunk.page_content)
+"""
+
+import logging
+import os
+from typing import Generator, Dict, Any, Optional
+from langchain_core.documents import Document
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+from .code.codeparser import parse_code_files_for_db
+from .sematic.markdown_chunker import markdown_chunker
+from .sematic.json_chunker import json_chunker
+
+logger = logging.getLogger(__name__)
+
+
+# File extension mappings
+MARKDOWN_EXTENSIONS = {'.md', '.markdown', '.mdown', '.mkd', '.mdx'}
+JSON_EXTENSIONS = {'.json', '.jsonl', '.jsonc'}
+CODE_EXTENSIONS = {
+    '.py', '.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx',
+    '.java', '.kt', '.rs', '.go', '.cpp', '.c', '.cs', 
+    '.hs', '.rb', '.scala', '.lua'
+}
+
+
+def get_file_extension(file_path: str) -> str:
+    """Extract file extension from path."""
+    return os.path.splitext(file_path)[-1].lower()
+
+
+def get_file_type(file_path: str) -> str:
+    """
+    Determine the file type category for chunking.
+    
+    Returns:
+        'markdown', 'json', 'code', or 'text'
+    """
+    ext = get_file_extension(file_path)
+    
+    if ext in MARKDOWN_EXTENSIONS:
+        return 'markdown'
+    elif ext in JSON_EXTENSIONS:
+        return 'json'
+    elif ext in CODE_EXTENSIONS:
+        return 'code'
+    else:
+        return 'text'
+
+
+def _default_text_chunker(
+    documents: Generator[Document, None, None], 
+    config: Dict[str, Any]
+) -> Generator[Document, None, None]:
+    """
+    Default text chunker for unknown file types.
+    Uses recursive character splitting.
+    """
+    chunk_size = config.get('chunk_size', 1000)
+    chunk_overlap = config.get('chunk_overlap', 100)
+    
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        length_function=len,
+    )
+    
+    for doc in documents:
+        chunks = splitter.split_documents([doc])
+        for idx, chunk in enumerate(chunks, 1):
+            chunk.metadata['chunk_id'] = idx
+            chunk.metadata['chunk_type'] = 'text'
+            chunk.metadata['method_name'] = 'text'
+            yield chunk
+
+
+def _code_chunker_from_documents(
+    documents: Generator[Document, None, None],
+    config: Dict[str, Any]
+) -> Generator[Document, None, None]:
+    """
+    Adapter to convert Document generator to code parser format.
+    """
+    def file_content_generator():
+        for doc in documents:
+            yield {
+                'file_name': doc.metadata.get('file_path', doc.metadata.get('filename', 'unknown')),
+                'file_content': doc.page_content,
+                'commit_hash': doc.metadata.get('commit_hash', ''),
+            }
+    
+    # parse_code_files_for_db returns chunks with proper metadata
+    for chunk in parse_code_files_for_db(file_content_generator()):
+        # Ensure file_path is preserved
+        if 'file_path' not in chunk.metadata and 'filename' in chunk.metadata:
+            chunk.metadata['file_path'] = chunk.metadata['filename']
+        yield chunk
+
+
+def universal_chunker(
+    documents: Generator[Document, None, None],
+    config: Optional[Dict[str, Any]] = None
+) -> Generator[Document, None, None]:
+    """
+    Universal chunker that routes documents to appropriate chunkers based on file type.
+    
+    Each document is inspected for its file extension (from metadata.file_path or
+    metadata.file_name) and routed to the appropriate chunker:
+    
+    - Markdown files → markdown_chunker (header-based splitting)
+    - JSON files → json_chunker (recursive JSON splitting)
+    - Code files → code parser (TreeSitter-based parsing)
+    - Other files → default text chunker (recursive character splitting)
+    
+    Args:
+        documents: Generator yielding Document objects with file content
+        config: Optional configuration dict with:
+            - markdown_config: Config for markdown chunker
+            - json_config: Config for JSON chunker
+            - code_config: Config for code chunker
+            - text_config: Config for default text chunker
+            
+    Yields:
+        Document objects with chunked content and preserved metadata
+    """
+    if config is None:
+        config = {}
+    
+    # Default configs for each chunker type
+    markdown_config = config.get('markdown_config', {
+        'strip_header': False,
+        'return_each_line': False,
+        'headers_to_split_on': [
+            ('#', 'Header 1'),
+            ('##', 'Header 2'),
+            ('###', 'Header 3'),
+            ('####', 'Header 4'),
+        ],
+        'max_tokens': 1024,
+        'token_overlap': 50,
+        'min_chunk_chars': 100,  # Merge chunks smaller than this
+    })
+    
+    json_config = config.get('json_config', {
+        'max_tokens': 512,
+    })
+    
+    code_config = config.get('code_config', {})
+    
+    text_config = config.get('text_config', {
+        'chunk_size': 1000,
+        'chunk_overlap': 100,
+    })
+    
+    # Buffer documents by type for batch processing
+    # This is more efficient than processing one at a time
+    markdown_docs = []
+    json_docs = []
+    code_docs = []
+    text_docs = []
+    
+    # Buffer size before flushing
+    BUFFER_SIZE = 10
+    
+    def flush_markdown():
+        if markdown_docs:
+            def gen():
+                for d in markdown_docs:
+                    yield d
+            for chunk in markdown_chunker(gen(), markdown_config):
+                yield chunk
+            markdown_docs.clear()
+    
+    def flush_json():
+        if json_docs:
+            def gen():
+                for d in json_docs:
+                    yield d
+            for chunk in json_chunker(gen(), json_config):
+                yield chunk
+            json_docs.clear()
+    
+    def flush_code():
+        if code_docs:
+            def gen():
+                for d in code_docs:
+                    yield d
+            for chunk in _code_chunker_from_documents(gen(), code_config):
+                yield chunk
+            code_docs.clear()
+    
+    def flush_text():
+        if text_docs:
+            def gen():
+                for d in text_docs:
+                    yield d
+            for chunk in _default_text_chunker(gen(), text_config):
+                yield chunk
+            text_docs.clear()
+    
+    for doc in documents:
+        # Get file path from metadata
+        file_path = (doc.metadata.get('file_path') or 
+                    doc.metadata.get('file_name') or 
+                    doc.metadata.get('source') or 
+                    'unknown')
+        
+        # Ensure file_path is in metadata for downstream use
+        doc.metadata['file_path'] = file_path
+        
+        file_type = get_file_type(file_path)
+        
+        if file_type == 'markdown':
+            markdown_docs.append(doc)
+            if len(markdown_docs) >= BUFFER_SIZE:
+                yield from flush_markdown()
+        elif file_type == 'json':
+            json_docs.append(doc)
+            if len(json_docs) >= BUFFER_SIZE:
+                yield from flush_json()
+        elif file_type == 'code':
+            code_docs.append(doc)
+            if len(code_docs) >= BUFFER_SIZE:
+                yield from flush_code()
+        else:
+            text_docs.append(doc)
+            if len(text_docs) >= BUFFER_SIZE:
+                yield from flush_text()
+    
+    # Flush remaining documents
+    yield from flush_markdown()
+    yield from flush_json()
+    yield from flush_code()
+    yield from flush_text()
+
+
+def chunk_single_document(
+    doc: Document,
+    config: Optional[Dict[str, Any]] = None
+) -> Generator[Document, None, None]:
+    """
+    Convenience function to chunk a single document.
+    
+    Args:
+        doc: Single Document to chunk
+        config: Optional chunker configuration
+        
+    Yields:
+        Chunked Document objects
+    """
+    def single_doc_gen():
+        yield doc
+    
+    yield from universal_chunker(single_doc_gen(), config)