alita-sdk 0.3.257__py3-none-any.whl → 0.3.562__py3-none-any.whl

alita_sdk/tools/chunkers/universal_chunker.py

@@ -0,0 +1,270 @@
+"""
+Universal Chunker - Routes documents to appropriate chunkers based on file type.
+
+This module provides a universal chunking interface that automatically selects
+the appropriate chunking strategy based on the file extension:
+
+- .md, .markdown → Markdown chunker (header-based splitting)
+- .py, .js, .ts, .java, etc. → TreeSitter code chunker
+- .json → JSON chunker  
+- other → Default text chunker
+
+Usage:
+    from alita_sdk.tools.chunkers.universal_chunker import universal_chunker
+    
+    # Chunk documents from a loader
+    for chunk in universal_chunker(document_generator, config):
+        print(chunk.page_content)
+"""
+
+import logging
+import os
+from typing import Generator, Dict, Any, Optional
+from langchain_core.documents import Document
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+from .code.codeparser import parse_code_files_for_db
+from .sematic.markdown_chunker import markdown_chunker
+from .sematic.json_chunker import json_chunker
+
+logger = logging.getLogger(__name__)
+
+
+# File extension mappings
+MARKDOWN_EXTENSIONS = {'.md', '.markdown', '.mdown', '.mkd', '.mdx'}
+JSON_EXTENSIONS = {'.json', '.jsonl', '.jsonc'}
+CODE_EXTENSIONS = {
+    '.py', '.js', '.jsx', '.mjs', '.cjs', '.ts', '.tsx',
+    '.java', '.kt', '.rs', '.go', '.cpp', '.c', '.cs', 
+    '.hs', '.rb', '.scala', '.lua'
+}
+
+
+def get_file_extension(file_path: str) -> str:
+    """Extract file extension from path."""
+    return os.path.splitext(file_path)[-1].lower()
+
+
+def get_file_type(file_path: str) -> str:
+    """
+    Determine the file type category for chunking.
+    
+    Returns:
+        'markdown', 'json', 'code', or 'text'
+    """
+    ext = get_file_extension(file_path)
+    
+    if ext in MARKDOWN_EXTENSIONS:
+        return 'markdown'
+    elif ext in JSON_EXTENSIONS:
+        return 'json'
+    elif ext in CODE_EXTENSIONS:
+        return 'code'
+    else:
+        return 'text'
+
+
+def _default_text_chunker(
+    documents: Generator[Document, None, None], 
+    config: Dict[str, Any]
+) -> Generator[Document, None, None]:
+    """
+    Default text chunker for unknown file types.
+    Uses recursive character splitting.
+    """
+    chunk_size = config.get('chunk_size', 1000)
+    chunk_overlap = config.get('chunk_overlap', 100)
+    
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        length_function=len,
+    )
+    
+    for doc in documents:
+        chunks = splitter.split_documents([doc])
+        for idx, chunk in enumerate(chunks, 1):
+            chunk.metadata['chunk_id'] = idx
+            chunk.metadata['chunk_type'] = 'text'
+            chunk.metadata['method_name'] = 'text'
+            yield chunk
+
+
+def _code_chunker_from_documents(
+    documents: Generator[Document, None, None],
+    config: Dict[str, Any]
+) -> Generator[Document, None, None]:
+    """
+    Adapter to convert Document generator to code parser format.
+    """
+    def file_content_generator():
+        for doc in documents:
+            yield {
+                'file_name': doc.metadata.get('file_path', doc.metadata.get('filename', 'unknown')),
+                'file_content': doc.page_content,
+                'commit_hash': doc.metadata.get('commit_hash', ''),
+            }
+    
+    # parse_code_files_for_db returns chunks with proper metadata
+    for chunk in parse_code_files_for_db(file_content_generator()):
+        # Ensure file_path is preserved
+        if 'file_path' not in chunk.metadata and 'filename' in chunk.metadata:
+            chunk.metadata['file_path'] = chunk.metadata['filename']
+        yield chunk
+
+
+def universal_chunker(
+    documents: Generator[Document, None, None],
+    config: Optional[Dict[str, Any]] = None
+) -> Generator[Document, None, None]:
+    """
+    Universal chunker that routes documents to appropriate chunkers based on file type.
+    
+    Each document is inspected for its file extension (from metadata.file_path or
+    metadata.file_name) and routed to the appropriate chunker:
+    
+    - Markdown files → markdown_chunker (header-based splitting)
+    - JSON files → json_chunker (recursive JSON splitting)
+    - Code files → code parser (TreeSitter-based parsing)
+    - Other files → default text chunker (recursive character splitting)
+    
+    Args:
+        documents: Generator yielding Document objects with file content
+        config: Optional configuration dict with:
+            - markdown_config: Config for markdown chunker
+            - json_config: Config for JSON chunker
+            - code_config: Config for code chunker
+            - text_config: Config for default text chunker
+            
+    Yields:
+        Document objects with chunked content and preserved metadata
+    """
+    if config is None:
+        config = {}
+    
+    # Default configs for each chunker type
+    markdown_config = config.get('markdown_config', {
+        'strip_header': False,
+        'return_each_line': False,
+        'headers_to_split_on': [
+            ('#', 'Header 1'),
+            ('##', 'Header 2'),
+            ('###', 'Header 3'),
+            ('####', 'Header 4'),
+        ],
+        'max_tokens': 1024,
+        'token_overlap': 50,
+        'min_chunk_chars': 100,  # Merge chunks smaller than this
+    })
+    
+    json_config = config.get('json_config', {
+        'max_tokens': 512,
+    })
+    
+    code_config = config.get('code_config', {})
+    
+    text_config = config.get('text_config', {
+        'chunk_size': 1000,
+        'chunk_overlap': 100,
+    })
+    
+    # Buffer documents by type for batch processing
+    # This is more efficient than processing one at a time
+    markdown_docs = []
+    json_docs = []
+    code_docs = []
+    text_docs = []
+    
+    # Buffer size before flushing
+    BUFFER_SIZE = 10
+    
+    def flush_markdown():
+        if markdown_docs:
+            def gen():
+                for d in markdown_docs:
+                    yield d
+            for chunk in markdown_chunker(gen(), markdown_config):
+                yield chunk
+            markdown_docs.clear()
+    
+    def flush_json():
+        if json_docs:
+            def gen():
+                for d in json_docs:
+                    yield d
+            for chunk in json_chunker(gen(), json_config):
+                yield chunk
+            json_docs.clear()
+    
+    def flush_code():
+        if code_docs:
+            def gen():
+                for d in code_docs:
+                    yield d
+            for chunk in _code_chunker_from_documents(gen(), code_config):
+                yield chunk
+            code_docs.clear()
+    
+    def flush_text():
+        if text_docs:
+            def gen():
+                for d in text_docs:
+                    yield d
+            for chunk in _default_text_chunker(gen(), text_config):
+                yield chunk
+            text_docs.clear()
+    
+    for doc in documents:
+        # Get file path from metadata
+        file_path = (doc.metadata.get('file_path') or 
+                    doc.metadata.get('file_name') or 
+                    doc.metadata.get('source') or 
+                    'unknown')
+        
+        # Ensure file_path is in metadata for downstream use
+        doc.metadata['file_path'] = file_path
+        
+        file_type = get_file_type(file_path)
+        
+        if file_type == 'markdown':
+            markdown_docs.append(doc)
+            if len(markdown_docs) >= BUFFER_SIZE:
+                yield from flush_markdown()
+        elif file_type == 'json':
+            json_docs.append(doc)
+            if len(json_docs) >= BUFFER_SIZE:
+                yield from flush_json()
+        elif file_type == 'code':
+            code_docs.append(doc)
+            if len(code_docs) >= BUFFER_SIZE:
+                yield from flush_code()
+        else:
+            text_docs.append(doc)
+            if len(text_docs) >= BUFFER_SIZE:
+                yield from flush_text()
+    
+    # Flush remaining documents
+    yield from flush_markdown()
+    yield from flush_json()
+    yield from flush_code()
+    yield from flush_text()
+
+
+def chunk_single_document(
+    doc: Document,
+    config: Optional[Dict[str, Any]] = None
+) -> Generator[Document, None, None]:
+    """
+    Convenience function to chunk a single document.
+    
+    Args:
+        doc: Single Document to chunk
+        config: Optional chunker configuration
+        
+    Yields:
+        Chunked Document objects
+    """
+    def single_doc_gen():
+        yield doc
+    
+    yield from universal_chunker(single_doc_gen(), config)

alita_sdk/tools/cloud/aws/__init__.py

@@ -5,7 +5,8 @@ from langchain_core.tools import BaseToolkit, BaseTool
 
 from .api_wrapper import AWSToolConfig
 from ...base.tool import BaseAction
-from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...elitea_base import filter_missconfigured_index_tools
+from ...utils import clean_string, get_max_toolkit_length
 
 name = "aws"
 
@@ -21,12 +22,10 @@ def get_tools(tool):
 
 class AWSToolkit(BaseToolkit):
     tools: list[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
         selected_tools = {x['name']: x['args_schema'].schema() for x in AWSToolConfig.model_construct().get_available_tools()}
-        AWSToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
         return create_model(
             name,
             region=(str, Field(default="", title="Region", description="AWS region")),
@@ -46,21 +45,26 @@ class AWSToolkit(BaseToolkit):
         )
 
     @classmethod
+    @filter_missconfigured_index_tools
     def get_toolkit(cls, selected_tools: list[str] | None = None, toolkit_name: Optional[str] = None, **kwargs):
         if selected_tools is None:
             selected_tools = []
         aws_tool_config = AWSToolConfig(**kwargs)
         available_tools = aws_tool_config.get_available_tools()
         tools = []
-        prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         for tool in available_tools:
             if selected_tools and tool["name"] not in selected_tools:
                 continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=aws_tool_config,
-                name=prefix + tool["name"],
-                description=tool["description"],
-                args_schema=tool["args_schema"]
+                name=tool["name"],
+                description=description,
+                args_schema=tool["args_schema"],
+                metadata={"toolkit_name": toolkit_name} if toolkit_name else {}
             ))
         return cls(tools=tools)
 

alita_sdk/tools/cloud/azure/__init__.py

@@ -5,7 +5,8 @@ from pydantic import create_model, BaseModel, ConfigDict, Field, SecretStr
 
 from .api_wrapper import AzureApiWrapper
 from ...base.tool import BaseAction
-from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...elitea_base import filter_missconfigured_index_tools
+from ...utils import clean_string, get_max_toolkit_length
 
 name = "azure"
 
@@ -22,12 +23,10 @@ def get_tools(tool):
 
 class AzureToolkit(BaseToolkit):
     tools: list[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
         selected_tools = {x['name']: x['args_schema'].schema() for x in AzureApiWrapper.model_construct().get_available_tools()}
-        AzureToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
         return create_model(
             name,
             subscription_id=(str, Field(default="", title="Subscription ID", description="Azure subscription ID")),
@@ -39,21 +38,26 @@ class AzureToolkit(BaseToolkit):
         )
 
     @classmethod
+    @filter_missconfigured_index_tools
     def get_toolkit(cls, selected_tools: list[str] | None = None, toolkit_name: Optional[str] = None, **kwargs):
         if selected_tools is None:
             selected_tools = []
         azure_api_wrapper = AzureApiWrapper(**kwargs)
         available_tools = azure_api_wrapper.get_available_tools()
         tools = []
-        prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         for tool in available_tools:
             if selected_tools and tool["name"] not in selected_tools:
                 continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=azure_api_wrapper,
-                name=prefix + tool["name"],
-                description=tool["description"],
-                args_schema=tool["args_schema"]
+                name=tool["name"],
+                description=description,
+                args_schema=tool["args_schema"],
+                metadata={"toolkit_name": toolkit_name} if toolkit_name else {}
             ))
         return cls(tools=tools)
 

alita_sdk/tools/cloud/gcp/__init__.py

@@ -5,7 +5,8 @@ from pydantic import create_model, BaseModel, ConfigDict, Field, SecretStr
 
 from .api_wrapper import GCPApiWrapper
 from ...base.tool import BaseAction
-from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...elitea_base import filter_missconfigured_index_tools
+from ...utils import clean_string, get_max_toolkit_length
 
 name = "gcp"
 
@@ -19,12 +20,10 @@ def get_tools(tool):
 
 class GCPToolkit(BaseToolkit):
     tools: list[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
         selected_tools = {x['name']: x['args_schema'].schema() for x in GCPApiWrapper.model_construct().get_available_tools()}
-        GCPToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
         return create_model(
             name,
             api_key=(SecretStr, Field(default="", title="API key", description="GCP API key", json_schema_extra={'secret': True})),
@@ -33,21 +32,26 @@ class GCPToolkit(BaseToolkit):
         )
 
     @classmethod
+    @filter_missconfigured_index_tools
     def get_toolkit(cls, selected_tools: list[str] | None = None, toolkit_name: Optional[str] = None, **kwargs):
         if selected_tools is None:
             selected_tools = []
         gcp_api_wrapper = GCPApiWrapper(**kwargs)
         available_tools = gcp_api_wrapper.get_available_tools()
         tools = []
-        prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         for tool in available_tools:
             if selected_tools and tool["name"] not in selected_tools:
                 continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=gcp_api_wrapper,
-                name=prefix + tool["name"],
-                description=tool["description"],
-                args_schema=tool["args_schema"]
+                name=tool["name"],
+                description=description,
+                args_schema=tool["args_schema"],
+                metadata={"toolkit_name": toolkit_name} if toolkit_name else {}
             ))
         return cls(tools=tools)
 

alita_sdk/tools/cloud/k8s/__init__.py

@@ -5,7 +5,8 @@ from pydantic import create_model, BaseModel, ConfigDict, Field, SecretStr
 
 from .api_wrapper import KubernetesApiWrapper
 from ...base.tool import BaseAction
-from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...elitea_base import filter_missconfigured_index_tools
+from ...utils import clean_string, get_max_toolkit_length
 
 name = "kubernetes"
 
@@ -20,12 +21,10 @@ def get_tools(tool):
 
 class KubernetesToolkit(BaseToolkit):
     tools: list[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
         selected_tools = {x['name']: x['args_schema'].schema() for x in KubernetesApiWrapper.model_construct().get_available_tools()}
-        KubernetesToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
         return create_model(
             name,
             url=(str, Field(default="", title="Cluster URL", description="The URL of the Kubernetes cluster")),
@@ -43,21 +42,26 @@ class KubernetesToolkit(BaseToolkit):
         )
 
     @classmethod
+    @filter_missconfigured_index_tools
     def get_toolkit(cls, selected_tools: list[str] | None = None, toolkit_name: Optional[str] = None, **kwargs):
         if selected_tools is None:
             selected_tools = []
         kubernetes_api_wrapper = KubernetesApiWrapper(**kwargs)
         available_tools = kubernetes_api_wrapper.get_available_tools()
         tools = []
-        prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         for tool in available_tools:
             if selected_tools and tool["name"] not in selected_tools:
                 continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=kubernetes_api_wrapper,
-                name=prefix + tool["name"],
-                description=tool["description"],
-                args_schema=tool["args_schema"]
+                name=tool["name"],
+                description=description,
+                args_schema=tool["args_schema"],
+                metadata={"toolkit_name": toolkit_name} if toolkit_name else {}
             ))
         return cls(tools=tools)
 

alita_sdk/tools/code/linter/__init__.py

@@ -5,7 +5,7 @@ from pydantic import BaseModel, create_model, Field
 
 from .api_wrapper import PythonLinter
 from ...base.tool import BaseAction
-from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...utils import clean_string, get_max_toolkit_length
 
 name = "python_linter"
 
@@ -19,11 +19,9 @@ def get_tools(tool):
 
 class PythonLinterToolkit(BaseToolkit):
     tools: list[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
-        PythonLinterToolkit.toolkit_max_length = get_max_toolkit_length([])
         return create_model(
             name,
             error_codes=(str, Field(description="Error codes to be used by the linter")),
@@ -39,16 +37,19 @@ class PythonLinterToolkit(BaseToolkit):
         python_linter = PythonLinter(**kwargs)
         available_tools = python_linter.get_available_tools()
         tools = []
-        toolkit_max_length = get_max_toolkit_length(selected_tools)
-        prefix = clean_string(toolkit_name, PythonLinterToolkit.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         for tool in available_tools:
             if selected_tools and tool["name"] not in selected_tools:
                 continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=python_linter,
-                name=prefix + tool["name"],
-                description=tool["description"],
-                args_schema=tool["args_schema"]
+                name=tool["name"],
+                description=description,
+                args_schema=tool["args_schema"],
+                metadata={"toolkit_name": toolkit_name} if toolkit_name else {}
             ))
         return cls(tools=tools)
 

alita_sdk/tools/code/loaders/codesearcher.py

@@ -4,8 +4,9 @@ def search_format(items):
     results = []
     for (doc, score) in items:
         res_chunk = ''
-        language = get_programming_language(get_file_extension(doc.metadata["filename"]))
-        res_chunk += doc.metadata["filename"] + " -> " + doc.metadata["method_name"] + " (score: " + str(score) + ")"
+        language = get_programming_language(get_file_extension(doc.metadata.get("filename", "unknown")))
+        method_name = doc.metadata.get("method_name", "text")
+        res_chunk += doc.metadata.get("filename", "unknown") + " -> " + method_name + " (score: " + str(score) + ")"
         res_chunk += "\n\n```" + language.value + "\n"+ doc.page_content + "\n```\n\n"
         results.append(res_chunk)
     return results

alita_sdk/tools/code/sonar/__init__.py

@@ -1,36 +1,34 @@
 from typing import List, Literal, Optional
 from langchain_core.tools import BaseToolkit, BaseTool
-from pydantic import create_model, BaseModel, ConfigDict, Field, SecretStr
+from pydantic import create_model, BaseModel, ConfigDict, Field
 
 from .api_wrapper import SonarApiWrapper
 from ...base.tool import BaseAction
-from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...elitea_base import filter_missconfigured_index_tools
+from ...utils import clean_string, get_max_toolkit_length
+from ....configurations.sonar import SonarConfiguration
 
 name = "sonar"
 
 def get_tools(tool):
     return SonarToolkit().get_toolkit(
         selected_tools=tool['settings'].get('selected_tools', []),
-        url=tool['settings']['url'],
-        sonar_token=tool['settings']['sonar_token'],
         sonar_project_name=tool['settings']['sonar_project_name'],
+        sonar_configuration=tool['settings']['sonar_configuration'],
         toolkit_name=tool.get('toolkit_name')
     ).get_tools()
 
 
 class SonarToolkit(BaseToolkit):
     tools: list[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
         selected_tools = {x['name']: x['args_schema'].schema() for x in SonarApiWrapper.model_construct().get_available_tools()}
-        SonarToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
         return create_model(
             name,
-            url=(str, Field(description="SonarQube Server URL", json_schema_extra={'toolkit_name': True, 'max_toolkit_length': SonarToolkit.toolkit_max_length})),
-            sonar_token=(SecretStr, Field(description="SonarQube user token for authentication", json_schema_extra={'secret': True})),
             sonar_project_name=(str, Field(description="Project name of the desired repository")),
+            sonar_configuration=(SonarConfiguration, Field(description="Sonar Configuration", json_schema_extra={'configuration_types': ['sonar']})),
             selected_tools=(List[Literal[tuple(selected_tools)]], Field(default=[], json_schema_extra={'args_schemas': selected_tools})),
             __config__=ConfigDict(json_schema_extra=
                                   {
@@ -44,21 +42,30 @@ class SonarToolkit(BaseToolkit):
         )
 
     @classmethod
+    @filter_missconfigured_index_tools
     def get_toolkit(cls, selected_tools: list[str] | None = None, toolkit_name: Optional[str] = None, **kwargs):
         if selected_tools is None:
             selected_tools = []
-        sonar_api_wrapper = SonarApiWrapper(**kwargs)
+        wrapper_payload = {
+            **kwargs,
+            **kwargs.get('sonar_configuration', {}),
+        }
+        sonar_api_wrapper = SonarApiWrapper(**wrapper_payload)
         available_tools = sonar_api_wrapper.get_available_tools()
         tools = []
-        prefix = clean_string(toolkit_name, SonarToolkit.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         for tool in available_tools:
             if selected_tools and tool["name"] not in selected_tools:
                 continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=sonar_api_wrapper,
-                name=prefix + tool["name"],
-                description=tool["description"],
-                args_schema=tool["args_schema"]
+                name=tool["name"],
+                description=description,
+                args_schema=tool["args_schema"],
+                metadata={"toolkit_name": toolkit_name} if toolkit_name else {}
             ))
         return cls(tools=tools)