alita-sdk 0.3.351__py3-none-any.whl → 0.3.499__py3-none-any.whl

alita_sdk/tools/cloud/aws/__init__.py

@@ -6,7 +6,7 @@ from langchain_core.tools import BaseToolkit, BaseTool
 from .api_wrapper import AWSToolConfig
 from ...base.tool import BaseAction
 from ...elitea_base import filter_missconfigured_index_tools
-from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...utils import clean_string, get_max_toolkit_length
 
 name = "aws"
 
@@ -22,12 +22,10 @@ def get_tools(tool):
 
 class AWSToolkit(BaseToolkit):
     tools: list[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
         selected_tools = {x['name']: x['args_schema'].schema() for x in AWSToolConfig.model_construct().get_available_tools()}
-        AWSToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
         return create_model(
             name,
             region=(str, Field(default="", title="Region", description="AWS region")),
@@ -54,14 +52,17 @@ class AWSToolkit(BaseToolkit):
         aws_tool_config = AWSToolConfig(**kwargs)
         available_tools = aws_tool_config.get_available_tools()
         tools = []
-        prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         for tool in available_tools:
             if selected_tools and tool["name"] not in selected_tools:
                 continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=aws_tool_config,
-                name=prefix + tool["name"],
-                description=tool["description"],
+                name=tool["name"],
+                description=description,
                 args_schema=tool["args_schema"]
             ))
         return cls(tools=tools)

alita_sdk/tools/cloud/azure/__init__.py

@@ -6,7 +6,7 @@ from pydantic import create_model, BaseModel, ConfigDict, Field, SecretStr
 from .api_wrapper import AzureApiWrapper
 from ...base.tool import BaseAction
 from ...elitea_base import filter_missconfigured_index_tools
-from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...utils import clean_string, get_max_toolkit_length
 
 name = "azure"
 
@@ -23,12 +23,10 @@ def get_tools(tool):
 
 class AzureToolkit(BaseToolkit):
     tools: list[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
         selected_tools = {x['name']: x['args_schema'].schema() for x in AzureApiWrapper.model_construct().get_available_tools()}
-        AzureToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
         return create_model(
             name,
             subscription_id=(str, Field(default="", title="Subscription ID", description="Azure subscription ID")),
@@ -47,14 +45,17 @@ class AzureToolkit(BaseToolkit):
         azure_api_wrapper = AzureApiWrapper(**kwargs)
         available_tools = azure_api_wrapper.get_available_tools()
         tools = []
-        prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         for tool in available_tools:
             if selected_tools and tool["name"] not in selected_tools:
                 continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=azure_api_wrapper,
-                name=prefix + tool["name"],
-                description=tool["description"],
+                name=tool["name"],
+                description=description,
                 args_schema=tool["args_schema"]
             ))
         return cls(tools=tools)

alita_sdk/tools/cloud/gcp/__init__.py

@@ -6,7 +6,7 @@ from pydantic import create_model, BaseModel, ConfigDict, Field, SecretStr
 from .api_wrapper import GCPApiWrapper
 from ...base.tool import BaseAction
 from ...elitea_base import filter_missconfigured_index_tools
-from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...utils import clean_string, get_max_toolkit_length
 
 name = "gcp"
 
@@ -20,12 +20,10 @@ def get_tools(tool):
 
 class GCPToolkit(BaseToolkit):
     tools: list[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
         selected_tools = {x['name']: x['args_schema'].schema() for x in GCPApiWrapper.model_construct().get_available_tools()}
-        GCPToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
         return create_model(
             name,
             api_key=(SecretStr, Field(default="", title="API key", description="GCP API key", json_schema_extra={'secret': True})),
@@ -41,14 +39,17 @@ class GCPToolkit(BaseToolkit):
         gcp_api_wrapper = GCPApiWrapper(**kwargs)
         available_tools = gcp_api_wrapper.get_available_tools()
         tools = []
-        prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         for tool in available_tools:
             if selected_tools and tool["name"] not in selected_tools:
                 continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=gcp_api_wrapper,
-                name=prefix + tool["name"],
-                description=tool["description"],
+                name=tool["name"],
+                description=description,
                 args_schema=tool["args_schema"]
             ))
         return cls(tools=tools)

alita_sdk/tools/cloud/k8s/__init__.py

@@ -6,7 +6,7 @@ from pydantic import create_model, BaseModel, ConfigDict, Field, SecretStr
 from .api_wrapper import KubernetesApiWrapper
 from ...base.tool import BaseAction
 from ...elitea_base import filter_missconfigured_index_tools
-from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...utils import clean_string, get_max_toolkit_length
 
 name = "kubernetes"
 
@@ -21,12 +21,10 @@ def get_tools(tool):
 
 class KubernetesToolkit(BaseToolkit):
     tools: list[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
         selected_tools = {x['name']: x['args_schema'].schema() for x in KubernetesApiWrapper.model_construct().get_available_tools()}
-        KubernetesToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
         return create_model(
             name,
             url=(str, Field(default="", title="Cluster URL", description="The URL of the Kubernetes cluster")),
@@ -51,14 +49,17 @@ class KubernetesToolkit(BaseToolkit):
         kubernetes_api_wrapper = KubernetesApiWrapper(**kwargs)
         available_tools = kubernetes_api_wrapper.get_available_tools()
         tools = []
-        prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         for tool in available_tools:
             if selected_tools and tool["name"] not in selected_tools:
                 continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=kubernetes_api_wrapper,
-                name=prefix + tool["name"],
-                description=tool["description"],
+                name=tool["name"],
+                description=description,
                 args_schema=tool["args_schema"]
             ))
         return cls(tools=tools)

alita_sdk/tools/code/linter/__init__.py

@@ -5,7 +5,7 @@ from pydantic import BaseModel, create_model, Field
 
 from .api_wrapper import PythonLinter
 from ...base.tool import BaseAction
-from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...utils import clean_string, get_max_toolkit_length
 
 name = "python_linter"
 
@@ -19,11 +19,9 @@ def get_tools(tool):
 
 class PythonLinterToolkit(BaseToolkit):
     tools: list[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
-        PythonLinterToolkit.toolkit_max_length = get_max_toolkit_length([])
         return create_model(
             name,
             error_codes=(str, Field(description="Error codes to be used by the linter")),
@@ -39,15 +37,17 @@ class PythonLinterToolkit(BaseToolkit):
         python_linter = PythonLinter(**kwargs)
         available_tools = python_linter.get_available_tools()
         tools = []
-        toolkit_max_length = get_max_toolkit_length(selected_tools)
-        prefix = clean_string(toolkit_name, PythonLinterToolkit.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         for tool in available_tools:
             if selected_tools and tool["name"] not in selected_tools:
                 continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=python_linter,
-                name=prefix + tool["name"],
-                description=tool["description"],
+                name=tool["name"],
+                description=description,
                 args_schema=tool["args_schema"]
             ))
         return cls(tools=tools)

alita_sdk/tools/code/loaders/codesearcher.py

@@ -4,8 +4,9 @@ def search_format(items):
     results = []
     for (doc, score) in items:
         res_chunk = ''
-        language = get_programming_language(get_file_extension(doc.metadata["filename"]))
-        res_chunk += doc.metadata["filename"] + " -> " + doc.metadata["method_name"] + " (score: " + str(score) + ")"
+        language = get_programming_language(get_file_extension(doc.metadata.get("filename", "unknown")))
+        method_name = doc.metadata.get("method_name", "text")
+        res_chunk += doc.metadata.get("filename", "unknown") + " -> " + method_name + " (score: " + str(score) + ")"
         res_chunk += "\n\n```" + language.value + "\n"+ doc.page_content + "\n```\n\n"
         results.append(res_chunk)
     return results

alita_sdk/tools/code/sonar/__init__.py

@@ -5,7 +5,7 @@ from pydantic import create_model, BaseModel, ConfigDict, Field
 from .api_wrapper import SonarApiWrapper
 from ...base.tool import BaseAction
 from ...elitea_base import filter_missconfigured_index_tools
-from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...utils import clean_string, get_max_toolkit_length
 from ....configurations.sonar import SonarConfiguration
 
 name = "sonar"
@@ -21,15 +21,13 @@ def get_tools(tool):
 
 class SonarToolkit(BaseToolkit):
     tools: list[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
         selected_tools = {x['name']: x['args_schema'].schema() for x in SonarApiWrapper.model_construct().get_available_tools()}
-        SonarToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
         return create_model(
             name,
-            sonar_project_name=(str, Field(description="Project name of the desired repository", json_schema_extra={'toolkit_name': True, 'max_toolkit_length': SonarToolkit.toolkit_max_length})),
+            sonar_project_name=(str, Field(description="Project name of the desired repository")),
             sonar_configuration=(SonarConfiguration, Field(description="Sonar Configuration", json_schema_extra={'configuration_types': ['sonar']})),
             selected_tools=(List[Literal[tuple(selected_tools)]], Field(default=[], json_schema_extra={'args_schemas': selected_tools})),
             __config__=ConfigDict(json_schema_extra=
@@ -55,14 +53,17 @@ class SonarToolkit(BaseToolkit):
         sonar_api_wrapper = SonarApiWrapper(**wrapper_payload)
         available_tools = sonar_api_wrapper.get_available_tools()
         tools = []
-        prefix = clean_string(toolkit_name, SonarToolkit.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         for tool in available_tools:
             if selected_tools and tool["name"] not in selected_tools:
                 continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=sonar_api_wrapper,
-                name=prefix + tool["name"],
-                description=tool["description"],
+                name=tool["name"],
+                description=description,
                 args_schema=tool["args_schema"]
             ))
         return cls(tools=tools)

alita_sdk/tools/code_indexer_toolkit.py

@@ -0,0 +1,199 @@
+import ast
+import fnmatch
+import json
+import logging
+from typing import Optional, List, Generator
+
+from langchain_core.documents import Document
+from langchain_core.tools import ToolException
+from pydantic import Field
+
+from alita_sdk.tools.base_indexer_toolkit import BaseIndexerToolkit
+
+logger = logging.getLogger(__name__)
+
+
+class CodeIndexerToolkit(BaseIndexerToolkit):
+    def _get_indexed_data(self, index_name: str):
+        self._ensure_vectorstore_initialized()
+        if not self.vector_adapter:
+            raise ToolException("Vector adapter is not initialized. "
+                             "Check your configuration: embedding_model and vectorstore_type.")
+        return self.vector_adapter.get_code_indexed_data(self, index_name)
+
+    def key_fn(self, document: Document):
+        return document.metadata.get("filename")
+
+    def compare_fn(self, document: Document, idx_data):
+        return (document.metadata.get('commit_hash') and
+            idx_data.get('commit_hashes') and
+            document.metadata.get('commit_hash') in idx_data.get('commit_hashes')
+        )
+
+    def remove_ids_fn(self, idx_data, key: str):
+        return idx_data[key]['ids']
+
+    def _base_loader(
+            self,
+            branch: Optional[str] = None,
+            whitelist: Optional[List[str]] = None,
+            blacklist: Optional[List[str]] = None,
+            **kwargs) -> Generator[Document, None, None]:
+        """Index repository files in the vector store using code parsing."""
+        yield from self.loader(
+            branch=branch,
+            whitelist=whitelist,
+            blacklist=blacklist
+        )
+
+    def _extend_data(self, documents: Generator[Document, None, None]):
+        yield from documents
+
+    def _index_tool_params(self):
+        """Return the parameters for indexing data."""
+        return {
+            "branch": (Optional[str], Field(
+                description="Branch to index files from. Defaults to active branch if None.",
+                default=None)),
+            "whitelist": (Optional[List[str]], Field(
+                description='File extensions or paths to include. Defaults to all files if None. Example: ["*.md", "*.java"]',
+                default=None)),
+            "blacklist": (Optional[List[str]], Field(
+                description='File extensions or paths to exclude. Defaults to no exclusions if None. Example: ["*.md", "*.java"]',
+                default=None)),
+        }
+
+    def loader(self,
+               branch: Optional[str] = None,
+               whitelist: Optional[List[str]] = None,
+               blacklist: Optional[List[str]] = None,
+               chunked: bool = True) -> Generator[Document, None, None]:
+        """
+        Generates Documents from files in a branch, respecting whitelist and blacklist patterns.
+    
+        Parameters:
+        - branch (Optional[str]): Branch for listing files. Defaults to the current branch if None.
+        - whitelist (Optional[List[str]]): File extensions or paths to include. Defaults to all files if None.
+        - blacklist (Optional[List[str]]): File extensions or paths to exclude. Defaults to no exclusions if None.
+        - chunked (bool): If True (default), applies universal chunker based on file type.
+                         If False, returns raw Documents without chunking.
+    
+        Returns:
+        - generator: Yields Documents from files matching the whitelist but not the blacklist.
+                    Each document has exactly the key 'filename' in metadata, which is used as an ID
+                    for further operations (indexing, deduplication, and retrieval).
+    
+        Example:
+        # Use 'feature-branch', include '.py' files, exclude 'test_' files
+        for doc in loader(branch='feature-branch', whitelist=['*.py'], blacklist=['*test_*']):
+            print(doc.page_content)
+    
+        Notes:
+        - Whitelist and blacklist use Unix shell-style wildcards.
+        - Files must match the whitelist and not the blacklist to be included.
+        - Each document MUST have exactly the key 'filename' in metadata. This key is used as an ID
+          for further operations such as indexing, deduplication, and retrieval.
+        - When chunked=True:
+          - .md files → markdown chunker (header-based splitting)
+          - .py/.js/.ts/etc → code parser (TreeSitter-based)
+          - .json files → JSON chunker
+          - other files → default text chunker
+        """
+        import hashlib
+    
+        _files = self.__handle_get_files("", self.__get_branch(branch))
+        self._log_tool_event(message="Listing files in branch", tool_name="loader")
+        logger.info(f"Files in branch: {_files}")
+    
+        def is_whitelisted(file_path: str) -> bool:
+            if whitelist:
+                return (any(fnmatch.fnmatch(file_path, pattern) for pattern in whitelist)
+                        or any(file_path.endswith(f'.{pattern}') for pattern in whitelist))
+            return True
+    
+        def is_blacklisted(file_path: str) -> bool:
+            if blacklist:
+                return (any(fnmatch.fnmatch(file_path, pattern) for pattern in blacklist)
+                        or any(file_path.endswith(f'.{pattern}') for pattern in blacklist))
+            return False
+    
+        def raw_document_generator() -> Generator[Document, None, None]:
+            """Yields raw Documents without chunking."""
+            self._log_tool_event(message="Reading the files", tool_name="loader")
+            total_files = len(_files)
+            processed = 0
+            
+            for idx, file in enumerate(_files, 1):
+                if is_whitelisted(file) and not is_blacklisted(file):
+                    try:
+                        file_content = self._read_file(file, self.__get_branch(branch))
+                    except Exception as e:
+                        logger.error(f"Failed to read file {file}: {e}")
+                        continue
+                    
+                    if not file_content:
+                        continue
+                    
+                    # Ensure file content is a string
+                    if isinstance(file_content, bytes):
+                        file_content = file_content.decode("utf-8", errors="ignore")
+                    elif isinstance(file_content, dict) and file.endswith('.json'):
+                        file_content = json.dumps(file_content)
+                    elif not isinstance(file_content, str):
+                        file_content = str(file_content)
+                    
+                    # Hash the file content for uniqueness tracking
+                    file_hash = hashlib.sha256(file_content.encode("utf-8")).hexdigest()
+                    processed += 1
+                    
+                    yield Document(
+                        page_content=file_content,
+                        metadata={
+                            'file_path': file,
+                            'filename': file,
+                            'source': file,
+                            'commit_hash': file_hash,
+                        }
+                    )
+                
+                if idx % 10 == 0 or idx == total_files:
+                    self._log_tool_event(
+                        message=f"{idx} out of {total_files} files checked, {processed} matched",
+                        tool_name="loader"
+                    )
+            
+            self._log_tool_event(message=f"{processed} files loaded", tool_name="loader")
+
+        if not chunked:
+            # Return raw documents without chunking
+            return raw_document_generator()
+        
+        # Apply universal chunker based on file type
+        from .chunkers.universal_chunker import universal_chunker
+        return universal_chunker(raw_document_generator())
+
+    def __handle_get_files(self, path: str, branch: str):
+        """
+        Handles the retrieval of files from a specific path and branch.
+        This method should be implemented in subclasses to provide the actual file retrieval logic.
+        """
+        _files = self._get_files(path=path, branch=branch)
+        if isinstance(_files, str):
+            try:
+                # Attempt to convert the string to a list using ast.literal_eval
+                _files = ast.literal_eval(_files)
+                # Ensure that the result is actually a list of strings
+                if not isinstance(_files, list) or not all(isinstance(item, str) for item in _files):
+                    raise ValueError("The evaluated result is not a list of strings")
+            except (SyntaxError, ValueError):
+                # Handle the case where the string cannot be converted to a list
+                raise ValueError("Expected a list of strings, but got a string that cannot be converted")
+
+            # Ensure _files is a list of strings
+        if not isinstance(_files, list) or not all(isinstance(item, str) for item in _files):
+            raise ValueError("Expected a list of strings")
+        return _files
+
+    def __get_branch(self, branch):
+       return (branch or getattr(self, 'active_branch', None)
+               or getattr(self, '_active_branch', None) or getattr(self, 'branch', None))

alita_sdk/tools/confluence/__init__.py

@@ -6,7 +6,7 @@ from ..base.tool import BaseAction
 from pydantic import create_model, BaseModel, ConfigDict, Field
 
 from ..elitea_base import filter_missconfigured_index_tools
-from ..utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length, parse_list, check_connection_response
+from ..utils import clean_string, get_max_toolkit_length, parse_list, check_connection_response
 from ...configurations.confluence import ConfluenceConfiguration
 from ...configurations.pgvector import PgVectorConfiguration
 import requests
@@ -38,13 +38,11 @@ def get_tools(tool):
 
 class ConfluenceToolkit(BaseToolkit):
     tools: List[BaseTool] = []
-    toolkit_max_length: int = 0
 
     @staticmethod
     def toolkit_config_schema() -> BaseModel:
         selected_tools = {x['name']: x['args_schema'].schema() for x in
                           ConfluenceAPIWrapper.model_construct().get_available_tools()}
-        ConfluenceToolkit.toolkit_max_length = get_max_toolkit_length(selected_tools)
 
         @check_connection_response
         def check_connection(self):
@@ -67,8 +65,7 @@ class ConfluenceToolkit(BaseToolkit):
 
         model = create_model(
             name,
-            space=(str, Field(description="Space", json_schema_extra={'toolkit_name': True,
-                                                                                    'max_toolkit_length': ConfluenceToolkit.toolkit_max_length})),
+            space=(str, Field(description="Space")),
             cloud=(bool, Field(description="Hosting Option", json_schema_extra={'configuration': True})),
             limit=(int, Field(description="Pages limit per request", default=5)),
             labels=(Optional[str], Field(
@@ -115,17 +112,21 @@ class ConfluenceToolkit(BaseToolkit):
             **(kwargs.get('pgvector_configuration') or {}),
         }
         confluence_api_wrapper = ConfluenceAPIWrapper(**wrapper_payload)
-        prefix = clean_string(toolkit_name, ConfluenceToolkit.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         available_tools = confluence_api_wrapper.get_available_tools()
         tools = []
         for tool in available_tools:
             if selected_tools:
                 if tool["name"] not in selected_tools:
                     continue
+            description = tool["description"]
+            if toolkit_name:
+                description = f"Toolkit: {toolkit_name}\n{description}"
+            description = f"Confluence space: {confluence_api_wrapper.space}\n{description}"
+            description = description[:1000]
             tools.append(BaseAction(
                 api_wrapper=confluence_api_wrapper,
-                name=prefix + tool["name"],
-                description=f"Confluence space: {confluence_api_wrapper.space}" + tool["description"],
+                name=tool["name"],
+                description=description,
                 args_schema=tool["args_schema"]
             ))
         return cls(tools=tools)