alita-sdk 0.3.229__py3-none-any.whl → 0.3.231__py3-none-any.whl

alita_sdk/tools/ado/wiki/__init__.py

@@ -1,9 +1,11 @@
 from typing import List, Literal, Optional
 from .ado_wrapper import AzureDevOpsApiWrapper  # Import the API wrapper for Azure DevOps
 from langchain_core.tools import BaseTool, BaseToolkit
-from pydantic import create_model, BaseModel, Field, SecretStr
+from pydantic import create_model, BaseModel, Field
 
 import requests
+from ....configurations.ado import AdoConfiguration
+from ....configurations.pgvector import PgVectorConfiguration
 from ...base.tool import BaseAction
 from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length, check_connection_response
 
@@ -25,22 +27,9 @@ class AzureDevOpsWikiToolkit(BaseToolkit):
                                  'toolkit_name': True,
                                  'max_toolkit_length': AzureDevOpsWikiToolkit.toolkit_max_length})
                   ),
-            organization_url=(str, Field(title="Organization URL",
-                                         description="ADO organization url", json_schema_extra={
-                    'configuration': True,
-                })),
-            project=(str, Field(description="ADO project",
-                                json_schema_extra={
-                                    'configuration': True
-                                })),
-            token=(SecretStr,
-                   Field(description="ADO token",
-                         json_schema_extra={'secret': True, 'configuration': True }
-                         )),
+            ado_configuration=(AdoConfiguration, Field(description="Ado configuration", json_schema_extra={'configuration_types': ['ado']})),
             # indexer settings
-            connection_string=(Optional[SecretStr], Field(description="Connection string for vectorstore",
-                                                          default=None,
-                                                          json_schema_extra={'secret': True})),
+            pgvector_configuration=(Optional[PgVectorConfiguration], Field(description="PgVector Configuration", json_schema_extra={'configuration_types': ['pgvector']})),
             # embedder settings
             embedding_model=(str, Field(description="Embedding model: i.e. 'HuggingFaceEmbeddings', etc.",
                                         default="HuggingFaceEmbeddings")),
@@ -53,7 +42,7 @@ class AzureDevOpsWikiToolkit(BaseToolkit):
                 'json_schema_extra': {
                     'metadata': {
                         "label": "ADO wiki",
-                        "icon_url": None,
+                        "icon_url": "ado-wiki-icon.svg",
                         "categories": ["documentation"],
                         "extra_categories": ["knowledge base", "documentation management", "wiki"],
                         "sections": {
@@ -77,9 +66,12 @@ class AzureDevOpsWikiToolkit(BaseToolkit):
 
         @check_connection_response
         def check_connection(self):
+            ado_config = self.ado_wiki_configuration.ado_configuration if self.ado_wiki_configuration else None
+            if not ado_config:
+                raise ValueError("ADO wiki configuration is required")
             response = requests.get(
-                f'{self.organization_url}/{self.project}/_apis/wiki/wikis?api-version=7.0',
-                headers={'Authorization': f'Bearer {self.token}'},
+                f'{ado_config.organization_url}/{ado_config.project}/_apis/wiki/wikis?api-version=7.0',
+                headers={'Authorization': f'Bearer {ado_config.token}'},
                 timeout=5
             )
             return response
@@ -94,7 +86,13 @@ class AzureDevOpsWikiToolkit(BaseToolkit):
             environ['AZURE_DEVOPS_CACHE_DIR'] = '/tmp/.azure-devops'
         if selected_tools is None:
             selected_tools = []
-        azure_devops_api_wrapper = AzureDevOpsApiWrapper(**kwargs)
+        wrapper_payload = {
+            **kwargs,
+            # TODO use ado_configuration fields in AzureDevOpsApiWrapper
+            **kwargs['ado_configuration'],
+            **(kwargs.get('pgvector_configuration') or {}),
+        }
+        azure_devops_api_wrapper = AzureDevOpsApiWrapper(**wrapper_payload)
         available_tools = azure_devops_api_wrapper.get_available_tools()
         tools = []
         prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''

alita_sdk/tools/ado/wiki/ado_wrapper.py

@@ -61,6 +61,7 @@ RenamePageInput = create_model(
 
 
 class AzureDevOpsApiWrapper(BaseVectorStoreToolApiWrapper):
+    # TODO use ado_configuration instead of organization_url, project and token
     organization_url: str
     project: str
     token: SecretStr

alita_sdk/tools/ado/work_item/__init__.py

@@ -1,9 +1,11 @@
 from typing import List, Optional, Literal
 from .ado_wrapper import AzureDevOpsApiWrapper  # Import the API wrapper for Azure DevOps
 from langchain_core.tools import BaseTool, BaseToolkit
-from pydantic import create_model, BaseModel, Field, SecretStr
+from pydantic import create_model, BaseModel, Field
 
 import requests
+from ....configurations.ado import AdoConfiguration
+from ....configurations.pgvector import PgVectorConfiguration
 from ...base.tool import BaseAction
 from ...utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length, check_connection_response
 
@@ -24,23 +26,11 @@ class AzureDevOpsWorkItemsToolkit(BaseToolkit):
                                  'toolkit_name': True,
                                  'max_toolkit_length': AzureDevOpsWorkItemsToolkit.toolkit_max_length})
                   ),
-            organization_url=(str, Field(title="Organization URL",
-                                         description="ADO organization url",
-                                         json_schema_extra={
-                                             'configuration': True
-                                         })),
-            project=(str, Field(description="ADO project",
-                                json_schema_extra={
-                                    'configuration': True
-                                }
-                                )),
-            token=(SecretStr, Field(description="ADO token", json_schema_extra={'secret': True, 'configuration': True})),
+            ado_configuration=(AdoConfiguration, Field(description="Ado Work Item configuration", json_schema_extra={'configuration_types': ['ado_work_item']})),
             limit=(Optional[int], Field(description="ADO plans limit used for limitation of the list with results", default=5)),
             selected_tools=(List[Literal[tuple(selected_tools)]], Field(default=[], json_schema_extra={'args_schemas': selected_tools})),
             # indexer settings
-            connection_string = (Optional[SecretStr], Field(description="Connection string for vectorstore",
-                                                            default=None,
-                                                            json_schema_extra={'secret': True})),
+            pgvector_configuration=(Optional[PgVectorConfiguration], Field(description="PgVector Configuration", json_schema_extra={'configuration_types': ['pgvector']})),
             # embedder settings
             embedding_model=(str, Field(description="Embedding model: i.e. 'HuggingFaceEmbeddings', etc.",
                                         default="HuggingFaceEmbeddings")),
@@ -51,7 +41,7 @@ class AzureDevOpsWorkItemsToolkit(BaseToolkit):
                 'json_schema_extra': {
                     'metadata': {
                         "label": "ADO boards",
-                        "icon_url": None,
+                        "icon_url": "ado-boards-icon.svg",
                         "categories": ["project management"],
                         "extra_categories": ["work item management", "issue tracking", "agile boards"],
                         "sections": {
@@ -75,9 +65,12 @@ class AzureDevOpsWorkItemsToolkit(BaseToolkit):
 
         @check_connection_response
         def check_connection(self):
+            ado_config = self.ado_work_item_configuration.ado_configuration if self.ado_work_item_configuration else None
+            if not ado_config:
+                raise ValueError("ADO work item configuration is required")
             response = requests.get(
-                f'{self.organization_url}/{self.project}/_apis/wit/workitemtypes?api-version=7.0',
-                headers={'Authorization': f'Bearer {self.token}'},
+                f'{ado_config.organization_url}/{ado_config.project}/_apis/wit/workitemtypes?api-version=7.0',
+                headers={'Authorization': f'Bearer {ado_config.token}'},
                 timeout=5
             )
             return response
@@ -93,7 +86,13 @@ class AzureDevOpsWorkItemsToolkit(BaseToolkit):
         if selected_tools is None:
             selected_tools = []
 
-        azure_devops_api_wrapper = AzureDevOpsApiWrapper(**kwargs)
+        wrapper_payload = {
+            **kwargs,
+            # TODO use ado_configuration fields in AzureDevOpsApiWrapper
+            **kwargs['ado_configuration'],
+            **(kwargs.get('pgvector_configuration') or {}),
+        }
+        azure_devops_api_wrapper = AzureDevOpsApiWrapper(**wrapper_payload)
         available_tools = azure_devops_api_wrapper.get_available_tools()
         tools = []
         prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''

alita_sdk/tools/ado/work_item/ado_wrapper.py

@@ -95,6 +95,7 @@ ADOUnlinkWorkItemsFromWikiPage = create_model(
 )
 
 class AzureDevOpsApiWrapper(BaseVectorStoreToolApiWrapper):
+    # TODO use ado_configuration instead of organization_url, project and token
     organization_url: str
     project: str
     token: SecretStr

alita_sdk/tools/azure_ai/search/__init__.py

@@ -15,10 +15,8 @@ name = "azure_search"
 def get_tools(tool):
     return AzureSearchToolkit().get_toolkit(
             selected_tools=tool['settings'].get('selected_tools', []),
-            api_key=tool['settings'].get('azure_search_configuration', {}).get('api_key', None),
-            endpoint=tool['settings'].get('azure_search_configuration', {}).get('endpoint', None),
+            azure_search_configuration=tool['settings']['azure_search_configuration'],
             index_name=tool['settings'].get('index_name', None),
-            api_base=tool['settings'].get('azure_search_configuration', {}).get('api_base', None),
             api_version=tool['settings'].get('api_version', None),
             openai_api_key=tool['settings'].get('access_token', None),
             model_name=tool['settings'].get('model_name', None),
@@ -71,7 +69,12 @@ class AzureSearchToolkit(BaseToolkit):
     def get_toolkit(cls, selected_tools: list[str] | None = None, toolkit_name: Optional[str] = None, **kwargs):
         if selected_tools is None:
             selected_tools = []
-        azure_search_api_wrapper = AzureSearchApiWrapper(**kwargs)
+        wrapper_payload = {
+            **kwargs,
+            # TODO use azure_search_configuration fields
+            **kwargs['azure_search_configuration'],
+        }
+        azure_search_api_wrapper = AzureSearchApiWrapper(**wrapper_payload)
         available_tools = azure_search_api_wrapper.get_available_tools()
         prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         tools = []

alita_sdk/tools/bitbucket/__init__.py

@@ -22,13 +22,12 @@ def get_tools(tool):
         url=tool['settings']['url'],
         project=tool['settings']['project'],
         repository=tool['settings']['repository'],
-        username=tool['settings'].get('bitbucket_configuration', {}).get('username', ''),
-        password=tool['settings'].get('bitbucket_configuration', {}).get('password', ''),
+        bitbucket_configuration=tool['settings']['bitbucket_configuration'],
         branch=tool['settings']['branch'],
         cloud=tool['settings'].get('cloud'),
         llm=tool['settings'].get('llm', None),
         alita=tool['settings'].get('alita', None),
-        connection_string=tool['settings'].get('pgvector_configuration', {}).get('connection_string', None),
+        pgvector_configuration=tool['settings'].get('pgvector_configuration', {}),
         collection_name=str(tool['toolkit_name']),
         doctype='code',
         embedding_model="HuggingFaceEmbeddings",
@@ -91,7 +90,13 @@ class AlitaBitbucketToolkit(BaseToolkit):
             selected_tools = []
         if kwargs["cloud"] is None:
             kwargs["cloud"] = True if "bitbucket.org" in kwargs.get('url') else False
-        bitbucket_api_wrapper = BitbucketAPIWrapper(**kwargs)
+        wrapper_payload = {
+            **kwargs,
+            # TODO use bitbucket_configuration fields
+            **kwargs['bitbucket_configuration'],
+            **(kwargs.get('pgvector_configuration') or {}),
+        }
+        bitbucket_api_wrapper = BitbucketAPIWrapper(**wrapper_payload)
         available_tools: List[Dict] = __all__
         prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         tools = []

alita_sdk/tools/confluence/__init__.py

@@ -17,9 +17,7 @@ def get_tools(tool):
         base_url=tool['settings']['base_url'],
         space=tool['settings'].get('space', None),
         cloud=tool['settings'].get('cloud', True),
-        api_key=tool['settings'].get('confluence_configuration', {}).get('api_key', None),
-        username=tool['settings'].get('confluence_configuration', {}).get('username', None),
-        token=tool['settings'].get('confluence_configuration', {}).get('token', None),
+        confluence_configuration=tool['settings']['confluence_configuration'],
         limit=tool['settings'].get('limit', 5),
         labels=parse_list(tool['settings'].get('labels', None)),
         additional_fields=tool['settings'].get('additional_fields', []),
@@ -28,7 +26,7 @@ def get_tools(tool):
         llm=tool['settings'].get('llm', None),
         toolkit_name=tool.get('toolkit_name'),
         # indexer settings
-        connection_string = tool['settings'].get('pgvector_configuration', {}).get('connection_string', None),
+        pgvector_configuration=tool['settings'].get('pgvector_configuration', {}),
         collection_name=str(tool['toolkit_name']),
         doctype='doc',
         embedding_model="HuggingFaceEmbeddings",
@@ -120,7 +118,13 @@ class ConfluenceToolkit(BaseToolkit):
     def get_toolkit(cls, selected_tools: list[str] | None = None, toolkit_name: Optional[str] = None, **kwargs):
         if selected_tools is None:
             selected_tools = []
-        confluence_api_wrapper = ConfluenceAPIWrapper(**kwargs)
+        wrapper_payload = {
+            **kwargs,
+            # TODO use confluence_configuration fields
+            **kwargs['confluence_configuration'],
+            **(kwargs.get('pgvector_configuration') or {}),
+        }
+        confluence_api_wrapper = ConfluenceAPIWrapper(**wrapper_payload)
         prefix = clean_string(toolkit_name, ConfluenceToolkit.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         available_tools = confluence_api_wrapper.get_available_tools()
         tools = []

alita_sdk/tools/elitea_base.py

@@ -30,13 +30,13 @@ LoaderSchema = create_model(
 # Base Vector Store Schema Models
 BaseIndexParams = create_model(
     "BaseIndexParams",
-    collection_suffix=(Optional[str], Field(description="Optional suffix for collection name (max 7 characters)", default="", max_length=7)),
+    collection_suffix=(str, Field(description="Suffix for collection name (max 7 characters) used to separate datasets", min_length=1, max_length=7)),
     vectorstore_type=(Optional[str], Field(description="Vectorstore type (Chroma, PGVector, Elastic, etc.)", default="PGVector")),
 )
 
 BaseCodeIndexParams = create_model(
     "BaseCodeIndexParams",
-    collection_suffix=(Optional[str], Field(description="Optional suffix for collection name (max 7 characters)", default="", max_length=7)),
+    collection_suffix=(str, Field(description="Suffix for collection name (max 7 characters) used to separate datasets", min_length=1, max_length=7)),
     vectorstore_type=(Optional[str], Field(description="Vectorstore type (Chroma, PGVector, Elastic, etc.)", default="PGVector")),
     branch=(Optional[str], Field(description="Branch to index files from. Defaults to active branch if None.", default=None)),
     whitelist=(Optional[List[str]], Field(description="File extensions or paths to include. Defaults to all files if None.", default=None)),
@@ -51,7 +51,9 @@ RemoveIndexParams = create_model(
 BaseSearchParams = create_model(
     "BaseSearchParams",
     query=(str, Field(description="Query text to search in the index")),
-    collection_suffix=(Optional[str], Field(description="Optional suffix for collection name (max 7 characters)", default="", max_length=7)),
+    collection_suffix=(Optional[str], Field(
+        description="Optional suffix for collection name (max 7 characters). Leave empty to search across all datasets",
+        default="", max_length=7)),
     vectorstore_type=(Optional[str], Field(description="Vectorstore type (Chroma, PGVector, Elastic, etc.)", default="PGVector")),
     filter=(Optional[dict | str], Field(
         description="Filter to apply to the search results. Can be a dictionary or a JSON string.",
@@ -219,6 +221,7 @@ class BaseVectorStoreToolApiWrapper(BaseToolApiWrapper):
     embedding_model: Optional[str] = "HuggingFaceEmbeddings"
     embedding_model_params: Optional[Dict[str, Any]] = {"model_name": "sentence-transformers/all-MiniLM-L6-v2"}
     vectorstore_type: Optional[str] = "PGVector"
+    _vector_store: Optional[Any] = None
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -297,9 +300,9 @@ class BaseVectorStoreToolApiWrapper(BaseToolApiWrapper):
         collection_suffix = kwargs.get("collection_suffix")
         progress_step = kwargs.get("progress_step")
         clean_index = kwargs.get("clean_index")
-        vs = self._init_vector_store(collection_suffix, embeddings=embedding)
+        vs = self._init_vector_store(embeddings=embedding)
         #
-        return vs.index_documents(docs, progress_step=progress_step, clean_index=clean_index)
+        return vs.index_documents(docs, collection_suffix=collection_suffix, progress_step=progress_step, clean_index=clean_index)
 
     def _process_documents(self, documents: List[Document]) -> Generator[Document, None, None]:
         """
@@ -333,42 +336,31 @@ class BaseVectorStoreToolApiWrapper(BaseToolApiWrapper):
 
 
     # TODO: init store once and re-use the instance
-    def _init_vector_store(self, collection_suffix: str = "", embeddings: Optional[Any] = None):
+    def _init_vector_store(self, embeddings: Optional[Any] = None):
         """Initializes the vector store wrapper with the provided parameters."""
         try:
             from alita_sdk.runtime.tools.vectorstore import VectorStoreWrapper
         except ImportError:
             from alita_sdk.runtime.tools.vectorstore import VectorStoreWrapper
 
-        # Validate collection_suffix length
-        if collection_suffix and len(collection_suffix.strip()) > 7:
-            raise ToolException("collection_suffix must be 7 characters or less")
-
-        # Create collection name with suffix if provided
-        collection_name = str(self.collection_name)
-        if collection_suffix and collection_suffix.strip():
-            collection_name = f"{self.collection_name}_{collection_suffix.strip()}"
-
-        # Get database-specific parameters using adapter
-        connection_string = self.connection_string.get_secret_value() if self.connection_string else None
-        vectorstore_params = self._adapter.get_vectorstore_params(collection_name, connection_string)
-
-        return VectorStoreWrapper(
-            llm=self.llm,
-            vectorstore_type=self.vectorstore_type,
-            embedding_model=self.embedding_model,
-            embedding_model_params=self.embedding_model_params,
-            vectorstore_params=vectorstore_params,
-            embeddings=embeddings,
-            process_document_func=self._process_documents,
-        )
+        if not self._vector_store:
+            connection_string = self.connection_string.get_secret_value() if self.connection_string else None
+            vectorstore_params = self._adapter.get_vectorstore_params(self.collection_name, connection_string)
+            self._vector_store = VectorStoreWrapper(
+                llm=self.llm,
+                vectorstore_type=self.vectorstore_type,
+                embedding_model=self.embedding_model,
+                embedding_model_params=self.embedding_model_params,
+                vectorstore_params=vectorstore_params,
+                embeddings=embeddings,
+                process_document_func=self._process_documents,
+            )
+        return self._vector_store
 
     def remove_index(self, collection_suffix: str = ""):
         """Cleans the indexed data in the collection."""
-        vectorstore_wrapper = self._init_vector_store(collection_suffix)
-        collection_name = f"{self.collection_name}_{collection_suffix}" if collection_suffix else str(self.collection_name)
-        self._adapter.remove_collection(vectorstore_wrapper, collection_name)
-        return (f"Collection '{collection_name}' has been removed from the vector store.\n"
+        self._init_vector_store()._clean_collection(collection_suffix=collection_suffix)
+        return (f"Collection '{collection_suffix}' has been removed from the vector store.\n"
                 f"Available collections: {self.list_collections()}")
 
     def list_collections(self):
@@ -386,7 +378,14 @@ class BaseVectorStoreToolApiWrapper(BaseToolApiWrapper):
                      extended_search: Optional[List[str]] = None,
                      **kwargs):
         """ Searches indexed documents in the vector store."""
-        vectorstore = self._init_vector_store(collection_suffix)
+        vectorstore = self._init_vector_store()
+        # build filter on top of collection_suffix
+        filter = filter if isinstance(filter, dict) else json.loads(filter)
+        if collection_suffix:
+            filter.update({"collection": {
+                "$eq": collection_suffix.strip()
+            }})
+
         found_docs = vectorstore.search_documents(
             query,
             doctype=self.doctype,
@@ -579,22 +578,20 @@ class BaseCodeToolApiWrapper(BaseVectorStoreToolApiWrapper):
         return parse_code_files_for_db(file_content_generator())
     
     def index_data(self,
+                   collection_suffix: str,
                    branch: Optional[str] = None,
                    whitelist: Optional[List[str]] = None,
                    blacklist: Optional[List[str]] = None,
-                   collection_suffix: str = "",
                    **kwargs) -> str:
         """Index repository files in the vector store using code parsing."""
         
-        
-        
         documents = self.loader(
             branch=branch,
             whitelist=whitelist,
             blacklist=blacklist
         )
-        vectorstore = self._init_vector_store(collection_suffix)
-        return vectorstore.index_documents(documents, clean_index=False, is_code=True)
+        vectorstore = self._init_vector_store()
+        return vectorstore.index_documents(documents, collection_suffix=collection_suffix, clean_index=False, is_code=True)
 
     def _get_vector_search_tools(self):
         """

alita_sdk/tools/figma/__init__.py

@@ -6,6 +6,7 @@ from pydantic import BaseModel, ConfigDict, Field, create_model, SecretStr
 from ..base.tool import BaseAction
 from .api_wrapper import FigmaApiWrapper, GLOBAL_LIMIT
 from ..utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
+from ...configurations.pgvector import PgVectorConfiguration
 
 name = "figma"
 
@@ -21,7 +22,7 @@ def get_tools(tool):
             toolkit_name=tool.get('toolkit_name'),
             # indexer settings
             llm=tool['settings'].get('llm', None),
-            connection_string = tool['settings'].get('connection_string', None),
+            pgvector_configuration=tool['settings'].get('pgvector_configuration', {}),
             collection_name=str(tool['toolkit_name']),
             doctype='doc',
             embedding_model="HuggingFaceEmbeddings",
@@ -54,9 +55,7 @@ class FigmaToolkit(BaseToolkit):
                 Field(default=[], json_schema_extra={"args_schemas": selected_tools}),
             ),
             # indexer settings
-            connection_string = (Optional[SecretStr], Field(description="Connection string for vectorstore",
-                                                            default=None,
-                                                            json_schema_extra={'secret': True})),
+            pgvector_configuration=(Optional[PgVectorConfiguration], Field(description="PgVector Configuration", json_schema_extra={'configuration_types': ['pgvector']})),
 
             # embedder settings
             embedding_model=(str, Field(description="Embedding model: i.e. 'HuggingFaceEmbeddings', etc.", default="HuggingFaceEmbeddings")),
@@ -93,7 +92,11 @@ class FigmaToolkit(BaseToolkit):
     def get_toolkit(cls, selected_tools: list[str] | None = None, toolkit_name: Optional[str] = None, **kwargs):
         if selected_tools is None:
             selected_tools = []
-        figma_api_wrapper = FigmaApiWrapper(**kwargs)
+        wrapper_payload = {
+            **kwargs,
+            **(kwargs.get('pgvector_configuration') or {}),
+        }
+        figma_api_wrapper = FigmaApiWrapper(**wrapper_payload)
         prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         available_tools = figma_api_wrapper.get_available_tools()
         tools = []

alita_sdk/tools/figma/api_wrapper.py

@@ -4,7 +4,7 @@ import json
 import logging
 import re
 from enum import Enum
-from typing import Dict, Generator, Optional, Union
+from typing import Dict, List, Generator, Optional, Union
 
 import requests
 from FigmaPy import FigmaPy
@@ -13,6 +13,7 @@ from langchain_core.tools import ToolException
 from pydantic import Field, PrivateAttr, create_model, model_validator, SecretStr
 
 from ..elitea_base import BaseVectorStoreToolApiWrapper, extend_with_vector_tools
+from ..utils.content_parser import load_content_from_bytes
 
 GLOBAL_LIMIT = 10000
 
@@ -235,15 +236,27 @@ class FigmaApiWrapper(BaseVectorStoreToolApiWrapper):
     global_regexp: Optional[str] = Field(default=None)
     _client: Optional[FigmaPy] = PrivateAttr()
 
-    def _base_loader(self, project_id: str, **kwargs) -> Generator[Document, None, None]:
-        files = json.loads(self.get_project_files(project_id)).get('files', [])
-        for file in files:
-            yield Document(page_content=json.dumps(file), metadata={
-                'id': file.get('key', ''),
-                'file_key': file.get('key', ''),
-                'name': file.get('name', ''),
-                'updated_on': file.get('last_modified', '')
-            })
+    def _base_loader(self, project_id: Optional[str] = None, file_keys: Optional[List[str]] = None, **kwargs) -> Generator[Document, None, None]:
+        files = []
+        if project_id:
+            files = json.loads(self.get_project_files(project_id)).get('files', [])
+            for file in files:
+                yield Document(page_content=json.dumps(file), metadata={
+                    'id': file.get('key', ''),
+                    'file_key': file.get('key', ''),
+                    'name': file.get('name', ''),
+                    'updated_on': file.get('last_modified', '')
+                })
+        elif file_keys:
+            for file_key in file_keys:
+                file = self._client.get_file(file_key)
+                metadata = {
+                    'id': file_key,
+                    'file_key': file_key,
+                    'name': file.name,
+                    'updated_on': file.last_modified
+                }
+                yield Document(page_content=json.dumps(metadata), metadata=metadata)
 
     def _process_document(self, document: Document) -> Generator[Document, None, None]:
         file_key = document.metadata.get('id', '')
@@ -257,12 +270,19 @@ class FigmaApiWrapper(BaseVectorStoreToolApiWrapper):
 
         # iterate over images values
         for node_id, image_url in images.items():
+            if not image_url:
+                logging.warning(f"Image URL not found for node_id {node_id} in file {file_key}. Skipping.")
+                continue
             response = requests.get(image_url)
             if response.status_code == 200:
                 content_type = response.headers.get('Content-Type', '')
                 if 'text/html' not in content_type.lower():
+                    extension = f".{content_type.split('/')[-1]}" if content_type.startswith('image') else '.txt'
+                    page_content = load_content_from_bytes(
+                        file_content=response.content,
+                        extension=extension, llm = self.llm)
                     yield Document(
-                                    page_content=base64.b64encode(response.content).decode("utf-8"),
+                                    page_content=page_content,
                                     metadata={
                                         'file_key': file_key,
                                         'node_id': node_id,
@@ -273,7 +293,12 @@ class FigmaApiWrapper(BaseVectorStoreToolApiWrapper):
     def _index_tool_params(self):
         """Return the parameters for indexing data."""
         return {
-            "project_id": (str, Field(description="ID of the project to list files from", examples=["55391681"]))
+            "project_id": (Optional[str], Field(
+                description="ID of the project to list files from: i.e. '55391681'",
+                default=None)),
+            'file_keys': (Optional[List[str]], Field(
+                description="List of file keys to index: i.e. ['Fp24FuzPwH0L74ODSrCnQo', 'jmhAr6q78dJoMRqt48zisY']",
+                default=None))
         }
 
     def _send_request(

alita_sdk/tools/github/__init__.py

@@ -19,14 +19,10 @@ def _get_toolkit(tool) -> BaseToolkit:
         github_repository=tool['settings']['repository'],
         active_branch=tool['settings']['active_branch'],
         github_base_branch=tool['settings']['base_branch'],
-        github_access_token=tool['settings'].get('github_configuration', {}).get('access_token', ''),
-        github_username=tool['settings'].get('github_configuration', {}).get('username', ''),
-        github_password=tool['settings'].get('github_configuration', {}).get('password', ''),
-        github_app_id=tool['settings'].get('github_configuration', {}).get('app_id', None),
-        github_app_private_key=tool['settings'].get('github_configuration', {}).get('app_private_key', None),
+        github_configuration=tool['settings']['github_configuration'],
         llm=tool['settings'].get('llm', None),
         alita=tool['settings'].get('alita', None),
-        connection_string=tool['settings'].get('pgvector_configuration', {}).get('connection_string', None),
+        pgvector_configuration=tool['settings'].get('pgvector_configuration', {}),
         collection_name=str(tool['toolkit_name']),
         doctype='code',
         embedding_model="HuggingFaceEmbeddings",
@@ -71,9 +67,6 @@ class AlitaGitHubToolkit(BaseToolkit):
             active_branch=(Optional[str], Field(description="Active branch", default="main")),
             base_branch=(Optional[str], Field(description="Github Base branch", default="main")),
             # indexer settings
-            connection_string=(Optional[SecretStr], Field(description="Connection string for vectorstore",
-                                                          default=None,
-                                                          json_schema_extra={'secret': True})),
             embedding_model=(str, Field(description="Embedding model: i.e. 'HuggingFaceEmbeddings', etc.", default="HuggingFaceEmbeddings")),
             embedding_model_params=(dict, Field(description="Embedding model parameters: i.e. `{'model_name': 'sentence-transformers/all-MiniLM-L6-v2'}", default={"model_name": "sentence-transformers/all-MiniLM-L6-v2"})),
             selected_tools=(List[Literal[tuple(selected_tools)]],
@@ -84,7 +77,13 @@ class AlitaGitHubToolkit(BaseToolkit):
     def get_toolkit(cls, selected_tools: list[str] | None = None, toolkit_name: Optional[str] = None, **kwargs):
         if selected_tools is None:
             selected_tools = []
-        github_api_wrapper = AlitaGitHubAPIWrapper(**kwargs)
+        wrapper_payload = {
+            **kwargs,
+            # TODO use github_configuration fields
+            **kwargs['github_configuration'],
+            **(kwargs.get('pgvector_configuration') or {}),
+        }
+        github_api_wrapper = AlitaGitHubAPIWrapper(**wrapper_payload)
         available_tools: List[Dict] = github_api_wrapper.get_available_tools()
         tools = []
         prefix = clean_string(toolkit_name, AlitaGitHubToolkit.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''

alita_sdk/tools/gitlab/__init__.py

@@ -20,11 +20,11 @@ def get_tools(tool):
         url=tool['settings']['url'],
         repository=tool['settings']['repository'],
         branch=tool['settings']['branch'],
-        private_token=tool['settings'].get('gitlab_configuration', {}).get('private_token', ''),
+        gitlab_configuration=tool['settings']['gitlab_configuration'],
 
         llm=tool['settings'].get('llm', None),
         alita=tool['settings'].get('alita', None),
-        connection_string=tool['settings'].get('pgvector_configuration', {}).get('connection_string', None),
+        pgvector_configuration=tool['settings'].get('pgvector_configuration', {}),
         collection_name=str(tool['toolkit_name']),
         doctype='code',
         embedding_model="HuggingFaceEmbeddings",
@@ -49,8 +49,7 @@ class AlitaGitlabToolkit(BaseToolkit):
             gitlab_configuration=(Optional[GitlabConfiguration], Field(description="GitLab configuration", json_schema_extra={'configuration_types': ['gitlab']})),
             branch=(str, Field(description="Main branch", default="main")),
             # indexer settings
-            pgvector_configuration=(Optional[PgVectorConfiguration], Field(description="PgVector configuration", default=None,
-                                    json_schema_extra={'configuration_types': ['pgvector']})),
+            pgvector_configuration=(Optional[PgVectorConfiguration], Field(description="PgVector Configuration", json_schema_extra={'configuration_types': ['pgvector']})),
             # embedder settings
             embedding_model=(str, Field(description="Embedding model: i.e. 'HuggingFaceEmbeddings', etc.", default="HuggingFaceEmbeddings")),
             embedding_model_params=(dict, Field(description="Embedding model parameters: i.e. `{'model_name': 'sentence-transformers/all-MiniLM-L6-v2'}", default={"model_name": "sentence-transformers/all-MiniLM-L6-v2"})),
@@ -80,7 +79,13 @@ class AlitaGitlabToolkit(BaseToolkit):
     def get_toolkit(cls, selected_tools: list[str] | None = None, toolkit_name: Optional[str] = None, **kwargs):
         if selected_tools is None:
             selected_tools = []
-        gitlab_api_wrapper = GitLabAPIWrapper(**kwargs)
+        wrapper_payload = {
+            **kwargs,
+            # TODO use gitlab_configuration fields
+            **kwargs['gitlab_configuration'],
+            **(kwargs.get('pgvector_configuration') or {}),
+        }
+        gitlab_api_wrapper = GitLabAPIWrapper(**wrapper_payload)
         prefix = clean_string(toolkit_name, cls.toolkit_max_length) + TOOLKIT_SPLITTER if toolkit_name else ''
         available_tools: List[Dict] = gitlab_api_wrapper.get_available_tools()
         tools = []