alita-sdk 0.3.262__tar.gz → 0.3.264__tar.gz

{alita_sdk-0.3.262 → alita_sdk-0.3.264}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.262
+Version: 0.3.264
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedjik@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.262 → alita_sdk-0.3.264}/alita_sdk/runtime/clients/artifact.py

@@ -77,4 +77,11 @@ class Artifact:
         if not bucket_name:
             bucket_name = self.bucket_name
         return self.create(artifact_name, new_data, bucket_name)
+    
+    def get_content_bytes(self,
+            artifact_name: str,
+            bucket_name: str = None):
+        if not bucket_name:
+            bucket_name = self.bucket_name
+        return self.client.download_artifact(bucket_name, artifact_name)
     

{alita_sdk-0.3.262 → alita_sdk-0.3.264}/alita_sdk/runtime/langchain/document_loaders/constants.py

@@ -127,11 +127,6 @@ loaders_map = {
             'extract_images': True
         }
     },
-    '.doc': {
-        'class': AlitaTextLoader,
-        'is_multimodal_processing': True,
-        'kwargs': {}
-    },
     '.json': {
         'class': AlitaJSONLoader,
         'is_multimodal_processing': False,

{alita_sdk-0.3.262 → alita_sdk-0.3.264}/alita_sdk/runtime/toolkits/artifact.py

@@ -3,7 +3,7 @@ from typing import List, Any, Literal, Optional
 from alita_sdk.tools.utils import clean_string, TOOLKIT_SPLITTER, get_max_toolkit_length
 from langchain_community.agent_toolkits.base import BaseToolkit
 from langchain_core.tools import BaseTool
-from pydantic import create_model, BaseModel, ConfigDict, Field, SecretStr
+from pydantic import create_model, BaseModel, ConfigDict, Field
 from pydantic.fields import FieldInfo
 from ..tools.artifact import ArtifactWrapper
 from alita_sdk.tools.base.tool import BaseAction

{alita_sdk-0.3.262 → alita_sdk-0.3.264}/alita_sdk/runtime/toolkits/tools.py

@@ -74,10 +74,9 @@ def get_tools(tools_list: list, alita_client, llm, memory_store: BaseStore = Non
                 selected_tools=tool['settings'].get('selected_tools', []),
                 llm=llm,
                 # indexer settings
-                connection_string=tool['settings'].get('connection_string', None),
-                collection_name=tool.get('toolkit_name'),
+                pgvector_configuration=tool['settings'].get('pgvector_configuration', {}),
                 embedding_model=tool['settings'].get('embedding_model'),
-                vectorstore_type="PGVector"
+                collection_name=f"{tool.get('toolkit_name')}",
             ).get_tools())
         elif tool['type'] == 'vectorstore':
             tools.extend(VectorStoreToolkit.get_toolkit(

{alita_sdk-0.3.262 → alita_sdk-0.3.264}/alita_sdk/runtime/tools/artifact.py

@@ -1,15 +1,17 @@
 import hashlib
 import json
 import logging
-from typing import Any, Optional, Generator, List
+from typing import Any, Optional, Generator
 
 from langchain_core.documents import Document
 from langchain_core.tools import ToolException
 from pydantic import create_model, Field, model_validator
 
-from alita_sdk.tools.elitea_base import BaseVectorStoreToolApiWrapper, extend_with_vector_tools
+from alita_sdk.tools.non_code_indexer_toolkit import NonCodeIndexerToolkit
+from alita_sdk.tools.utils.available_tools_decorator import extend_with_parent_available_tools
 
-class ArtifactWrapper(BaseVectorStoreToolApiWrapper):
+
+class ArtifactWrapper(NonCodeIndexerToolkit):
     bucket: str
     artifact: Optional[Any] = None
     
@@ -21,7 +23,7 @@ class ArtifactWrapper(BaseVectorStoreToolApiWrapper):
         if not values.get('bucket'):
             raise ValueError("Bucket is required.")
         values["artifact"] = values['alita'].artifact(values['bucket'])
-        return values
+        return super().validate_toolkit(values)
 
     def list_files(self, bucket_name = None, return_as_string = True):
         return self.artifact.list(bucket_name, return_as_string)
@@ -76,20 +78,18 @@ class ArtifactWrapper(BaseVectorStoreToolApiWrapper):
         hasher.update(file_name.encode('utf-8'))
         return hasher.hexdigest()
 
-    def _process_document(self, document: Document) -> Generator[Document, None, None]:
-        try:
-            page_content = self.read_file(document.metadata['name'], is_capture_image=True, excel_by_sheets=True)
-        except Exception as e:
-            logging.error(f"Failed while parsing the file 'document.metadata['Path']': {e}")
-        if isinstance(page_content, dict):
-            for key, value in page_content.items():
-                metadata = document.metadata
-                metadata['page'] = key
-                yield Document(page_content=str(value), metadata=metadata)
-        else:
-            document.page_content = json.dumps(str(page_content))
-
-    @extend_with_vector_tools
+    def _extend_data(self, documents: Generator[Document, None, None]):
+        for document in documents:
+            try:
+                page_content = self.artifact.get_content_bytes(artifact_name=document.metadata['name'])
+                document.metadata['loader_content'] = page_content
+                document.metadata['loader_content_type'] = document.metadata['name']
+                yield document
+            except Exception as e:
+                logging.error(f"Failed while parsing the file '{document.metadata['name']}': {e}")
+                yield document
+
+    @extend_with_parent_available_tools
     def get_available_tools(self):
         bucket_name = (Optional[str], Field(description="Name of the bucket to work with."
                                                         "If bucket is not specified by user directly, the name should be taken from chat history."

{alita_sdk-0.3.262 → alita_sdk-0.3.264}/alita_sdk/runtime/tools/vectorstore_base.py

@@ -1,5 +1,6 @@
 import json
 import math
+from collections import OrderedDict
 from logging import getLogger
 from typing import Any, Optional, List, Dict, Generator
 
@@ -398,11 +399,18 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
             
         # Initialize document map for tracking by ID
         doc_map = {
-            f"{doc.metadata.get('id', f'idx_{i}')}_{doc.metadata['chunk_id']}"
-            if 'chunk_id' in doc.metadata
-            else doc.metadata.get('id', f"idx_{i}"): (doc, score)
+            (
+                f"{doc.metadata.get('id', f'idx_{i}')}_{doc.metadata['chunk_id']}"
+                if 'chunk_id' in doc.metadata
+                else doc.metadata.get('id', f"idx_{i}")
+            ): (doc, 1 - score)
             for i, (doc, score) in enumerate(vector_items)
         }
+
+        # Sort the items by the new score in descending order
+        doc_map = OrderedDict(
+            sorted(doc_map.items(), key=lambda x: x[1][1], reverse=True)
+        )
         
         # Process full-text search if configured
         if full_text_search and full_text_search.get('enabled') and full_text_search.get('fields'):
@@ -452,7 +460,7 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
         
         # Apply cutoff filter
         if cut_off:
-            combined_items = [item for item in combined_items if abs(item[1]) <= cut_off]
+            combined_items = [item for item in combined_items if abs(item[1]) >= cut_off]
         
         # Sort by score and limit results
         # DISABLED: for chroma we want ascending order (lower score is better), for others descending

{alita_sdk-0.3.262 → alita_sdk-0.3.264}/alita_sdk/tools/base_indexer_toolkit.py

@@ -34,7 +34,7 @@ BaseSearchParams = create_model(
         default={},
         examples=["{\"key\": \"value\"}", "{\"status\": \"active\"}"]
     )),
-    cut_off=(Optional[float], Field(description="Cut-off score for search results", default=0.5)),
+    cut_off=(Optional[float], Field(description="Cut-off score for search results", default=0.5, ge=0, le=1)),
     search_top=(Optional[int], Field(description="Number of top results to return", default=10)),
     full_text_search=(Optional[Dict[str, Any]], Field(
         description="Full text search parameters. Can be a dictionary with search options.",
@@ -64,7 +64,7 @@ BaseStepbackSearchParams = create_model(
         default={},
         examples=["{\"key\": \"value\"}", "{\"status\": \"active\"}"]
     )),
-    cut_off=(Optional[float], Field(description="Cut-off score for search results", default=0.5)),
+    cut_off=(Optional[float], Field(description="Cut-off score for search results", default=0.5, ge=0, le=1)),
     search_top=(Optional[int], Field(description="Number of top results to return", default=10)),
     reranker=(Optional[dict], Field(
         description="Reranker configuration. Can be a dictionary with reranking parameters.",
@@ -100,11 +100,8 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
 
     doctype: str = "document"
 
-    llm: Any = None
     connection_string: Optional[SecretStr] = None
     collection_name: Optional[str] = None
-    embedding_model: Optional[str] = "HuggingFaceEmbeddings"
-    vectorstore_type: Optional[str] = "PGVector"
     _embedding: Optional[Any] = None
     alita: Any = None # Elitea client, if available
 
@@ -115,10 +112,7 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
         
         if 'embedding_model' not in kwargs:
             kwargs['embedding_model'] = 'HuggingFaceEmbeddings'
-        if 'vectorstore_type' not in kwargs:
-            kwargs['vectorstore_type'] = 'PGVector'
-        vectorstore_type = kwargs.get('vectorstore_type')
-        kwargs['vectorstore_params'] = VectorStoreAdapterFactory.create_adapter(vectorstore_type).get_vectorstore_params(collection_name, connection_string)
+        kwargs['vectorstore_params'] = VectorStoreAdapterFactory.create_adapter(self.vectorstore_type).get_vectorstore_params(collection_name, connection_string)
         kwargs['_embedding'] = kwargs.get('alita').get_embeddings(kwargs.get('embedding_model'))
         super().__init__(**kwargs)
 

{alita_sdk-0.3.262 → alita_sdk-0.3.264}/alita_sdk/tools/sharepoint/api_wrapper.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Optional, Generator
+import re
+from typing import Optional, Generator, List
 
 from langchain_core.documents import Document
 from langchain_core.tools import ToolException
@@ -23,7 +24,11 @@ ReadList = create_model(
 GetFiles = create_model(
     "GetFiles",
     folder_name=(Optional[str], Field(description="Folder name to get list of the files.", default=None)),
-    limit_files=(Optional[int], Field(description="Limit (maximum number) of files to be returned. Can be called with synonyms, such as First, Top, etc., or can be reflected just by a number for example 'Top 10 files'. Use default value if not specified in a query WITH NO EXTRA CONFIRMATION FROM A USER", default=100)),
+    limit_files=(Optional[int], Field(description="Limit (maximum number) of files to be returned."
+                                                  "Can be called with synonyms, such as First, Top, etc., "
+                                                  "or can be reflected just by a number for example 'Top 10 files'. "
+                                                  "Use default value if not specified in a query WITH NO EXTRA "
+                                                  "CONFIRMATION FROM A USER", default=100)),
 )
 
 ReadDocument = create_model(
@@ -100,7 +105,8 @@ class SharepointApiWrapper(NonCodeIndexerToolkit):
         """ If folder name is specified, lists all files in this folder under Shared Documents path. If folder name is empty, lists all files under root catalog (Shared Documents). Number of files is limited by limit_files (default is 100)."""
         try:
             result = []
-
+            if not limit_files:
+                limit_files = 100
             target_folder_url = f"Shared Documents/{folder_name}" if folder_name else "Shared Documents"
             files = (self._client.web.get_folder_by_server_relative_path(target_folder_url)
                      .get_files(True)
@@ -146,13 +152,45 @@ class SharepointApiWrapper(NonCodeIndexerToolkit):
                                   excel_by_sheets=excel_by_sheets,
                                   llm=self.llm)
 
+    def _index_tool_params(self):
+        return {
+            'limit_files': (Optional[int], Field(
+                description="Limit (maximum number) of files to be returned. Can be called with synonyms, "
+                            "such as First, Top, etc., or can be reflected just by a number for example 'Top 10 files'. "
+                            "Use default value if not specified in a query WITH NO EXTRA CONFIRMATION FROM A USER",
+                default=1000, ge=0)),
+            'include_extensions': (Optional[List[str]], Field(
+                description="List of file extensions to include when processing: i.e. ['*.png', '*.jpg']. "
+                            "If empty, all files will be processed (except skip_extensions).",
+                default=[])),
+            'skip_extensions': (Optional[List[str]], Field(
+                description="List of file extensions to skip when processing: i.e. ['*.png', '*.jpg']",
+                default=[])),
+        }
+
     def _base_loader(self, **kwargs) -> Generator[Document, None, None]:
         try:
-            all_files = self.get_files_list()
+            all_files = self.get_files_list(kwargs.get('limit_files', 10000))
         except Exception as e:
             raise ToolException(f"Unable to extract files: {e}")
 
+        include_extensions = kwargs.get('include_extensions', [])
+        skip_extensions = kwargs.get('skip_extensions', [])
+
         for file in all_files:
+            file_name = file.get('Name', '')
+
+            # Check if file should be skipped based on skip_extensions
+            if any(re.match(pattern.replace('*', '.*') + '$', file_name, re.IGNORECASE)
+                   for pattern in skip_extensions):
+                continue
+
+            # Check if file should be included based on include_extensions
+            # If include_extensions is empty, process all files (that weren't skipped)
+            if not (any(re.match(pattern.replace('*', '.*') + '$', file_name, re.IGNORECASE)
+                        for pattern in include_extensions)):
+                continue
+
             metadata = {
                 ("updated_on" if k == "Modified" else k): str(v)
                 for k, v in file.items()

{alita_sdk-0.3.262 → alita_sdk-0.3.264}/alita_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.262
+Version: 0.3.264
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedjik@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.262 → alita_sdk-0.3.264}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "alita_sdk"
-version = "0.3.262"
+version = "0.3.264"
 description = "SDK for building langchain agents using resources from Alita"
 readme = "README.md"
 requires-python = ">=3.10"