alita-sdk 0.3.263__tar.gz → 0.3.264__tar.gz

{alita_sdk-0.3.263 → alita_sdk-0.3.264}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.263
+Version: 0.3.264
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedjik@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.263 → alita_sdk-0.3.264}/alita_sdk/runtime/langchain/document_loaders/constants.py

@@ -127,11 +127,6 @@ loaders_map = {
             'extract_images': True
         }
     },
-    '.doc': {
-        'class': AlitaTextLoader,
-        'is_multimodal_processing': True,
-        'kwargs': {}
-    },
     '.json': {
         'class': AlitaJSONLoader,
         'is_multimodal_processing': False,

{alita_sdk-0.3.263 → alita_sdk-0.3.264}/alita_sdk/runtime/tools/vectorstore_base.py

@@ -1,5 +1,6 @@
 import json
 import math
+from collections import OrderedDict
 from logging import getLogger
 from typing import Any, Optional, List, Dict, Generator
 
@@ -398,11 +399,18 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
             
         # Initialize document map for tracking by ID
         doc_map = {
-            f"{doc.metadata.get('id', f'idx_{i}')}_{doc.metadata['chunk_id']}"
-            if 'chunk_id' in doc.metadata
-            else doc.metadata.get('id', f"idx_{i}"): (doc, score)
+            (
+                f"{doc.metadata.get('id', f'idx_{i}')}_{doc.metadata['chunk_id']}"
+                if 'chunk_id' in doc.metadata
+                else doc.metadata.get('id', f"idx_{i}")
+            ): (doc, 1 - score)
             for i, (doc, score) in enumerate(vector_items)
         }
+
+        # Sort the items by the new score in descending order
+        doc_map = OrderedDict(
+            sorted(doc_map.items(), key=lambda x: x[1][1], reverse=True)
+        )
         
         # Process full-text search if configured
         if full_text_search and full_text_search.get('enabled') and full_text_search.get('fields'):
@@ -452,7 +460,7 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
         
         # Apply cutoff filter
         if cut_off:
-            combined_items = [item for item in combined_items if abs(item[1]) <= cut_off]
+            combined_items = [item for item in combined_items if abs(item[1]) >= cut_off]
         
         # Sort by score and limit results
         # DISABLED: for chroma we want ascending order (lower score is better), for others descending

{alita_sdk-0.3.263 → alita_sdk-0.3.264}/alita_sdk/tools/base_indexer_toolkit.py

@@ -34,7 +34,7 @@ BaseSearchParams = create_model(
         default={},
         examples=["{\"key\": \"value\"}", "{\"status\": \"active\"}"]
     )),
-    cut_off=(Optional[float], Field(description="Cut-off score for search results", default=0.5)),
+    cut_off=(Optional[float], Field(description="Cut-off score for search results", default=0.5, ge=0, le=1)),
     search_top=(Optional[int], Field(description="Number of top results to return", default=10)),
     full_text_search=(Optional[Dict[str, Any]], Field(
         description="Full text search parameters. Can be a dictionary with search options.",
@@ -64,7 +64,7 @@ BaseStepbackSearchParams = create_model(
         default={},
         examples=["{\"key\": \"value\"}", "{\"status\": \"active\"}"]
     )),
-    cut_off=(Optional[float], Field(description="Cut-off score for search results", default=0.5)),
+    cut_off=(Optional[float], Field(description="Cut-off score for search results", default=0.5, ge=0, le=1)),
     search_top=(Optional[int], Field(description="Number of top results to return", default=10)),
     reranker=(Optional[dict], Field(
         description="Reranker configuration. Can be a dictionary with reranking parameters.",
@@ -100,11 +100,8 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
 
     doctype: str = "document"
 
-    llm: Any = None
     connection_string: Optional[SecretStr] = None
     collection_name: Optional[str] = None
-    embedding_model: Optional[str] = "HuggingFaceEmbeddings"
-    vectorstore_type: Optional[str] = "PGVector"
     _embedding: Optional[Any] = None
     alita: Any = None # Elitea client, if available
 
@@ -115,10 +112,7 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
         
         if 'embedding_model' not in kwargs:
             kwargs['embedding_model'] = 'HuggingFaceEmbeddings'
-        if 'vectorstore_type' not in kwargs:
-            kwargs['vectorstore_type'] = 'PGVector'
-        vectorstore_type = kwargs.get('vectorstore_type')
-        kwargs['vectorstore_params'] = VectorStoreAdapterFactory.create_adapter(vectorstore_type).get_vectorstore_params(collection_name, connection_string)
+        kwargs['vectorstore_params'] = VectorStoreAdapterFactory.create_adapter(self.vectorstore_type).get_vectorstore_params(collection_name, connection_string)
         kwargs['_embedding'] = kwargs.get('alita').get_embeddings(kwargs.get('embedding_model'))
         super().__init__(**kwargs)
 

{alita_sdk-0.3.263 → alita_sdk-0.3.264}/alita_sdk/tools/sharepoint/api_wrapper.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Optional, Generator
+import re
+from typing import Optional, Generator, List
 
 from langchain_core.documents import Document
 from langchain_core.tools import ToolException
@@ -23,7 +24,11 @@ ReadList = create_model(
 GetFiles = create_model(
     "GetFiles",
     folder_name=(Optional[str], Field(description="Folder name to get list of the files.", default=None)),
-    limit_files=(Optional[int], Field(description="Limit (maximum number) of files to be returned. Can be called with synonyms, such as First, Top, etc., or can be reflected just by a number for example 'Top 10 files'. Use default value if not specified in a query WITH NO EXTRA CONFIRMATION FROM A USER", default=100)),
+    limit_files=(Optional[int], Field(description="Limit (maximum number) of files to be returned."
+                                                  "Can be called with synonyms, such as First, Top, etc., "
+                                                  "or can be reflected just by a number for example 'Top 10 files'. "
+                                                  "Use default value if not specified in a query WITH NO EXTRA "
+                                                  "CONFIRMATION FROM A USER", default=100)),
 )
 
 ReadDocument = create_model(
@@ -100,7 +105,8 @@ class SharepointApiWrapper(NonCodeIndexerToolkit):
         """ If folder name is specified, lists all files in this folder under Shared Documents path. If folder name is empty, lists all files under root catalog (Shared Documents). Number of files is limited by limit_files (default is 100)."""
         try:
             result = []
-
+            if not limit_files:
+                limit_files = 100
             target_folder_url = f"Shared Documents/{folder_name}" if folder_name else "Shared Documents"
             files = (self._client.web.get_folder_by_server_relative_path(target_folder_url)
                      .get_files(True)
@@ -146,13 +152,45 @@ class SharepointApiWrapper(NonCodeIndexerToolkit):
                                   excel_by_sheets=excel_by_sheets,
                                   llm=self.llm)
 
+    def _index_tool_params(self):
+        return {
+            'limit_files': (Optional[int], Field(
+                description="Limit (maximum number) of files to be returned. Can be called with synonyms, "
+                            "such as First, Top, etc., or can be reflected just by a number for example 'Top 10 files'. "
+                            "Use default value if not specified in a query WITH NO EXTRA CONFIRMATION FROM A USER",
+                default=1000, ge=0)),
+            'include_extensions': (Optional[List[str]], Field(
+                description="List of file extensions to include when processing: i.e. ['*.png', '*.jpg']. "
+                            "If empty, all files will be processed (except skip_extensions).",
+                default=[])),
+            'skip_extensions': (Optional[List[str]], Field(
+                description="List of file extensions to skip when processing: i.e. ['*.png', '*.jpg']",
+                default=[])),
+        }
+
     def _base_loader(self, **kwargs) -> Generator[Document, None, None]:
         try:
-            all_files = self.get_files_list()
+            all_files = self.get_files_list(kwargs.get('limit_files', 10000))
         except Exception as e:
             raise ToolException(f"Unable to extract files: {e}")
 
+        include_extensions = kwargs.get('include_extensions', [])
+        skip_extensions = kwargs.get('skip_extensions', [])
+
         for file in all_files:
+            file_name = file.get('Name', '')
+
+            # Check if file should be skipped based on skip_extensions
+            if any(re.match(pattern.replace('*', '.*') + '$', file_name, re.IGNORECASE)
+                   for pattern in skip_extensions):
+                continue
+
+            # Check if file should be included based on include_extensions
+            # If include_extensions is empty, process all files (that weren't skipped)
+            if not (any(re.match(pattern.replace('*', '.*') + '$', file_name, re.IGNORECASE)
+                        for pattern in include_extensions)):
+                continue
+
             metadata = {
                 ("updated_on" if k == "Modified" else k): str(v)
                 for k, v in file.items()

{alita_sdk-0.3.263 → alita_sdk-0.3.264}/alita_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.263
+Version: 0.3.264
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedjik@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.263 → alita_sdk-0.3.264}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "alita_sdk"
-version = "0.3.263"
+version = "0.3.264"
 description = "SDK for building langchain agents using resources from Alita"
 readme = "README.md"
 requires-python = ">=3.10"