alita-sdk 0.3.371__tar.gz → 0.3.373__tar.gz

{alita_sdk-0.3.371 → alita_sdk-0.3.373}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.371
+Version: 0.3.373
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedj27@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.371 → alita_sdk-0.3.373}/alita_sdk/runtime/langchain/document_loaders/constants.py

@@ -30,10 +30,18 @@ from enum import Enum
 
 
 class LoaderProperties(Enum):
-    LLM = 'llm'
+    LLM = 'use_llm'
+    PROMPT_DEFAULT = 'use_default_prompt'
     PROMPT = 'prompt'
-    PROMPT_DEFAULT = 'prompt_default'
 
+DEFAULT_ALLOWED_BASE = {'max_tokens': 512}
+
+DEFAULT_ALLOWED_WITH_LLM = {
+    **DEFAULT_ALLOWED_BASE,
+    LoaderProperties.LLM.value: False,
+    LoaderProperties.PROMPT_DEFAULT.value: False,
+    LoaderProperties.PROMPT.value: "",
+}
 
 # Image file loaders mapping
 image_loaders_map = {
@@ -42,66 +50,42 @@ image_loaders_map = {
         'mime_type': 'image/png',
         'is_multimodal_processing': True,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ],
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM,
     },
     '.jpg': {
         'class': AlitaImageLoader,
         'mime_type': 'image/jpeg',
         'is_multimodal_processing': True,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.jpeg': {
         'class': AlitaImageLoader,
         'mime_type': 'image/jpeg',
         'is_multimodal_processing': True,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.gif': {
         'class': AlitaImageLoader,
         'mime_type': 'image/gif',
         'is_multimodal_processing': True,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.bmp': {
         'class': AlitaImageLoader,
         'mime_type': 'image/bmp',
         'is_multimodal_processing': True,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.svg': {
         'class': AlitaImageLoader,
         'mime_type': 'image/svg+xml',
         'is_multimodal_processing': True,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     }
 }
 
@@ -114,7 +98,7 @@ document_loaders_map = {
         'kwargs': {
             'autodetect_encoding': True
         },
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.yml': {
         'class': AlitaTextLoader,
@@ -123,7 +107,7 @@ document_loaders_map = {
         'kwargs': {
             'autodetect_encoding': True
         },
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.yaml': {
         'class': AlitaTextLoader,
@@ -132,7 +116,7 @@ document_loaders_map = {
         'kwargs': {
             'autodetect_encoding': True
         },
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.groovy': {
         'class': AlitaTextLoader,
@@ -141,14 +125,14 @@ document_loaders_map = {
         'kwargs': {
             'autodetect_encoding': True
         },
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.md': {
         'class': AlitaMarkdownLoader,
         'mime_type': 'text/markdown',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.csv': {
         'class': AlitaCSVLoader,
@@ -159,7 +143,7 @@ document_loaders_map = {
             'raw_content': True,
             'cleanse': False
         },
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.xlsx': {
         'class': AlitaExcelLoader,
@@ -171,11 +155,7 @@ document_loaders_map = {
             'raw_content': True,
             'cleanse': False
         },
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.xls': {
         'class': AlitaExcelLoader,
@@ -186,22 +166,14 @@ document_loaders_map = {
             'raw_content': True,
             'cleanse': False
         },
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.pdf': {
         'class': AlitaPDFLoader,
         'mime_type': 'application/pdf',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.docx': {
         'class': AlitaDocxMammothLoader,
@@ -211,58 +183,42 @@ document_loaders_map = {
         'kwargs': {
             'extract_images': True
         },
-        'allowed_to_override': [
-            'max_tokens', 'mode', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': {**DEFAULT_ALLOWED_WITH_LLM, 'mode': 'paged'}
     },
     '.json': {
         'class': AlitaJSONLoader,
         'mime_type': 'application/json',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.jsonl': {
         'class': AirbyteJSONLoader,
         'mime_type': 'application/jsonl',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.htm': {
         'class': UnstructuredHTMLLoader,
         'mime_type': 'text/html',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.html': {
         'class': UnstructuredHTMLLoader,
         'mime_type': 'text/html',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.xml': {
         'class': UnstructuredXMLLoader,
         'mime_type': 'text/xml',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.ppt': {
         'class': AlitaPowerPointLoader,
@@ -271,11 +227,7 @@ document_loaders_map = {
         'kwargs': {
             'mode': 'paged'
         },
-        'allowed_to_override': [
-            'max_tokens', 'mode', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': {**DEFAULT_ALLOWED_WITH_LLM, 'mode': 'paged'}
     },
     '.pptx': {
         'class': AlitaPowerPointLoader,
@@ -285,20 +237,19 @@ document_loaders_map = {
         'kwargs': {
             'mode': 'paged'
         },
-        'allowed_to_override': [
-            'max_tokens', 'mode', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value,
-            "pages_per_chunk",
-            "extract_images"
-        ]
+        'allowed_to_override': {
+            **DEFAULT_ALLOWED_WITH_LLM,
+            'mode': 'paged',
+            'pages_per_chunk': 5,
+            'extract_images': False,
+        }
     },
     '.py': {
         'class': AlitaPythonLoader,
         'mime_type': 'text/x-python',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     }
 }
 
@@ -335,10 +286,16 @@ default_loader_config = {
     'mime_type': 'text/plain',
     'is_multimodal_processing': False,
     'kwargs': {},
-    'allowed_to_override': ['max_tokens']
+    'allowed_to_override': DEFAULT_ALLOWED_BASE
 }
 
 code_loaders_map = {ext: default_loader_config for ext in code_extensions}
 
 # Combined mapping for backward compatibility
 loaders_map = {**image_loaders_map, **document_loaders_map, **code_loaders_map}
+
+loaders_allowed_to_override = {
+    extension: config.get('allowed_to_override')
+    for extension, config in loaders_map.items()
+    if 'allowed_to_override' in config
+}

{alita_sdk-0.3.371 → alita_sdk-0.3.373}/alita_sdk/runtime/tools/vectorstore_base.py

@@ -177,6 +177,37 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
             except Exception as e:
                 logger.error(f"Failed to initialize PGVectorSearch: {str(e)}")
 
+    def _similarity_search_with_score(self, query: str, filter: dict = None, k: int = 10):
+        """
+        Perform similarity search with proper exception handling for DataException.
+
+        Args:
+            query: Search query string
+            filter: Optional filter dictionary
+            k: Number of results to return
+
+        Returns:
+            List of (Document, score) tuples
+
+        Raises:
+            ToolException: When DataException occurs or other search errors
+        """
+        try:
+            return self.vectorstore.similarity_search_with_score(
+                query, filter=filter, k=k
+            )
+        except DataException as dimException:
+            exception_str = str(dimException)
+            if 'different vector dimensions' in exception_str:
+                logger.error(f"Data exception: {exception_str}")
+                raise ToolException(f"Global search cannot be completed since collections were indexed using "
+                                    f"different embedding models. Use search within a single collection."
+                                    f"\nDetails: {exception_str}")
+            raise ToolException(f"Data exception during search. Possibly invalid filter: {exception_str}")
+        except Exception as e:
+            logger.error(f"Error during similarity search: {str(e)}")
+            raise ToolException(f"Search failed: {str(e)}")
+
     def list_collections(self) -> List[str]:
         """List all collections in the vectorstore."""
 
@@ -311,7 +342,7 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
                 }
                 
             try:
-                document_items = self.vectorstore.similarity_search_with_score(
+                document_items = self._similarity_search_with_score(
                     query, filter=document_filter, k=search_top
                 )                
                 # Add document results to unique docs
@@ -324,15 +355,6 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
                     if doc_id not in unique_docs or score > chunk_type_scores.get(doc_id, 0):
                         unique_docs[doc_id] = doc
                         chunk_type_scores[doc_id] = score
-            except DataException as dimException:
-                exception_str = str(dimException)
-                if 'different vector dimensions' in exception_str:
-                    logger.error(f"Data exception: {exception_str}")
-                    raise ToolException(f"Global search cannot be completed since collections were indexed using "
-                                        f"different embedding models. Use search within a single collection."
-                                        f"\nDetails: {exception_str}")
-                raise ToolException(f"Data exception during search. Possibly invalid filter: {exception_str}")
-
             except Exception as e:
                 logger.warning(f"Error searching for document chunks: {str(e)}")
             
@@ -353,18 +375,16 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
                     }
                 
                 try:
-                    chunk_items = self.vectorstore.similarity_search_with_score(
+                    chunk_items = self._similarity_search_with_score(
                         query, filter=chunk_filter, k=search_top
                     )
-                    
-                    logger.debug(f"Chunk items for {chunk_type}: {chunk_items[0]}")
-                    
+
                     for doc, score in chunk_items:
                         # Create unique identifier for document
                         source = doc.metadata.get('source')
                         chunk_id = doc.metadata.get('chunk_id')
                         doc_id = f"{source}_{chunk_id}" if source and chunk_id else str(doc.metadata.get('id', id(doc)))
-                        
+
                         # Store document and its score
                         if doc_id not in unique_docs:
                             unique_docs[doc_id] = doc
@@ -384,9 +404,9 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
                                 doc_filter = {
                                     "$and": doc_filter_parts
                                 }
-                                
+
                             try:
-                                fetch_items = self.vectorstore.similarity_search_with_score(
+                                fetch_items = self._similarity_search_with_score(
                                     query, filter=doc_filter, k=1
                                 )
                                 if fetch_items:
@@ -400,7 +420,7 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
         else:
             # Default search behavior (unchanged)
             max_search_results = 30 if search_top * 3 > 30 else search_top * 3
-            vector_items = self.vectorstore.similarity_search_with_score(
+            vector_items = self._similarity_search_with_score(
                 query, filter=filter, k=max_search_results
             )
             
@@ -418,7 +438,7 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
         doc_map = OrderedDict(
             sorted(doc_map.items(), key=lambda x: x[1][1], reverse=True)
         )
-        
+
         # Process full-text search if configured
         if full_text_search and full_text_search.get('enabled') and full_text_search.get('fields'):
             language = full_text_search.get('language', 'english')
@@ -431,7 +451,7 @@ class VectorStoreWrapperBase(BaseToolApiWrapper):
                 for field_name in full_text_search.get('fields', []):
                     try:
                         text_results = self.pg_helper.full_text_search(field_name, query)
-                        
+
                         # Combine text search results with vector results
                         for result in text_results:
                             doc_id = result['id']

{alita_sdk-0.3.371 → alita_sdk-0.3.373}/alita_sdk/tools/base_indexer_toolkit.py

@@ -7,8 +7,10 @@ from typing import Any, Optional, List, Dict, Generator
 from langchain_core.documents import Document
 from pydantic import create_model, Field, SecretStr
 
+from .utils import make_json_serializable
 from .utils.content_parser import file_extension_by_chunker, process_document_by_type
 from .vector_adapters.VectorStoreAdapter import VectorStoreAdapterFactory
+from ..runtime.langchain.document_loaders.constants import loaders_allowed_to_override
 from ..runtime.tools.vectorstore_base import VectorStoreWrapperBase
 from ..runtime.utils.utils import IndexerKeywords
 
@@ -93,7 +95,7 @@ BaseIndexDataParams = create_model(
                        description="Optional flag to enforce clean existing index before indexing new data")),
     progress_step=(Optional[int], Field(default=10, ge=0, le=100,
                          description="Optional step size for progress reporting during indexing")),
-    chunking_config=(Optional[dict], Field(description="Chunking tool configuration", default_factory=dict)),
+    chunking_config=(Optional[dict], Field(description="Chunking tool configuration", default=loaders_allowed_to_override)),
 )
 
 
@@ -352,6 +354,9 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
             filter.update({"collection": {
                 "$eq": collection_suffix.strip()
             }})
+
+        if filter:
+            # Exclude index meta documents from search results
             filter = {
                 "$and": [
                     filter,
@@ -361,6 +366,11 @@ class BaseIndexerToolkit(VectorStoreWrapperBase):
                     ]},
                 ]
             }
+        else:
+            filter = {"$or": [
+                {"type": {"$exists": False}},
+                {"type": {"$ne": IndexerKeywords.INDEX_META_TYPE.value}}
+            ]}
         return filter
 
     def search_index(self,

{alita_sdk-0.3.371 → alita_sdk-0.3.373}/alita_sdk/tools/utils/__init__.py

@@ -97,3 +97,20 @@ def check_connection_response(check_fun):
         else:
             return f"Service Unreachable: return code {response.status_code}"
     return _wrapper
+
+
+def make_json_serializable(obj):
+    if isinstance(obj, BaseModel):
+        return obj.model_dump()
+    if isinstance(obj, dict):
+        return {k: make_json_serializable(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [make_json_serializable(i) for i in obj]
+    if isinstance(obj, bool):
+        return bool(obj)
+    if isinstance(obj, (str, int, float)) or obj is None:
+        return obj
+    # Fallback: handle objects that look like booleans but were not caught above
+    if str(obj) in ("True", "False"):
+        return str(obj) == "True"
+    return str(obj)

{alita_sdk-0.3.371 → alita_sdk-0.3.373}/alita_sdk/tools/utils/content_parser.py

@@ -262,18 +262,18 @@ def process_content_by_type(content, filename: str, llm=None, chunking_config=No
                 loader_kwargs = loader_config['kwargs']
                 # Determine which loader configuration keys are allowed to be overridden by user input.
                 # If 'allowed_to_override' is specified in the loader configuration, use it; otherwise, allow all keys in loader_kwargs.
-                allowed_to_override = loader_config.get('allowed_to_override', list(loader_kwargs.keys()))
+                allowed_to_override = loader_config.get('allowed_to_override', loader_kwargs)
                 # If a chunking_config is provided and contains custom configuration for the current file extension,
-                # update loader_kwargs with user-supplied values, but only for keys explicitly permitted in allowed_to_override.
+                # update loader_kwargs with user-supplied values, but only for keys explicitly permitted in allowed_to_override and if value differs from default.
                 # This ensures that only safe and intended parameters can be customized, preventing accidental or unauthorized changes
                 # to critical loader settings.
                 if chunking_config and (users_config_for_extension := chunking_config.get(extension, {})):
-                    for key in set(users_config_for_extension.keys()) & set(allowed_to_override):
-                        loader_kwargs[key] = users_config_for_extension[key]
-                if LoaderProperties.LLM.value in loader_kwargs:
-                    loader_kwargs[LoaderProperties.LLM.value] = llm
-                if LoaderProperties.PROMPT_DEFAULT.value in loader_kwargs:
-                    loader_kwargs.pop(LoaderProperties.PROMPT_DEFAULT.value)
+                    for key in set(users_config_for_extension.keys()) & set(allowed_to_override.keys()):
+                        if users_config_for_extension[key] != allowed_to_override[key]:
+                            loader_kwargs[key] = users_config_for_extension[key]
+                if LoaderProperties.LLM.value in loader_kwargs and loader_kwargs.pop(LoaderProperties.LLM.value):
+                    loader_kwargs['llm'] = llm
+                if LoaderProperties.PROMPT_DEFAULT.value in loader_kwargs and loader_kwargs.pop(LoaderProperties.PROMPT_DEFAULT.value):
                     loader_kwargs[LoaderProperties.PROMPT.value] = image_processing_prompt
                 loader = loader_cls(file_path=temp_file_path, **loader_kwargs)
                 yield from loader.load()

{alita_sdk-0.3.371 → alita_sdk-0.3.373}/alita_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.371
+Version: 0.3.373
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedj27@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.371 → alita_sdk-0.3.373}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "alita_sdk"
-version = "0.3.371"
+version = "0.3.373"
 description = "SDK for building langchain agents using resources from Alita"
 readme = "README.md"
 requires-python = ">=3.10"