alita-sdk 0.3.351__py3-none-any.whl → 0.3.499__py3-none-any.whl

alita_sdk/runtime/langchain/document_loaders/AlitaTableLoader.py

@@ -63,7 +63,7 @@ class AlitaTableLoader(BaseLoader):
                 "source": f'{self.file_path}:{idx+1}',
                 "table_source": self.file_path,
             }
-            if len(docs) == 0:
+            if len(docs) == 0 and not self.raw_content:
                 header_metadata = metadata.copy()
                 header_metadata["header"] = "true"
                 header = "\t".join([str(value) for value in row.keys()])

alita_sdk/runtime/langchain/document_loaders/constants.py

@@ -27,81 +27,79 @@ from .AlitaTextLoader import AlitaTextLoader
 from .AlitaMarkdownLoader import AlitaMarkdownLoader
 from .AlitaPythonLoader import AlitaPythonLoader
 from enum import Enum
+from alita_sdk.runtime.langchain.constants import LOADER_MAX_TOKENS_DEFAULT
 
 
 class LoaderProperties(Enum):
-    LLM = 'llm'
+    LLM = 'use_llm'
+    PROMPT_DEFAULT = 'use_default_prompt'
     PROMPT = 'prompt'
-    PROMPT_DEFAULT = 'prompt_default'
 
+DEFAULT_ALLOWED_BASE = {'max_tokens': LOADER_MAX_TOKENS_DEFAULT}
 
-# Image file loaders mapping
+DEFAULT_ALLOWED_WITH_LLM = {
+    **DEFAULT_ALLOWED_BASE,
+    LoaderProperties.LLM.value: False,
+    LoaderProperties.PROMPT_DEFAULT.value: False,
+    LoaderProperties.PROMPT.value: "",
+}
+
+DEFAULT_ALLOWED_EXCEL = {**DEFAULT_ALLOWED_WITH_LLM, 'add_header_to_chunks': False, 'header_row_number': 1, 'max_tokens': -1, 'sheet_name': ''}
+
+# Image file loaders mapping - directly supported by LLM with image_url
 image_loaders_map = {
     '.png': {
         'class': AlitaImageLoader,
         'mime_type': 'image/png',
         'is_multimodal_processing': True,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ],
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM,
     },
     '.jpg': {
         'class': AlitaImageLoader,
         'mime_type': 'image/jpeg',
         'is_multimodal_processing': True,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.jpeg': {
         'class': AlitaImageLoader,
         'mime_type': 'image/jpeg',
         'is_multimodal_processing': True,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.gif': {
         'class': AlitaImageLoader,
         'mime_type': 'image/gif',
         'is_multimodal_processing': True,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
+    '.webp': {
+        'class': AlitaImageLoader,
+        'mime_type': 'image/webp',
+        'is_multimodal_processing': True,
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
+    }
+}
+
+# Image file loaders mapping - require conversion before sending to LLM
+image_loaders_map_converted = {
     '.bmp': {
         'class': AlitaImageLoader,
         'mime_type': 'image/bmp',
         'is_multimodal_processing': True,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.svg': {
         'class': AlitaImageLoader,
         'mime_type': 'image/svg+xml',
         'is_multimodal_processing': True,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     }
 }
 
@@ -114,25 +112,25 @@ document_loaders_map = {
         'kwargs': {
             'autodetect_encoding': True
         },
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.yml': {
         'class': AlitaTextLoader,
-        'mime_type': 'application/x-yaml',
+        'mime_type': 'application/yaml',
         'is_multimodal_processing': False,
         'kwargs': {
             'autodetect_encoding': True
         },
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.yaml': {
         'class': AlitaTextLoader,
-        'mime_type': 'application/x-yaml',
+        'mime_type': 'application/yaml',
         'is_multimodal_processing': False,
         'kwargs': {
             'autodetect_encoding': True
         },
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.groovy': {
         'class': AlitaTextLoader,
@@ -141,14 +139,14 @@ document_loaders_map = {
         'kwargs': {
             'autodetect_encoding': True
         },
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.md': {
         'class': AlitaMarkdownLoader,
         'mime_type': 'text/markdown',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.csv': {
         'class': AlitaCSVLoader,
@@ -159,7 +157,7 @@ document_loaders_map = {
             'raw_content': True,
             'cleanse': False
         },
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.xlsx': {
         'class': AlitaExcelLoader,
@@ -167,15 +165,12 @@ document_loaders_map = {
                       'spreadsheetml.sheet'),
         'is_multimodal_processing': False,
         'kwargs': {
-            'excel_by_sheets': True,
-            'raw_content': True,
-            'cleanse': False
+            'add_header_to_chunks': False,
+            'header_row_number': 1,
+            'max_tokens': -1,
+            'sheet_name': ''
         },
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_EXCEL
     },
     '.xls': {
         'class': AlitaExcelLoader,
@@ -186,22 +181,14 @@ document_loaders_map = {
             'raw_content': True,
             'cleanse': False
         },
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_EXCEL
     },
     '.pdf': {
         'class': AlitaPDFLoader,
         'mime_type': 'application/pdf',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.docx': {
         'class': AlitaDocxMammothLoader,
@@ -211,58 +198,42 @@ document_loaders_map = {
         'kwargs': {
             'extract_images': True
         },
-        'allowed_to_override': [
-            'max_tokens', 'mode', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': {**DEFAULT_ALLOWED_WITH_LLM, 'mode': 'paged'}
     },
     '.json': {
         'class': AlitaJSONLoader,
         'mime_type': 'application/json',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.jsonl': {
         'class': AirbyteJSONLoader,
         'mime_type': 'application/jsonl',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': ['max_tokens']
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.htm': {
         'class': UnstructuredHTMLLoader,
         'mime_type': 'text/html',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.html': {
         'class': UnstructuredHTMLLoader,
         'mime_type': 'text/html',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.xml': {
         'class': UnstructuredXMLLoader,
         'mime_type': 'text/xml',
         'is_multimodal_processing': False,
         'kwargs': {},
-        'allowed_to_override': [
-            'max_tokens', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.ppt': {
         'class': AlitaPowerPointLoader,
@@ -271,11 +242,7 @@ document_loaders_map = {
         'kwargs': {
             'mode': 'paged'
         },
-        'allowed_to_override': [
-            'max_tokens', 'mode', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': {**DEFAULT_ALLOWED_WITH_LLM, 'mode': 'paged'}
     },
     '.pptx': {
         'class': AlitaPowerPointLoader,
@@ -285,20 +252,70 @@ document_loaders_map = {
         'kwargs': {
             'mode': 'paged'
         },
-        'allowed_to_override': [
-            'max_tokens', 'mode', LoaderProperties.LLM.value,
-            LoaderProperties.PROMPT.value,
-            LoaderProperties.PROMPT_DEFAULT.value
-        ]
+        'allowed_to_override': {
+            **DEFAULT_ALLOWED_WITH_LLM,
+            'mode': 'paged',
+            'pages_per_chunk': 5,
+            'extract_images': False,
+        }
     },
-    '.py': {
-        'class': AlitaPythonLoader,
-        'mime_type': 'text/x-python',
-        'is_multimodal_processing': False,
-        'kwargs': {},
-        'allowed_to_override': ['max_tokens']
-    }
+    # '.py': {
+    #     'class': AlitaPythonLoader,
+    #     'mime_type': 'text/x-python',
+    #     'is_multimodal_processing': False,
+    #     'kwargs': {},
+    #     'allowed_to_override': DEFAULT_ALLOWED_BASE
+    # }
 }
 
+code_extensions = [
+    '.py',  # Python
+    '.js',  # JavaScript
+    '.ts',  # TypeScript
+    '.java',  # Java
+    '.cpp',  # C++
+    '.c',  # C
+    '.cs',  # C#
+    '.rb',  # Ruby
+    '.go',  # Go
+    '.php',  # PHP
+    '.swift',  # Swift
+    '.kt',  # Kotlin
+    '.rs',  # Rust
+    '.m',  # Objective-C
+    '.scala',  # Scala
+    '.pl',  # Perl
+    '.sh',  # Shell
+    '.bat',  # Batch
+    '.lua',  # Lua
+    '.r',  # R
+    '.pas',  # Pascal
+    '.asm',  # Assembly
+    '.dart',  # Dart
+    '.groovy',  # Groovy
+    '.sql',  # SQL
+]
+
+default_loader_config = {
+    'class': AlitaTextLoader,
+    'mime_type': 'text/plain',
+    'is_multimodal_processing': False,
+    'kwargs': {},
+    'allowed_to_override': DEFAULT_ALLOWED_BASE
+}
+
+code_loaders_map = {ext: default_loader_config for ext in code_extensions}
+
 # Combined mapping for backward compatibility
-loaders_map = {**image_loaders_map, **document_loaders_map}
+loaders_map = {
+    **image_loaders_map,
+    **image_loaders_map_converted,
+    **document_loaders_map,
+    **code_loaders_map
+}
+
+loaders_allowed_to_override = {
+    extension: config.get('allowed_to_override')
+    for extension, config in loaders_map.items()
+    if 'allowed_to_override' in config
+}

alita_sdk/runtime/langchain/interfaces/llm_processor.py

@@ -173,7 +173,7 @@ def get_vectorstore(vectorstore_type, vectorstore_params, embedding_func=None):
     #
     raise RuntimeError(f"Unknown VectorStore type: {vectorstore_type}")
 
-def add_documents(vectorstore, documents):
+def add_documents(vectorstore, documents, ids = None) -> list[str]:
     """ Add documents to vectorstore """
     if vectorstore is None:
         return None
@@ -189,7 +189,7 @@ def add_documents(vectorstore, documents):
             if isinstance(document.metadata[key], dict):
                 document.metadata[key] = dumps(document.metadata[key])
         metadata.append(document.metadata)
-    vectorstore.add_texts(texts, metadatas=metadata)
+    return vectorstore.add_texts(texts, metadatas=metadata, ids=ids)
 
 
 def generateResponse(