alita-sdk 0.3.257__py3-none-any.whl → 0.3.562__py3-none-any.whl

alita_sdk/runtime/langchain/document_loaders/constants.py

@@ -13,162 +13,310 @@
 # limitations under the License.
 
 from langchain_community.document_loaders import (
-    UnstructuredMarkdownLoader,
     AirbyteJSONLoader, UnstructuredHTMLLoader,
-    PythonLoader)
+    UnstructuredXMLLoader)
 
 from .AlitaCSVLoader import AlitaCSVLoader
 from .AlitaDocxMammothLoader import AlitaDocxMammothLoader
 from .AlitaExcelLoader import AlitaExcelLoader
 from .AlitaImageLoader import AlitaImageLoader
 from .AlitaJSONLoader import AlitaJSONLoader
+from .AlitaJSONLinesLoader import AlitaJSONLinesLoader
 from .AlitaPDFLoader import AlitaPDFLoader
 from .AlitaPowerPointLoader import AlitaPowerPointLoader
 from .AlitaTextLoader import AlitaTextLoader
+from .AlitaMarkdownLoader import AlitaMarkdownLoader
+from .AlitaPythonLoader import AlitaPythonLoader
+from enum import Enum
+from alita_sdk.runtime.langchain.constants import LOADER_MAX_TOKENS_DEFAULT
 
-loaders_map = {
+
+class LoaderProperties(Enum):
+    LLM = 'use_llm'
+    PROMPT_DEFAULT = 'use_default_prompt'
+    PROMPT = 'prompt'
+
+DEFAULT_ALLOWED_BASE = {'max_tokens': LOADER_MAX_TOKENS_DEFAULT}
+
+DEFAULT_ALLOWED_WITH_LLM = {
+    **DEFAULT_ALLOWED_BASE,
+    LoaderProperties.LLM.value: False,
+    LoaderProperties.PROMPT_DEFAULT.value: False,
+    LoaderProperties.PROMPT.value: "",
+}
+
+DEFAULT_ALLOWED_EXCEL = {**DEFAULT_ALLOWED_WITH_LLM, 'add_header_to_chunks': False, 'header_row_number': 1, 'max_tokens': -1, 'sheet_name': ''}
+
+# Image file loaders mapping - directly supported by LLM with image_url
+image_loaders_map = {
     '.png': {
         'class': AlitaImageLoader,
+        'mime_type': 'image/png',
         'is_multimodal_processing': True,
-        'kwargs': {}
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM,
     },
     '.jpg': {
         'class': AlitaImageLoader,
+        'mime_type': 'image/jpeg',
         'is_multimodal_processing': True,
-        'kwargs': {}
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.jpeg': {
         'class': AlitaImageLoader,
+        'mime_type': 'image/jpeg',
         'is_multimodal_processing': True,
-        'kwargs': {}
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.gif': {
         'class': AlitaImageLoader,
+        'mime_type': 'image/gif',
         'is_multimodal_processing': True,
-        'kwargs': {}
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
+    '.webp': {
+        'class': AlitaImageLoader,
+        'mime_type': 'image/webp',
+        'is_multimodal_processing': True,
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
+    }
+}
+
+# Image file loaders mapping - require conversion before sending to LLM
+image_loaders_map_converted = {
     '.bmp': {
         'class': AlitaImageLoader,
+        'mime_type': 'image/bmp',
         'is_multimodal_processing': True,
-        'kwargs': {}
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.svg': {
         'class': AlitaImageLoader,
+        'mime_type': 'image/svg+xml',
         'is_multimodal_processing': True,
-        'kwargs': {}
-    },
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
+    }
+}
+
+# Document file loaders mapping
+document_loaders_map = {
     '.txt': {
         'class': AlitaTextLoader,
+        'mime_type': 'text/plain',
         'is_multimodal_processing': False,
         'kwargs': {
             'autodetect_encoding': True
-        }
+        },
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.yml': {
         'class': AlitaTextLoader,
+        'mime_type': 'application/yaml',
         'is_multimodal_processing': False,
         'kwargs': {
             'autodetect_encoding': True
-        }
+        },
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.yaml': {
         'class': AlitaTextLoader,
+        'mime_type': 'application/yaml',
         'is_multimodal_processing': False,
         'kwargs': {
             'autodetect_encoding': True
-        }
+        },
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.groovy': {
         'class': AlitaTextLoader,
+        'mime_type': 'text/x-groovy',
         'is_multimodal_processing': False,
         'kwargs': {
             'autodetect_encoding': True
-        }
+        },
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.md': {
-        'class': UnstructuredMarkdownLoader,
+        'class': AlitaMarkdownLoader,
+        'mime_type': 'text/markdown',
         'is_multimodal_processing': False,
-        'kwargs': {}
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.csv': {
         'class': AlitaCSVLoader,
+        'mime_type': 'text/csv',
         'is_multimodal_processing': False,
         'kwargs': {
             'encoding': 'utf-8',
-            'raw_content': False,
+            'raw_content': True,
             'cleanse': False
-        }
+        },
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.xlsx': {
         'class': AlitaExcelLoader,
+        'mime_type': ('application/vnd.openxmlformats-officedocument.'
+                      'spreadsheetml.sheet'),
         'is_multimodal_processing': False,
         'kwargs': {
-            'raw_content': False,
-            'cleanse': False
-        }
+            'add_header_to_chunks': False,
+            'header_row_number': 1,
+            'max_tokens': -1,
+            'sheet_name': ''
+        },
+        'allowed_to_override': DEFAULT_ALLOWED_EXCEL
     },
     '.xls': {
         'class': AlitaExcelLoader,
+        'mime_type': 'application/vnd.ms-excel',
         'is_multimodal_processing': False,
         'kwargs': {
-            'raw_content': False,
+            'excel_by_sheets': True,
+            'raw_content': True,
             'cleanse': False
-        }
+        },
+        'allowed_to_override': DEFAULT_ALLOWED_EXCEL
     },
     '.pdf': {
         'class': AlitaPDFLoader,
+        'mime_type': 'application/pdf',
         'is_multimodal_processing': False,
-        'kwargs': {}
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.docx': {
         'class': AlitaDocxMammothLoader,
+        'mime_type': ('application/vnd.openxmlformats-officedocument.'
+                      'wordprocessingml.document'),
         'is_multimodal_processing': True,
         'kwargs': {
             'extract_images': True
-        }
-    },
-    '.doc': {
-        'class': AlitaTextLoader,
-        'is_multimodal_processing': True,
-        'kwargs': {}
+        },
+        'allowed_to_override': {**DEFAULT_ALLOWED_WITH_LLM, 'mode': 'paged'}
     },
     '.json': {
         'class': AlitaJSONLoader,
+        'mime_type': 'application/json',
         'is_multimodal_processing': False,
-        'kwargs': {}
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.jsonl': {
-        'class': AirbyteJSONLoader,
+        'class': AlitaJSONLinesLoader,
+        'mime_type': 'application/jsonl',
         'is_multimodal_processing': False,
-        'kwargs': {}
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_BASE
     },
     '.htm': {
         'class': UnstructuredHTMLLoader,
+        'mime_type': 'text/html',
         'is_multimodal_processing': False,
-        'kwargs': {}
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.html': {
         'class': UnstructuredHTMLLoader,
+        'mime_type': 'text/html',
+        'is_multimodal_processing': False,
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
+    },
+    '.xml': {
+        'class': UnstructuredXMLLoader,
+        'mime_type': 'text/xml',
         'is_multimodal_processing': False,
-        'kwargs': {}
+        'kwargs': {},
+        'allowed_to_override': DEFAULT_ALLOWED_WITH_LLM
     },
     '.ppt': {
         'class': AlitaPowerPointLoader,
+        'mime_type': 'application/vnd.ms-powerpoint',
         'is_multimodal_processing': False,
         'kwargs': {
             'mode': 'paged'
-        }
+        },
+        'allowed_to_override': {**DEFAULT_ALLOWED_WITH_LLM, 'mode': 'paged'}
     },
     '.pptx': {
         'class': AlitaPowerPointLoader,
+        'mime_type': ('application/vnd.openxmlformats-officedocument.'
+                      'presentationml.presentation'),
         'is_multimodal_processing': False,
         'kwargs': {
             'mode': 'paged'
+        },
+        'allowed_to_override': {
+            **DEFAULT_ALLOWED_WITH_LLM,
+            'mode': 'paged',
+            'pages_per_chunk': 5,
+            'extract_images': False,
         }
     },
-    '.py': {
-        'class': PythonLoader,
-        'is_multimodal_processing': False,
-        'kwargs': {}
-    }
+    # '.py': {
+    #     'class': AlitaPythonLoader,
+    #     'mime_type': 'text/x-python',
+    #     'is_multimodal_processing': False,
+    #     'kwargs': {},
+    #     'allowed_to_override': DEFAULT_ALLOWED_BASE
+    # }
+}
+
+code_extensions = [
+    '.py',  # Python
+    '.js',  # JavaScript
+    '.ts',  # TypeScript
+    '.java',  # Java
+    '.cpp',  # C++
+    '.c',  # C
+    '.cs',  # C#
+    '.rb',  # Ruby
+    '.go',  # Go
+    '.php',  # PHP
+    '.swift',  # Swift
+    '.kt',  # Kotlin
+    '.rs',  # Rust
+    '.m',  # Objective-C
+    '.scala',  # Scala
+    '.pl',  # Perl
+    '.sh',  # Shell
+    '.bat',  # Batch
+    '.lua',  # Lua
+    '.r',  # R
+    '.pas',  # Pascal
+    '.asm',  # Assembly
+    '.dart',  # Dart
+    '.groovy',  # Groovy
+    '.sql',  # SQL
+]
+
+default_loader_config = {
+    'class': AlitaTextLoader,
+    'mime_type': 'text/plain',
+    'is_multimodal_processing': False,
+    'kwargs': {},
+    'allowed_to_override': DEFAULT_ALLOWED_BASE
+}
+
+code_loaders_map = {ext: default_loader_config for ext in code_extensions}
+
+# Combined mapping for backward compatibility
+loaders_map = {
+    **image_loaders_map,
+    **image_loaders_map_converted,
+    **document_loaders_map,
+    **code_loaders_map
+}
+
+loaders_allowed_to_override = {
+    extension: config.get('allowed_to_override')
+    for extension, config in loaders_map.items()
+    if 'allowed_to_override' in config
 }

alita_sdk/runtime/langchain/interfaces/llm_processor.py

@@ -173,13 +173,15 @@ def get_vectorstore(vectorstore_type, vectorstore_params, embedding_func=None):
     #
     raise RuntimeError(f"Unknown VectorStore type: {vectorstore_type}")
 
-def add_documents(vectorstore, documents):
+def add_documents(vectorstore, documents, ids = None) -> list[str]:
     """ Add documents to vectorstore """
     if vectorstore is None:
         return None
     texts = []
     metadata = []
     for document in documents:
+        if not document.page_content:
+            continue
         texts.append(document.page_content)
         for key in document.metadata:
             if isinstance(document.metadata[key], list):
@@ -187,7 +189,7 @@ def add_documents(vectorstore, documents):
             if isinstance(document.metadata[key], dict):
                 document.metadata[key] = dumps(document.metadata[key])
         metadata.append(document.metadata)
-    vectorstore.add_texts(texts, metadatas=metadata)
+    return vectorstore.add_texts(texts, metadatas=metadata, ids=ids)
 
 
 def generateResponse(