alita-sdk 0.3.345__tar.gz → 0.3.346__tar.gz

{alita_sdk-0.3.345 → alita_sdk-0.3.346}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.345
+Version: 0.3.346
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedj27@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.345 → alita_sdk-0.3.346}/alita_sdk/runtime/langchain/document_loaders/AlitaPDFLoader.py

@@ -1,6 +1,6 @@
 import pymupdf
 import fitz
-from langchain_community.document_loaders import PyPDFLoader
+from langchain_community.document_loaders import PyPDFium2Loader
 
 from .ImageParser import ImageParser
 from .utils import perform_llm_prediction_for_image_bytes, create_temp_file
@@ -23,6 +23,7 @@ class AlitaPDFLoader:
         self.headers = kwargs.get('headers', None)
         self.extraction_mode = kwargs.get('extraction_mode', "plain")
         self.extraction_kwargs = kwargs.get('extraction_kwargs', None)
+        self.images_parser=ImageParser(llm=self.llm, prompt=self.prompt)
 
     def get_content(self):
         if hasattr(self, 'file_path'):
@@ -119,13 +120,13 @@ class AlitaPDFLoader:
             return self._load_docs()
 
     def _load_docs(self):
-        docs = PyPDFLoader(file_path=self.file_path,
-                        password=self.password,
-                        headers=self.headers,
-                        extract_images=self.extract_images,
-                        extraction_mode=self.extraction_mode,
-                        images_parser=ImageParser(llm=self.llm, prompt=self.prompt),
-                        extraction_kwargs=self.extraction_kwargs).load()
+        docs = PyPDFium2Loader(
+                file_path = self.file_path,
+                password=self.password,
+                headers=self.headers,
+                extract_images = self.extract_images,
+                images_parser = ImageParser(llm=self.llm, prompt=self.prompt),
+            ).load()
         for doc in docs:
             doc.metadata['chunk_id'] = doc.metadata['page']
         return docs

alita_sdk-0.3.346/alita_sdk/runtime/langchain/document_loaders/ImageParser.py

@@ -0,0 +1,30 @@
+from typing import Iterator
+
+from langchain_community.document_loaders.parsers.images import BaseImageBlobParser
+from langchain_core.documents import Document
+from langchain_core.documents.base import Blob
+
+from alita_sdk.runtime.langchain.document_loaders.AlitaImageLoader import AlitaImageLoader
+
+class ImageParser(BaseImageBlobParser):
+
+    def __init__(self, **kwargs):
+        self.llm = kwargs.get('llm')
+        self.prompt = kwargs.get('prompt')
+
+    def lazy_parse(self, blob: Blob) -> Iterator[Document]:
+        try:
+            yield from super().lazy_parse(blob)
+        except Exception:
+            yield Document(page_content="[Image: Unknown]")
+
+    def _analyze_image(self, img) -> str:
+        from io import BytesIO
+
+        byte_stream = BytesIO()
+        img.save(byte_stream, format='PNG')
+        image_bytes = byte_stream.getvalue()
+        try:
+            return AlitaImageLoader(file_content=image_bytes, file_name="image.png", prompt=self.prompt, llm=self.llm).get_content()
+        except Exception:
+            return "Image: unknown"

{alita_sdk-0.3.345 → alita_sdk-0.3.346}/alita_sdk/runtime/tools/artifact.py

@@ -1,7 +1,7 @@
 import hashlib
-import json
 import logging
-from typing import Any, Optional, Generator
+import re
+from typing import Any, Optional, Generator, List
 
 from langchain_core.documents import Document
 from langchain_core.tools import ToolException
@@ -59,18 +59,53 @@ class ArtifactWrapper(NonCodeIndexerToolkit):
     def create_new_bucket(self, bucket_name: str, expiration_measure = "weeks", expiration_value = 1):
         return self.artifact.client.create_bucket(bucket_name, expiration_measure, expiration_value)
 
+    def _index_tool_params(self):
+        return {
+            'include_extensions': (Optional[List[str]], Field(
+                description="List of file extensions to include when processing: i.e. ['*.png', '*.jpg']. "
+                            "If empty, all files will be processed (except skip_extensions).",
+                default=[])),
+            'skip_extensions': (Optional[List[str]], Field(
+                description="List of file extensions to skip when processing: i.e. ['*.png', '*.jpg']",
+                default=[])),
+        }
+
     def _base_loader(self, **kwargs) -> Generator[Document, None, None]:
         try:
-            all_files = self.list_files(self.bucket, False)
+            all_files = self.list_files(self.bucket, False)['rows']
         except Exception as e:
             raise ToolException(f"Unable to extract files: {e}")
 
-        for file in all_files['rows']:
+        include_extensions = kwargs.get('include_extensions', [])
+        skip_extensions = kwargs.get('skip_extensions', [])
+        self._log_tool_event(message=f"Files filtering started. Include extensions: {include_extensions}. "
+                                     f"Skip extensions: {skip_extensions}", tool_name="loader")
+        # show the progress of filtering
+        total_files = len(all_files) if isinstance(all_files, list) else 0
+        filtered_files_count = 0
+        for file in all_files:
+            filtered_files_count += 1
+            if filtered_files_count % 10 == 0 or filtered_files_count == total_files:
+                self._log_tool_event(message=f"Files filtering progress: {filtered_files_count}/{total_files}",
+                                     tool_name="loader")
+            file_name = file['name']
+
+            # Check if file should be skipped based on skip_extensions
+            if any(re.match(pattern.replace('*', '.*') + '$', file_name, re.IGNORECASE)
+                   for pattern in skip_extensions):
+                continue
+
+            # Check if file should be included based on include_extensions
+            # If include_extensions is empty, process all files (that weren't skipped)
+            if include_extensions and not (any(re.match(pattern.replace('*', '.*') + '$', file_name, re.IGNORECASE)
+                                               for pattern in include_extensions)):
+                continue
+
             metadata = {
                 ("updated_on" if k == "modified" else k): str(v)
                 for k, v in file.items()
             }
-            metadata['id'] = self.get_hash_from_bucket_and_file_name(self.bucket, file['name'])
+            metadata['id'] = self.get_hash_from_bucket_and_file_name(self.bucket, file_name)
             yield Document(page_content="", metadata=metadata)
 
     def get_hash_from_bucket_and_file_name(self, bucket, file_name):

{alita_sdk-0.3.345 → alita_sdk-0.3.346}/alita_sdk/tools/github/github_client.py

@@ -105,7 +105,7 @@ class GitHubClient(BaseModel):
                     self._github_repo_instance = None
             except Exception as e:
                 # Only raise when accessed, not during initialization
-                return ToolException(e)
+                raise ToolException(e)
         return self._github_repo_instance
 
     @model_validator(mode='before')

{alita_sdk-0.3.345 → alita_sdk-0.3.346}/alita_sdk.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: alita_sdk
-Version: 0.3.345
+Version: 0.3.346
 Summary: SDK for building langchain agents using resources from Alita
 Author-email: Artem Rozumenko <artyom.rozumenko@gmail.com>, Mikalai Biazruchka <mikalai_biazruchka@epam.com>, Roman Mitusov <roman_mitusov@epam.com>, Ivan Krakhmaliuk <lifedj27@gmail.com>, Artem Dubrovskiy <ad13box@gmail.com>
 License-Expression: Apache-2.0

{alita_sdk-0.3.345 → alita_sdk-0.3.346}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "alita_sdk"
-version = "0.3.345"
+version = "0.3.346"
 description = "SDK for building langchain agents using resources from Alita"
 readme = "README.md"
 requires-python = ">=3.10"

alita_sdk-0.3.345/alita_sdk/runtime/langchain/document_loaders/ImageParser.py

@@ -1,17 +0,0 @@
-from langchain_community.document_loaders.parsers.images import BaseImageBlobParser
-
-from alita_sdk.runtime.langchain.document_loaders.AlitaImageLoader import AlitaImageLoader
-
-class ImageParser(BaseImageBlobParser):
-
-    def __init__(self, **kwargs):
-        self.llm = kwargs.get('llm')
-        self.prompt = kwargs.get('prompt')
-
-    def _analyze_image(self, img) -> str:
-        from io import BytesIO
-
-        byte_stream = BytesIO()
-        img.save(byte_stream, format='PNG')
-        image_bytes = byte_stream.getvalue()
-        return AlitaImageLoader(file_content=image_bytes, file_name="image.png", prompt=self.prompt, llm=self.llm).get_content()