unstructured-ingest 1.0.11__tar.gz → 1.0.13__tar.gz

{unstructured_ingest-1.0.11 → unstructured_ingest-1.0.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.0.11
+Version: 1.0.13
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

unstructured_ingest-1.0.13/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.0.13"  # pragma: no cover

{unstructured_ingest-1.0.11 → unstructured_ingest-1.0.13}/unstructured_ingest/processes/connectors/google_drive.py

@@ -1,4 +1,3 @@
-import io
 import json
 from contextlib import contextmanager
 from dataclasses import dataclass, field
@@ -16,7 +15,6 @@ from unstructured_ingest.data_types.file_data import (
 )
 from unstructured_ingest.error import (
     SourceConnectionError,
-    SourceConnectionNetworkError,
 )
 from unstructured_ingest.interfaces import (
     AccessConfig,
@@ -34,25 +32,34 @@ from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:
     from googleapiclient.discovery import Resource as GoogleAPIResource
-    from googleapiclient.http import MediaIoBaseDownload
 
 CONNECTOR_TYPE = "google_drive"
 
-GOOGLE_DRIVE_EXPORT_TYPES = {
-    "application/vnd.google-apps.document": "application/"
-    "vnd.openxmlformats-officedocument.wordprocessingml.document",
-    "application/vnd.google-apps.spreadsheet": "application/"
-    "vnd.openxmlformats-officedocument.spreadsheetml.sheet",
-    "application/vnd.google-apps.presentation": "application/"
-    "vnd.openxmlformats-officedocument.presentationml.presentation",
-    "application/vnd.google-apps.photo": "image/jpeg",
+
+# Maps Google-native Drive MIME types → export MIME types
+GOOGLE_EXPORT_MIME_MAP = {
+    "application/vnd.google-apps.document": \
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "application/vnd.google-apps.spreadsheet": \
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "application/vnd.google-apps.presentation": \
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+}
+
+# Maps export MIME types → file extensions
+EXPORT_EXTENSION_MAP = {
+    "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
+    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",
+    "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
+    "application/pdf": ".pdf",
+    "text/html": ".html",
 }
 
 
 class GoogleDriveAccessConfig(AccessConfig):
-    service_account_key: Optional[Annotated[dict, BeforeValidator(conform_string_to_dict)]] = Field(
-        default=None, description="Credentials values to use for authentication"
-    )
+    service_account_key: Optional[
+        Annotated[dict, BeforeValidator(conform_string_to_dict)]
+    ] = Field(default=None, description="Credentials values to use for authentication")
     service_account_key_path: Optional[Path] = Field(
         default=None,
         description="File path to credentials values to use for authentication",
@@ -171,10 +178,14 @@ class GoogleDriveIndexer(Indexer):
                     Please enable it in the Google Cloud Console."
                 )
             else:
-                raise SourceConnectionError("Google drive API unreachable for an unknown reason!")
+                raise SourceConnectionError(
+                    "Google drive API unreachable for an unknown reason!"
+                )
 
     @staticmethod
-    def count_files_recursively(files_client, folder_id: str, extensions: list[str] = None) -> int:
+    def count_files_recursively(
+        files_client, folder_id: str, extensions: list[str] = None
+    ) -> int:
         """
         Count non-folder files recursively under the given folder.
         If `extensions` is provided, only count files
@@ -255,7 +266,9 @@ class GoogleDriveIndexer(Indexer):
                         #     that the service account has proper permissions."
                         # )
                     else:
-                        logger.info(f"Found {file_count} files recursively in the folder.")
+                        logger.info(
+                            f"Found {file_count} files recursively in the folder."
+                        )
                 else:
                     # Non-recursive: check for at least one immediate non-folder child.
                     response = client.list(
@@ -306,7 +319,9 @@ class GoogleDriveIndexer(Indexer):
         date_modified_str = root_info.pop("modifiedTime", None)
         parent_path = root_info.pop("parent_path", None)
         parent_root_path = root_info.pop("parent_root_path", None)
-        date_modified_dt = parser.parse(date_modified_str) if date_modified_str else None
+        date_modified_dt = (
+            parser.parse(date_modified_str) if date_modified_str else None
+        )
         if (
             parent_path
             and isinstance(parent_path, str)
@@ -406,7 +421,9 @@ class GoogleDriveIndexer(Indexer):
     ) -> list[FileData]:
         root_info = self.get_root_info(files_client=files_client, object_id=object_id)
         if not self.is_dir(root_info):
-            root_info["permissions"] = self.extract_permissions(root_info.get("permissions"))
+            root_info["permissions"] = self.extract_permissions(
+                root_info.get("permissions")
+            )
             data = [self.map_file_data(root_info)]
         else:
             file_contents = self.get_paginated_results(
@@ -427,7 +444,7 @@ class GoogleDriveIndexer(Indexer):
     def extract_permissions(self, permissions: Optional[list[dict]]) -> list[dict]:
         if not permissions:
             logger.debug("no permissions found")
-            return {}
+            return [{}]
 
         # https://developers.google.com/workspace/drive/api/guides/ref-roles
         role_mapping = {
@@ -478,61 +495,136 @@ class GoogleDriveDownloaderConfig(DownloaderConfig):
 
 @dataclass
 class GoogleDriveDownloader(Downloader):
+    """
+    Downloads files from Google Drive using authenticated direct HTTP requests
+    via `exportLinks` (for Google-native files) and `webContentLink` (for binary files).
+
+    These links emulate the behavior of Google Drive's "File > Download as..." options
+    in the UI and bypass the size limitations of `files.export()`.
+
+    Behavior:
+    - Google-native formats are downloaded using `exportLinks` in appropriate MIME formats.
+    - Binary files (non-Google-native) are downloaded using `webContentLink`.
+    - All downloads are performed via `requests.get()` using a valid bearer token.
+    """
+
     connection_config: GoogleDriveConnectionConfig
     download_config: GoogleDriveDownloaderConfig = field(
         default_factory=lambda: GoogleDriveDownloaderConfig()
     )
     connector_type: str = CONNECTOR_TYPE
 
-    @SourceConnectionNetworkError.wrap
-    def _get_content(self, downloader: "MediaIoBaseDownload") -> bool:
-        downloaded = False
-        while downloaded is False:
-            _, downloaded = downloader.next_chunk()
-        return downloaded
+    def _get_download_url_and_ext(
+        self, file_id: str, mime_type: str
+    ) -> tuple[str, str]:
+        """
+        Resolves the appropriate download URL and expected file extension for a Google Drive file.
 
-    def _write_file(self, file_data: FileData, file_contents: io.BytesIO) -> DownloadResponse:
-        download_path = self.get_download_path(file_data=file_data)
-        download_path.parent.mkdir(parents=True, exist_ok=True)
-        logger.debug(f"writing {file_data.source_identifiers.fullpath} to {download_path}")
-        with open(download_path, "wb") as handler:
-            handler.write(file_contents.getbuffer())
-        return self.generate_download_response(file_data=file_data, download_path=download_path)
+        - Google-native files use export MIME types from exportLinks (e.g., .docx, .xlsx).
+        - Binary files use webContentLink (e.g., uploaded PDFs or ZIPs).
 
-    @requires_dependencies(["googleapiclient"], extras="google-drive")
-    def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
-        from googleapiclient.http import MediaIoBaseDownload
+        Returns:
+            Tuple[str, str]: (download URL, file extension or "")
 
-        logger.debug(f"fetching file: {file_data.source_identifiers.fullpath}")
-        record_id = file_data.identifier
-        mime_type = file_data.additional_metadata["mimeType"]
-        if not mime_type:
-            raise TypeError(
-                f"File not supported. Name: {file_data.source_identifiers.filename} "
-                f"ID: {record_id} "
-                f"MimeType: {mime_type}"
-            )
+        Raises:
+            SourceConnectionError: If no valid export or download link is available.
+        """
         with self.connection_config.get_client() as client:
-            if (
-                mime_type.startswith("application/vnd.google-apps")
-                and mime_type in GOOGLE_DRIVE_EXPORT_TYPES
-            ):
-                export_mime = GOOGLE_DRIVE_EXPORT_TYPES.get(
-                    mime_type,  # type: ignore
+            metadata = client.get(
+                fileId=file_id, fields="exportLinks,webContentLink"
+            ).execute()
+
+        export_links = metadata.get("exportLinks", {})
+        web_link = metadata.get("webContentLink")
+
+        if export_mime := GOOGLE_EXPORT_MIME_MAP.get(mime_type):
+            url = export_links.get(export_mime)
+            if not url:
+                raise SourceConnectionError(
+                    f"No export link found for {file_id} as {export_mime}"
                 )
-                request = client.export_media(
-                    fileId=record_id,
-                    mimeType=export_mime,
+            ext = EXPORT_EXTENSION_MAP.get(export_mime, "")
+            return url, ext
+
+        if not web_link:
+            raise SourceConnectionError(
+                f"No webContentLink available for file {file_id}"
+            )
+        return web_link, ""
+
+    @requires_dependencies(["httpx", "google.auth"], extras="google-drive")
+    def _download_url(self, file_data: FileData, url: str, ext: str = "") -> Path:
+        """
+        Streams file content directly to disk using authenticated HTTP request.
+
+        Writes the file to the correct path in the download directory while downloading.
+        Avoids buffering large files in memory.
+
+        Returns:
+            Path to the downloaded file.
+
+        Raises:
+            SourceConnectionError: If the HTTP request fails.
+        """
+        import httpx
+        from google.auth.transport.requests import Request
+        from google.oauth2 import service_account
+
+        access_config = self.connection_config.access_config.get_secret_value()
+        key_data = access_config.get_service_account_key()
+        creds = service_account.Credentials.from_service_account_info(
+            key_data,
+            scopes=["https://www.googleapis.com/auth/drive.readonly"],
+        )
+        creds.refresh(Request())
+
+        headers = {
+            "Authorization": f"Bearer {creds.token}",
+        }
+
+        download_path = self.get_download_path(file_data)
+        if ext:
+            download_path = download_path.with_suffix(ext)
+
+        download_path.parent.mkdir(parents=True, exist_ok=True)
+        logger.debug(f"Streaming file to {download_path}")
+
+        with (
+            httpx.Client(timeout=None, follow_redirects=True) as client,
+            client.stream("GET", url, headers=headers) as response,
+        ):
+            if response.status_code != 200:
+                raise SourceConnectionError(
+                    f"Failed to stream download from {url}: {response.status_code}"
                 )
-            else:
-                request = client.get_media(fileId=record_id)
-
-        file_contents = io.BytesIO()
-        downloader = MediaIoBaseDownload(file_contents, request)
-        downloaded = self._get_content(downloader=downloader)
-        if not downloaded or not file_contents:
-            raise SourceConnectionError("nothing found to download")
-        return self._write_file(file_data=file_data, file_contents=file_contents)
+            with open(download_path, "wb") as f:
+                for chunk in response.iter_bytes():
+                    f.write(chunk)
+
+        return download_path
+
+    def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+        mime_type = file_data.additional_metadata.get("mimeType", "")
+        record_id = file_data.identifier
+
+        logger.debug(
+            f"Downloading file {file_data.source_identifiers.fullpath} of type {mime_type}"
+        )
+
+        download_url, ext = self._get_download_url_and_ext(record_id, mime_type)
+        download_path = self._download_url(file_data, download_url, ext)
+
+        file_data.additional_metadata.update(
+            {
+                "download_method": "export_link" if ext else "web_content_link",
+                "download_url_used": download_url,
+            }
+        )
+        file_data.local_download_path = str(download_path.resolve())
+
+        return self.generate_download_response(
+            file_data=file_data, download_path=download_path
+        )
 
 
 google_drive_source_entry = SourceRegistryEntry(

{unstructured_ingest-1.0.11 → unstructured_ingest-1.0.13}/unstructured_ingest/processes/connectors/ibm_watsonx/ibm_watsonx_s3.py

@@ -56,6 +56,12 @@ class IbmWatsonxConnectionConfig(ConnectionConfig):
     object_storage_endpoint: str = Field(description="Cloud Object Storage public endpoint")
     object_storage_region: str = Field(description="Cloud Object Storage region")
     catalog: str = Field(description="Catalog name")
+    max_retries_connection: int = Field(
+        default=10,
+        description="Maximum number of retries in case of a connection error (RESTError)",
+        ge=2,
+        le=100,
+    )
 
     _bearer_token: Optional[dict[str, Any]] = None
 
@@ -69,8 +75,8 @@ class IbmWatsonxConnectionConfig(ConnectionConfig):
 
     @property
     def bearer_token(self) -> str:
-        # Add 60 seconds to deal with edge cases where the token expires before the request is made
-        timestamp = int(time.time()) + 60
+        # Add 5 minutes to deal with edge cases where the token expires before the request is made
+        timestamp = int(time.time()) + (60 * 5)
         if self._bearer_token is None or self._bearer_token.get("expiration", 0) <= timestamp:
             self._bearer_token = self.generate_bearer_token()
         return self._bearer_token["access_token"]
@@ -145,10 +151,29 @@ class IbmWatsonxConnectionConfig(ConnectionConfig):
     @contextmanager
     def get_catalog(self) -> Generator["RestCatalog", None, None]:
         from pyiceberg.catalog import load_catalog
+        from pyiceberg.exceptions import RESTError
+        from tenacity import (
+            before_log,
+            retry,
+            retry_if_exception_type,
+            stop_after_attempt,
+            wait_exponential,
+        )
+
+        # Retry connection in case of a connection error
+        @retry(
+            stop=stop_after_attempt(self.max_retries_connection),
+            wait=wait_exponential(exp_base=2, multiplier=1, min=2, max=10),
+            retry=retry_if_exception_type(RESTError),
+            before=before_log(logger, logging.DEBUG),
+            reraise=True,
+        )
+        def _get_catalog(catalog_config: dict[str, Any]) -> "RestCatalog":
+            return load_catalog(**catalog_config)
 
         try:
             catalog_config = self.get_catalog_config()
-            catalog = load_catalog(**catalog_config)
+            catalog = _get_catalog(catalog_config)
         except Exception as e:
             logger.error(f"Failed to connect to catalog '{self.catalog}': {e}", exc_info=True)
             raise ProviderError(f"Failed to connect to catalog '{self.catalog}': {e}")
@@ -172,7 +197,10 @@ class IbmWatsonxUploaderConfig(UploaderConfig):
     namespace: str = Field(description="Namespace name")
     table: str = Field(description="Table name")
     max_retries: int = Field(
-        default=5, description="Maximum number of retries to upload data", ge=2, le=500
+        default=50,
+        description="Maximum number of retries to upload data (CommitFailedException)",
+        ge=2,
+        le=500,
     )
     record_id_key: str = Field(
         default=RECORD_ID_LABEL,
@@ -240,7 +268,7 @@ class IbmWatsonxUploader(SQLUploader):
     def upload_data_table(
         self, table: "Table", data_table: "ArrowTable", file_data: FileData
     ) -> None:
-        from pyiceberg.exceptions import CommitFailedException
+        from pyiceberg.exceptions import CommitFailedException, RESTError
         from tenacity import (
             before_log,
             retry,
@@ -265,21 +293,51 @@ class IbmWatsonxUploader(SQLUploader):
                 table.refresh()
                 logger.debug(e)
                 raise IcebergCommitFailedException(e)
+            except RESTError:
+                raise
             except Exception as e:
                 raise ProviderError(f"Failed to upload data to table: {e}")
 
         try:
             return _upload_data_table(table, data_table, file_data)
+        except RESTError:
+            raise
         except ProviderError:
             raise
         except Exception as e:
             raise ProviderError(f"Failed to upload data to table: {e}")
 
+    @requires_dependencies(["pyiceberg", "tenacity"], extras="ibm-watsonx-s3")
     def upload_dataframe(self, df: "DataFrame", file_data: FileData) -> None:
+        from pyiceberg.exceptions import RESTError
+        from tenacity import (
+            before_log,
+            retry,
+            retry_if_exception_type,
+            stop_after_attempt,
+            wait_exponential,
+        )
+
         data_table = self._df_to_arrow_table(df)
 
-        with self.get_table() as table:
-            self.upload_data_table(table, data_table, file_data)
+        # Retry connection in case of a connection error or token expiration
+        @retry(
+            stop=stop_after_attempt(self.connection_config.max_retries_connection),
+            wait=wait_exponential(exp_base=2, multiplier=1, min=2, max=10),
+            retry=retry_if_exception_type(RESTError),
+            before=before_log(logger, logging.DEBUG),
+            reraise=True,
+        )
+        def _upload_dataframe(data_table: Any, file_data: FileData) -> None:
+            with self.get_table() as table:
+                self.upload_data_table(table, data_table, file_data)
+
+        try:
+            return _upload_dataframe(data_table, file_data)
+        except ProviderError:
+            raise
+        except Exception as e:
+            raise ProviderError(f"Failed to upload data to table: {e}")
 
     @requires_dependencies(["pandas"], extras="ibm-watsonx-s3")
     def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:

{unstructured_ingest-1.0.11 → unstructured_ingest-1.0.13}/unstructured_ingest/processes/connectors/notion/types/database_properties/select.py

@@ -38,6 +38,7 @@ class Select(DBPropertyBase):
     id: str
     name: str
     select: SelectProp
+    description: Optional[str] = None
     type: str = "select"
 
     @classmethod

unstructured_ingest-1.0.11/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.0.11"  # pragma: no cover