unstructured-ingest 1.2.10__tar.gz → 1.2.13__tar.gz

{unstructured_ingest-1.2.10 → unstructured_ingest-1.2.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: unstructured_ingest
-Version: 1.2.10
+Version: 1.2.13
 Summary: Local ETL data pipeline to get data RAG ready
 Author-email: Unstructured Technologies <devops@unstructuredai.io>
 License-Expression: Apache-2.0

{unstructured_ingest-1.2.10 → unstructured_ingest-1.2.13}/pyproject.toml

@@ -143,6 +143,7 @@ test = [
     "pyarrow",
     "networkx",
     "htmlbuilder",
+    "office365-rest-python-client",
 ]
 # Add constraints needed for CI
 ci = [

unstructured_ingest-1.2.13/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "1.2.13"  # pragma: no cover

{unstructured_ingest-1.2.10 → unstructured_ingest-1.2.13}/unstructured_ingest/embed/bedrock.py

@@ -58,9 +58,22 @@ def conform_query(query: str, provider: str) -> dict:
 
 
 class BedrockEmbeddingConfig(EmbeddingConfig):
-    aws_access_key_id: SecretStr = Field(description="aws access key id")
-    aws_secret_access_key: SecretStr = Field(description="aws secret access key")
-    region_name: str = Field(description="aws region name", default="us-west-2")
+    aws_access_key_id: SecretStr | None = Field(description="aws access key id", default=None)
+    aws_secret_access_key: SecretStr | None = Field(
+        description="aws secret access key", default=None
+    )
+    region_name: str = Field(
+        description="aws region name", 
+        default_factory=lambda: (
+            os.getenv("BEDROCK_REGION_NAME") or 
+            os.getenv("AWS_DEFAULT_REGION") or 
+            "us-west-2"
+        )
+    )
+    endpoint_url: str | None = Field(description="custom bedrock endpoint url", default=None)
+    access_method: str = Field(
+        description="authentication method", default="credentials"
+    )  # "credentials" or "iam"
     embedder_model_name: str = Field(
         default="amazon.titan-embed-text-v1",
         alias="model_name",
@@ -96,6 +109,20 @@ class BedrockEmbeddingConfig(EmbeddingConfig):
         return e
 
     def run_precheck(self) -> None:
+        # Validate access method and credentials configuration
+        if self.access_method == "credentials":
+            if not (self.aws_access_key_id and self.aws_secret_access_key):
+                raise ValueError(
+                    "Credentials access method requires aws_access_key_id and aws_secret_access_key"
+                )
+        elif self.access_method == "iam":
+            # For IAM, credentials are handled by AWS SDK
+            pass
+        else:
+            raise ValueError(
+                f"Invalid access_method: {self.access_method}. Must be 'credentials' or 'iam'"
+            )
+            
         client = self.get_bedrock_client()
         try:
             model_info = client.list_foundation_models(byOutputModality="EMBEDDING")
@@ -113,11 +140,30 @@ class BedrockEmbeddingConfig(EmbeddingConfig):
             raise self.wrap_error(e=e)
 
     def get_client_kwargs(self) -> dict:
-        return {
-            "aws_access_key_id": self.aws_access_key_id.get_secret_value(),
-            "aws_secret_access_key": self.aws_secret_access_key.get_secret_value(),
+        kwargs = {
             "region_name": self.region_name,
         }
+        
+        if self.endpoint_url:
+            kwargs["endpoint_url"] = self.endpoint_url
+            
+        if self.access_method == "credentials":
+            if self.aws_access_key_id and self.aws_secret_access_key:
+                kwargs["aws_access_key_id"] = self.aws_access_key_id.get_secret_value()
+                kwargs["aws_secret_access_key"] = self.aws_secret_access_key.get_secret_value()
+            else:
+                raise ValueError(
+                    "Credentials access method requires aws_access_key_id and aws_secret_access_key"
+                )
+        elif self.access_method == "iam":
+            # For IAM, boto3 will use default credential chain (IAM roles, environment, etc.)
+            pass
+        else:
+            raise ValueError(
+                f"Invalid access_method: {self.access_method}. Must be 'credentials' or 'iam'"
+            )
+            
+        return kwargs
 
     @requires_dependencies(
         ["boto3"],

unstructured_ingest-1.2.13/unstructured_ingest/errors_v2.py

@@ -0,0 +1,156 @@
+from abc import ABC
+from functools import wraps
+from typing import Optional
+
+
+class UnstructuredIngestError(Exception, ABC):
+    error_string: str
+    status_code: Optional[int] = None
+
+    @classmethod
+    def wrap(cls, f):
+        """
+        Provides a wrapper for a function that catches any exception and
+        re-raises it as the customer error. If the exception itself is already an instance
+        of the custom error, re-raises original error.
+        """
+
+        @wraps(f)
+        def wrapper(*args, **kwargs):
+            try:
+                return f(*args, **kwargs)
+            except BaseException as error:
+                if not isinstance(error, cls) and not issubclass(type(error), cls):
+                    raise cls(cls.error_string.format(str(error))) from error
+                raise
+
+        return wrapper
+
+
+class ConnectionError(UnstructuredIngestError):
+    error_string = "Connection error: {}"
+    status_code: Optional[int] = 400
+
+
+class SourceConnectionError(ConnectionError):
+    error_string = "Error in getting data from upstream data source: {}"
+    status_code: Optional[int] = 400
+
+
+class SourceConnectionNetworkError(SourceConnectionError):
+    error_string = "Error in connecting to upstream data source: {}"
+    status_code: Optional[int] = 400
+
+
+class DestinationConnectionError(ConnectionError):
+    error_string = "Error in connecting to downstream data source: {}"
+    status_code: Optional[int] = 400
+
+
+class EmbeddingEncoderConnectionError(ConnectionError):
+    error_string = "Error in connecting to the embedding model provider: {}"
+    status_code: Optional[int] = 400
+
+
+class UserError(UnstructuredIngestError):
+    error_string = "User error: {}"
+    status_code: Optional[int] = 401
+
+
+class UserAuthError(UserError):
+    error_string = "User authentication error: {}"
+    status_code: Optional[int] = 401
+
+
+class RateLimitError(UserError):
+    error_string = "Rate limit error: {}"
+    status_code: Optional[int] = 429
+
+
+class NotFoundError(UnstructuredIngestError):
+    error_string = "Not found error: {}"
+    status_code: Optional[int] = 404
+
+
+class TimeoutError(UnstructuredIngestError):
+    error_string = "Timeout error: {}"
+    status_code: Optional[int] = 408
+
+
+class ResponseError(UnstructuredIngestError):
+    error_string = "Response error: {}"
+    status_code: Optional[int] = 400
+
+
+class WriteError(UnstructuredIngestError):
+    error_string = "Error in writing to downstream data source: {}"
+    status_code: Optional[int] = 400
+
+
+class ProviderError(UnstructuredIngestError):
+    error_string = "Provider error: {}"
+    status_code: Optional[int] = 500
+
+
+class ValueError(UnstructuredIngestError):
+    error_string = "Value error: {}"
+
+
+class PartitionError(UnstructuredIngestError):
+    error_string = "Error in partitioning content: {}"
+
+
+class QuotaError(UserError):
+    error_string = "Quota error: {}"
+
+
+class MissingCategoryError(UnstructuredIngestError):
+    error_string = "Missing category error: {}"
+
+
+class ValidationError(UnstructuredIngestError):
+    error_string = "Validation error: {}"
+
+
+class KeyError(UnstructuredIngestError):
+    error_string = "Key error: {}"
+
+
+class FileExistsError(UnstructuredIngestError):
+    error_string = "File exists error: {}"
+
+
+class TypeError(UnstructuredIngestError):
+    error_string = "Type error: {}"
+
+
+class IcebergCommitFailedException(UnstructuredIngestError):
+    error_string = "Failed to commit changes to the iceberg table"
+
+
+recognized_errors = [
+    UserError,
+    UserAuthError,
+    RateLimitError,
+    QuotaError,
+    ProviderError,
+    NotFoundError,
+    TypeError,
+    ValueError,
+    FileExistsError,
+    TimeoutError,
+    KeyError,
+    ResponseError,
+    ValidationError,
+    PartitionError,
+    WriteError,
+    ConnectionError,
+    SourceConnectionError,
+    SourceConnectionNetworkError,
+    DestinationConnectionError,
+    EmbeddingEncoderConnectionError,
+]
+
+
+def is_internal_error(e: Exception) -> bool:
+    return any(isinstance(e, recognized_error) for recognized_error in recognized_errors)

{unstructured_ingest-1.2.10 → unstructured_ingest-1.2.13}/unstructured_ingest/processes/connectors/milvus.py

@@ -10,7 +10,6 @@ from unstructured_ingest.data_types.file_data import FileData
 from unstructured_ingest.error import (
     DestinationConnectionError,
     KeyError,
-    ValueError,
     WriteError,
 )
 from unstructured_ingest.interfaces import (

{unstructured_ingest-1.2.10 → unstructured_ingest-1.2.13}/unstructured_ingest/processes/connectors/sharepoint.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import logging
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, AsyncIterator, Optional
 
@@ -210,7 +211,7 @@ class SharepointIndexer(OnedriveIndexer):
 
 
 class SharepointDownloaderConfig(OnedriveDownloaderConfig):
-    pass
+    max_retries: int = 10
 
 
 @dataclass
@@ -219,10 +220,22 @@ class SharepointDownloader(OnedriveDownloader):
     download_config: SharepointDownloaderConfig
     connector_type: str = CONNECTOR_TYPE
 
+    @staticmethod
+    def retry_on_status_code(exc):
+        error_msg = str(exc).lower()
+        return "429" in error_msg or "activitylimitreached" in error_msg or "throttled" in error_msg
+
     @SourceConnectionNetworkError.wrap
     @requires_dependencies(["office365"], extras="sharepoint")
     def _fetch_file(self, file_data: FileData) -> DriveItem:
         from office365.runtime.client_request_exception import ClientRequestException
+        from tenacity import (
+            before_log,
+            retry,
+            retry_if_exception,
+            stop_after_attempt,
+            wait_exponential,
+        )
 
         if file_data.source_identifiers is None or not file_data.source_identifiers.fullpath:
             raise ValueError(
@@ -233,13 +246,27 @@ class SharepointDownloader(OnedriveDownloader):
         server_relative_path = file_data.source_identifiers.fullpath
         client = self.connection_config.get_client()
 
-        try:
-            client_site = client.sites.get_by_url(self.connection_config.site).get().execute_query()
-            site_drive_item = self.connection_config._get_drive_item(client_site)
-        except ClientRequestException:
-            logger.info("Site not found")
-            raise SourceConnectionError(f"Site not found: {self.connection_config.site}")
-        file = site_drive_item.get_by_path(server_relative_path).get().execute_query()
+        @retry(
+            stop=stop_after_attempt(self.download_config.max_retries),
+            wait=wait_exponential(exp_base=2, multiplier=1, min=2, max=10),
+            retry=retry_if_exception(self.retry_on_status_code),
+            before=before_log(logger, logging.DEBUG),
+            reraise=True,
+        )
+        def _get_item_by_path() -> DriveItem:
+            try:
+                client_site = (
+                    client.sites.get_by_url(self.connection_config.site).get().execute_query()
+                )
+                site_drive_item = self.connection_config._get_drive_item(client_site)
+            except ClientRequestException:
+                logger.info(f"Site not found: {self.connection_config.site}")
+                raise SourceConnectionError(f"Site not found: {self.connection_config.site}")
+            file = site_drive_item.get_by_path(server_relative_path).get().execute_query()
+            return file
+
+        # Call the retry-wrapped function
+        file = _get_item_by_path()
 
         if not file:
             raise NotFoundError(f"file not found: {server_relative_path}")

unstructured_ingest-1.2.10/unstructured_ingest/__version__.py

@@ -1 +0,0 @@
-__version__ = "1.2.10"  # pragma: no cover