unstructured-ingest 1.2.8__py3-none-any.whl → 1.2.10__py3-none-any.whl

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.2.8"  # pragma: no cover
+__version__ = "1.2.10"  # pragma: no cover

unstructured_ingest/embed/bedrock.py

@@ -13,7 +13,7 @@ from unstructured_ingest.embed.interfaces import (
     BaseEmbeddingEncoder,
     EmbeddingConfig,
 )
-from unstructured_ingest.errors_v2 import (
+from unstructured_ingest.error import (
     ProviderError,
     RateLimitError,
     UserAuthError,

unstructured_ingest/embed/octoai.py

@@ -8,7 +8,7 @@ from unstructured_ingest.embed.interfaces import (
     BaseEmbeddingEncoder,
     EmbeddingConfig,
 )
-from unstructured_ingest.errors_v2 import (
+from unstructured_ingest.error import (
     ProviderError,
     QuotaError,
     RateLimitError,

unstructured_ingest/embed/openai.py

@@ -8,7 +8,7 @@ from unstructured_ingest.embed.interfaces import (
     BaseEmbeddingEncoder,
     EmbeddingConfig,
 )
-from unstructured_ingest.errors_v2 import (
+from unstructured_ingest.error import (
     ProviderError,
     QuotaError,
     RateLimitError,

unstructured_ingest/embed/togetherai.py

@@ -8,13 +8,13 @@ from unstructured_ingest.embed.interfaces import (
     BaseEmbeddingEncoder,
     EmbeddingConfig,
 )
-from unstructured_ingest.errors_v2 import (
+from unstructured_ingest.error import (
     ProviderError,
     UserAuthError,
     UserError,
     is_internal_error,
 )
-from unstructured_ingest.errors_v2 import (
+from unstructured_ingest.error import (
     RateLimitError as CustomRateLimitError,
 )
 from unstructured_ingest.logger import logger
@@ -46,8 +46,8 @@ class TogetherAIEmbeddingConfig(EmbeddingConfig):
             return UserAuthError(message)
         if isinstance(e, RateLimitError):
             return CustomRateLimitError(message)
-        
-        status_code = getattr(e, 'status_code', None)
+
+        status_code = getattr(e, "status_code", None)
         if status_code is not None:
             if 400 <= status_code < 500:
                 return UserError(message)

unstructured_ingest/embed/vertexai.py

@@ -13,7 +13,7 @@ from unstructured_ingest.embed.interfaces import (
     BaseEmbeddingEncoder,
     EmbeddingConfig,
 )
-from unstructured_ingest.errors_v2 import UserAuthError, is_internal_error
+from unstructured_ingest.error import UserAuthError, is_internal_error
 from unstructured_ingest.utils.dep_check import requires_dependencies
 
 if TYPE_CHECKING:

unstructured_ingest/embed/voyageai.py

@@ -8,8 +8,8 @@ from unstructured_ingest.embed.interfaces import (
     BaseEmbeddingEncoder,
     EmbeddingConfig,
 )
-from unstructured_ingest.errors_v2 import ProviderError, UserAuthError, UserError, is_internal_error
-from unstructured_ingest.errors_v2 import (
+from unstructured_ingest.error import ProviderError, UserAuthError, UserError, is_internal_error
+from unstructured_ingest.error import (
     RateLimitError as CustomRateLimitError,
 )
 from unstructured_ingest.logger import logger

unstructured_ingest/error.py

@@ -1,9 +1,11 @@
 from abc import ABC
 from functools import wraps
+from typing import Optional
 
 
-class CustomError(Exception, ABC):
+class UnstructuredIngestError(Exception, ABC):
     error_string: str
+    status_code: Optional[int] = None
 
     @classmethod
     def wrap(cls, f):
@@ -25,25 +27,130 @@ class CustomError(Exception, ABC):
         return wrapper
 
 
-class SourceConnectionError(CustomError):
+class ConnectionError(UnstructuredIngestError):
+    error_string = "Connection error: {}"
+    status_code: Optional[int] = 400
+
+
+class SourceConnectionError(ConnectionError):
     error_string = "Error in getting data from upstream data source: {}"
+    status_code: Optional[int] = 400
 
 
 class SourceConnectionNetworkError(SourceConnectionError):
     error_string = "Error in connecting to upstream data source: {}"
+    status_code: Optional[int] = 400
 
 
-class DestinationConnectionError(CustomError):
+class DestinationConnectionError(ConnectionError):
     error_string = "Error in connecting to downstream data source: {}"
+    status_code: Optional[int] = 400
 
 
-class EmbeddingEncoderConnectionError(CustomError):
+class EmbeddingEncoderConnectionError(ConnectionError):
     error_string = "Error in connecting to the embedding model provider: {}"
+    status_code: Optional[int] = 400
+
+
+class UserError(UnstructuredIngestError):
+    error_string = "User error: {}"
+    status_code: Optional[int] = 401
+
+
+class UserAuthError(UserError):
+    error_string = "User authentication error: {}"
+    status_code: Optional[int] = 401
+
+
+class RateLimitError(UserError):
+    error_string = "Rate limit error: {}"
+    status_code: Optional[int] = 429
+
 
+class NotFoundError(UnstructuredIngestError):
+    error_string = "Not found error: {}"
+    status_code: Optional[int] = 404
 
-class WriteError(CustomError):
+
+class TimeoutError(UnstructuredIngestError):
+    error_string = "Timeout error: {}"
+    status_code: Optional[int] = 408
+
+
+class ResponseError(UnstructuredIngestError):
+    error_string = "Response error: {}"
+    status_code: Optional[int] = 400
+
+
+class WriteError(UnstructuredIngestError):
     error_string = "Error in writing to downstream data source: {}"
+    status_code: Optional[int] = 400
+
+
+class ProviderError(UnstructuredIngestError):
+    error_string = "Provider error: {}"
+    status_code: Optional[int] = 500
+
+
+class ValueError(UnstructuredIngestError):
+    error_string = "Value error: {}"
 
 
-class PartitionError(CustomError):
+class PartitionError(UnstructuredIngestError):
     error_string = "Error in partitioning content: {}"
+
+
+class QuotaError(UserError):
+    error_string = "Quota error: {}"
+
+
+class MissingCategoryError(UnstructuredIngestError):
+    error_string = "Missing category error: {}"
+
+
+class ValidationError(UnstructuredIngestError):
+    error_string = "Validation error: {}"
+
+
+class KeyError(UnstructuredIngestError):
+    error_string = "Key error: {}"
+
+
+class FileExistsError(UnstructuredIngestError):
+    error_string = "File exists error: {}"
+
+
+class TypeError(UnstructuredIngestError):
+    error_string = "Type error: {}"
+
+
+class IcebergCommitFailedException(UnstructuredIngestError):
+    error_string = "Failed to commit changes to the iceberg table"
+
+
+recognized_errors = [
+    UserError,
+    UserAuthError,
+    RateLimitError,
+    QuotaError,
+    ProviderError,
+    NotFoundError,
+    TypeError,
+    ValueError,
+    FileExistsError,
+    TimeoutError,
+    KeyError,
+    ResponseError,
+    ValidationError,
+    PartitionError,
+    WriteError,
+    ConnectionError,
+    SourceConnectionError,
+    SourceConnectionNetworkError,
+    DestinationConnectionError,
+    EmbeddingEncoderConnectionError,
+]
+
+
+def is_internal_error(e: Exception) -> bool:
+    return any(isinstance(e, recognized_error) for recognized_error in recognized_errors)

unstructured_ingest/interfaces/downloader.py

@@ -36,11 +36,11 @@ class Downloader(BaseProcess, BaseConnector, ABC):
     def get_download_path(self, file_data: FileData) -> Optional[Path]:
         if not file_data.source_identifiers:
             return None
-        
+
         rel_path = file_data.source_identifiers.relative_path
         if not rel_path:
             return None
-        
+
         rel_path = rel_path[1:] if rel_path.startswith("/") else rel_path
         return self.download_dir / Path(rel_path)
 

unstructured_ingest/processes/connectors/airtable.py

@@ -6,6 +6,7 @@ from uuid import NAMESPACE_DNS, uuid5
 from pydantic import BaseModel, Field, Secret, field_validator
 
 from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.error import ValueError
 from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,

unstructured_ingest/processes/connectors/azure_ai_search.py

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Generator
 from pydantic import Field, Secret
 
 from unstructured_ingest.data_types.file_data import FileData
-from unstructured_ingest.error import DestinationConnectionError, WriteError
+from unstructured_ingest.error import DestinationConnectionError, ValueError, WriteError
 from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,

unstructured_ingest/processes/connectors/chroma.py

@@ -7,7 +7,7 @@ from pydantic import Field, Secret
 from pydantic.functional_validators import BeforeValidator
 
 from unstructured_ingest.data_types.file_data import FileData
-from unstructured_ingest.error import DestinationConnectionError
+from unstructured_ingest.error import DestinationConnectionError, ValueError
 from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -151,7 +151,7 @@ class ChromaUploader(Uploader):
                 metadatas=batch["metadatas"],
             )
         except Exception as e:
-            raise ValueError(f"chroma error: {e}") from e
+            raise DestinationConnectionError(f"chroma error: {e}") from e
 
     @staticmethod
     def prepare_chroma_list(chunk: tuple[dict[str, Any]]) -> dict[str, list[Any]]:

unstructured_ingest/processes/connectors/confluence.py

@@ -11,8 +11,12 @@ from unstructured_ingest.data_types.file_data import (
     FileDataSourceMetadata,
     SourceIdentifiers,
 )
-from unstructured_ingest.error import SourceConnectionError
-from unstructured_ingest.errors_v2 import UserAuthError, UserError
+from unstructured_ingest.error import (
+    SourceConnectionError,
+    UserAuthError,
+    UserError,
+    ValueError,
+)
 from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,

unstructured_ingest/processes/connectors/databricks/volumes.py

@@ -12,7 +12,7 @@ from unstructured_ingest.data_types.file_data import (
     FileDataSourceMetadata,
     SourceIdentifiers,
 )
-from unstructured_ingest.errors_v2 import (
+from unstructured_ingest.error import (
     ProviderError,
     RateLimitError,
     UserAuthError,

unstructured_ingest/processes/connectors/databricks/volumes_table.py

@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Generator, Optional
 from pydantic import Field
 
 from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.error import ValueError
 from unstructured_ingest.interfaces import (
     Uploader,
     UploaderConfig,

unstructured_ingest/processes/connectors/delta_table.py

@@ -9,7 +9,7 @@ from urllib.parse import urlparse
 from pydantic import Field, Secret
 
 from unstructured_ingest.data_types.file_data import FileData
-from unstructured_ingest.error import DestinationConnectionError
+from unstructured_ingest.error import DestinationConnectionError, ValueError
 from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -254,12 +254,12 @@ class DeltaTableUploader(Uploader):
                 if not queue.empty():
                     error_message = queue.get()
                     logger.error("Exception occurred in write_deltalake: %s", error_message)
-                    raise RuntimeError(f"Error in write_deltalake: {error_message}")
+                    raise DestinationConnectionError(f"Error in write_deltalake: {error_message}")
 
                 # If the subprocess terminated abnormally but produced no traceback (e.g., SIGABRT),
                 # still raise a helpful error for callers.
                 if not current_process().daemon and writer.exitcode != 0:
-                    raise RuntimeError(
+                    raise DestinationConnectionError(
                         f"write_deltalake subprocess exited with code {writer.exitcode}"
                     )
 

unstructured_ingest/processes/connectors/discord.py

@@ -9,7 +9,7 @@ from unstructured_ingest.data_types.file_data import (
     FileDataSourceMetadata,
     SourceIdentifiers,
 )
-from unstructured_ingest.error import SourceConnectionError
+from unstructured_ingest.error import UserAuthError, ValueError
 from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -70,9 +70,9 @@ class DiscordIndexer(Indexer):
 
     def precheck(self) -> None:
         if not self.connection_config.access_config.get_secret_value().token:
-            raise SourceConnectionError("Discord token is missing")
+            raise UserAuthError("Discord token is missing")
         if not self.index_config.channels:
-            raise SourceConnectionError("No channels provided")
+            raise ValueError("No channels provided")
 
     def get_channel_file_data(self, channel_id: str) -> Optional[FileData]:
         # Fetch channel metadata

unstructured_ingest/processes/connectors/duckdb/duckdb.py

@@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, Generator, Optional
 from pydantic import Field, Secret
 
 from unstructured_ingest.data_types.file_data import FileData
-from unstructured_ingest.error import DestinationConnectionError
+from unstructured_ingest.error import DestinationConnectionError, ValueError
 from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,

unstructured_ingest/processes/connectors/elasticsearch/elasticsearch.py

@@ -19,6 +19,7 @@ from unstructured_ingest.error import (
     DestinationConnectionError,
     SourceConnectionError,
     SourceConnectionNetworkError,
+    UnstructuredIngestError,
 )
 from unstructured_ingest.interfaces import (
     AccessConfig,
@@ -440,10 +441,10 @@ class ElasticsearchUploader(Uploader):
                     logger.error(
                         f"Batch upload failed - {e} - with following errors: {sanitized_errors}"
                     )
-                    raise e
+                    raise DestinationConnectionError(str(e))
                 except Exception as e:
                     logger.error(f"Batch upload failed - {e}")
-                    raise e
+                    raise UnstructuredIngestError(str(e))
 
     def _sanitize_bulk_index_error(self, error: dict[str, dict]) -> dict:
         """Remove data uploaded to index from the log, leave only error information.

unstructured_ingest/processes/connectors/fsspec/azure.py

@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, Generator, Optional
 from pydantic import Field, Secret
 
 from unstructured_ingest.data_types.file_data import FileDataSourceMetadata
-from unstructured_ingest.errors_v2 import ProviderError, UserAuthError, UserError
+from unstructured_ingest.error import ProviderError, UserAuthError, UserError, ValueError
 from unstructured_ingest.logger import logger
 from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,

unstructured_ingest/processes/connectors/fsspec/box.py

@@ -10,7 +10,7 @@ from pydantic import Field, Secret
 from pydantic.functional_validators import BeforeValidator
 
 from unstructured_ingest.data_types.file_data import FileDataSourceMetadata
-from unstructured_ingest.errors_v2 import ProviderError, UserAuthError, UserError
+from unstructured_ingest.error import ProviderError, UserAuthError, UserError
 from unstructured_ingest.logger import logger
 from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,

unstructured_ingest/processes/connectors/fsspec/dropbox.py

@@ -7,12 +7,13 @@ from typing import TYPE_CHECKING, Any, Optional
 from pydantic import Field, Secret
 
 from unstructured_ingest.data_types.file_data import FileDataSourceMetadata
-from unstructured_ingest.errors_v2 import (
+from unstructured_ingest.error import (
     ProviderError,
     UserAuthError,
     UserError,
+    ValueError,
 )
-from unstructured_ingest.errors_v2 import (
+from unstructured_ingest.error import (
     RateLimitError as CustomRateLimitError,
 )
 from unstructured_ingest.logger import logger

unstructured_ingest/processes/connectors/fsspec/fsspec.py

@@ -17,6 +17,7 @@ from unstructured_ingest.data_types.file_data import (
     FileDataSourceMetadata,
     SourceIdentifiers,
 )
+from unstructured_ingest.error import TypeError, ValueError
 from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -265,29 +266,26 @@ FsspecDownloaderConfigT = TypeVar("FsspecDownloaderConfigT", bound=FsspecDownloa
 @dataclass
 class FsspecDownloader(Downloader):
     TEMP_DIR_PREFIX = "unstructured_"
-    
+
     protocol: str
     connection_config: FsspecConnectionConfigT
     connector_type: str = CONNECTOR_TYPE
     download_config: Optional[FsspecDownloaderConfigT] = field(
         default_factory=lambda: FsspecDownloaderConfig()
     )
-    
+
     def get_download_path(self, file_data: FileData) -> Optional[Path]:
         has_source_identifiers = file_data.source_identifiers is not None
         has_filename = has_source_identifiers and file_data.source_identifiers.filename
-        
+
         if not (has_source_identifiers and has_filename):
             return None
-        
+
         filename = file_data.source_identifiers.filename
-        
+
         mkdir_concurrent_safe(self.download_dir)
-        
-        temp_dir = tempfile.mkdtemp(
-            prefix=self.TEMP_DIR_PREFIX, 
-            dir=self.download_dir
-        )
+
+        temp_dir = tempfile.mkdtemp(prefix=self.TEMP_DIR_PREFIX, dir=self.download_dir)
         return Path(temp_dir) / filename
 
     def is_async(self) -> bool:

unstructured_ingest/processes/connectors/fsspec/gcs.py

@@ -10,7 +10,7 @@ from dateutil import parser
 from pydantic import Field, Secret
 
 from unstructured_ingest.data_types.file_data import FileDataSourceMetadata
-from unstructured_ingest.errors_v2 import ProviderError, UserError
+from unstructured_ingest.error import ProviderError, UserError, ValueError
 from unstructured_ingest.logger import logger
 from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -125,7 +125,7 @@ class GcsConnectionConfig(FsspecConnectionConfig):
                 raise UserError(message)
             if http_error_code >= 500:
                 raise ProviderError(message)
-        logger.error(f"unhandled exception from gcs ({type(e)}): {e}", exc_info=True)
+        logger.error(f"({type(e)} from gcs): {e}", exc_info=True)
         return e
 
 

unstructured_ingest/processes/connectors/fsspec/s3.py

@@ -10,7 +10,7 @@ from pydantic import Field, Secret
 from unstructured_ingest.data_types.file_data import (
     FileDataSourceMetadata,
 )
-from unstructured_ingest.errors_v2 import ProviderError, UserAuthError, UserError
+from unstructured_ingest.error import ProviderError, UserAuthError, UserError
 from unstructured_ingest.logger import logger
 from unstructured_ingest.processes.connector_registry import (
     DestinationRegistryEntry,
@@ -118,6 +118,9 @@ class S3ConnectionConfig(FsspecConnectionConfig):
         if self.endpoint_url:
             access_configs["endpoint_url"] = self.endpoint_url
 
+        # This allows s3fs to properly follow AWS region redirects
+        access_configs["cache_regions"] = True
+
         return access_configs
 
     @requires_dependencies(["s3fs", "fsspec"], extras="s3")

unstructured_ingest/processes/connectors/github.py

@@ -12,7 +12,12 @@ from unstructured_ingest.data_types.file_data import (
     FileDataSourceMetadata,
     SourceIdentifiers,
 )
-from unstructured_ingest.errors_v2 import ProviderError, UserAuthError, UserError
+from unstructured_ingest.error import (
+    ProviderError,
+    UnstructuredIngestError,
+    UserAuthError,
+    UserError,
+)
 from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -85,7 +90,7 @@ class GithubConnectionConfig(ConnectionConfig):
         if status_code > 500:
             return ProviderError(e.response.text)
         logger.debug(f"unhandled http error: {e}")
-        return e
+        return UnstructuredIngestError(str(e))
 
     @requires_dependencies(["requests"], extras="github")
     def wrap_error(self, e: Exception) -> Exception:
@@ -97,7 +102,7 @@ class GithubConnectionConfig(ConnectionConfig):
         if isinstance(e, HTTPError):
             return self.wrap_http_error(e=e)
         logger.debug(f"unhandled error: {e}")
-        return e
+        return UnstructuredIngestError(str(e))
 
 
 class GithubIndexerConfig(IndexerConfig):

unstructured_ingest/processes/connectors/gitlab.py

@@ -13,7 +13,7 @@ from unstructured_ingest.data_types.file_data import (
     FileDataSourceMetadata,
     SourceIdentifiers,
 )
-from unstructured_ingest.error import SourceConnectionError
+from unstructured_ingest.error import SourceConnectionError, ValueError
 from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,

unstructured_ingest/processes/connectors/google_drive.py

@@ -13,9 +13,7 @@ from unstructured_ingest.data_types.file_data import (
     FileDataSourceMetadata,
     SourceIdentifiers,
 )
-from unstructured_ingest.error import (
-    SourceConnectionError,
-)
+from unstructured_ingest.error import SourceConnectionError, UserAuthError, ValueError
 from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -113,7 +111,7 @@ class GoogleDriveConnectionConfig(ConnectionConfig):
         except HttpError as exc:
             raise ValueError(f"{exc.reason}")
         except exceptions.DefaultCredentialsError:
-            raise ValueError("The provided API key is invalid.")
+            raise UserAuthError("The provided API key is invalid.")
 
 
 class GoogleDriveIndexerConfig(IndexerConfig):

unstructured_ingest/processes/connectors/ibm_watsonx/ibm_watsonx_s3.py

@@ -8,7 +8,13 @@ from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple
 from pydantic import Field, Secret
 
 from unstructured_ingest.data_types.file_data import FileData
-from unstructured_ingest.errors_v2 import ProviderError, UserAuthError, UserError
+from unstructured_ingest.error import (
+    DestinationConnectionError,
+    IcebergCommitFailedException,
+    ProviderError,
+    UserAuthError,
+    UserError,
+)
 from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -40,10 +46,6 @@ DEFAULT_ICEBERG_URI_PATH = "/mds/iceberg"
 DEFAULT_ICEBERG_CATALOG_TYPE = "rest"
 
 
-class IcebergCommitFailedException(Exception):
-    """Failed to commit changes to the iceberg table."""
-
-
 class IbmWatsonxAccessConfig(AccessConfig):
     iam_api_key: str = Field(description="IBM IAM API Key")
     access_key_id: str = Field(description="Cloud Object Storage HMAC Access Key ID")
@@ -292,16 +294,16 @@ class IbmWatsonxUploader(SQLUploader):
             except CommitFailedException as e:
                 table.refresh()
                 logger.debug(e)
-                raise IcebergCommitFailedException(e)
-            except RESTError:
-                raise
+                raise IcebergCommitFailedException(str(e))
+            except RESTError as e:
+                raise DestinationConnectionError(str(e))
             except Exception as e:
                 raise ProviderError(f"Failed to upload data to table: {e}")
 
         try:
             return _upload_data_table(table, data_table, file_data)
-        except RESTError:
-            raise
+        except RESTError as e:
+            raise DestinationConnectionError(str(e))
         except ProviderError:
             raise
         except Exception as e:

unstructured_ingest/processes/connectors/jira.py

@@ -12,7 +12,7 @@ from unstructured_ingest.data_types.file_data import (
     FileDataSourceMetadata,
     SourceIdentifiers,
 )
-from unstructured_ingest.error import SourceConnectionError
+from unstructured_ingest.error import SourceConnectionError, ValueError
 from unstructured_ingest.interfaces import (
     AccessConfig,
     ConnectionConfig,