unstructured-ingest 1.2.32__py3-none-any.whl

unstructured_ingest/error.py

@@ -0,0 +1,156 @@
+from abc import ABC
+from functools import wraps
+from typing import Optional
+
+
+class UnstructuredIngestError(Exception, ABC):
+    error_string: str
+    status_code: Optional[int] = None
+
+    @classmethod
+    def wrap(cls, f):
+        """
+        Provides a wrapper for a function that catches any exception and
+        re-raises it as the customer error. If the exception itself is already an instance
+        of the custom error, re-raises original error.
+        """
+
+        @wraps(f)
+        def wrapper(*args, **kwargs):
+            try:
+                return f(*args, **kwargs)
+            except BaseException as error:
+                if not isinstance(error, cls) and not issubclass(type(error), cls):
+                    raise cls(cls.error_string.format(str(error))) from error
+                raise
+
+        return wrapper
+
+
+class ConnectionError(UnstructuredIngestError):
+    error_string = "Connection error: {}"
+    status_code: Optional[int] = 400
+
+
+class SourceConnectionError(ConnectionError):
+    error_string = "Error in getting data from upstream data source: {}"
+    status_code: Optional[int] = 400
+
+
+class SourceConnectionNetworkError(SourceConnectionError):
+    error_string = "Error in connecting to upstream data source: {}"
+    status_code: Optional[int] = 400
+
+
+class DestinationConnectionError(ConnectionError):
+    error_string = "Error in connecting to downstream data source: {}"
+    status_code: Optional[int] = 400
+
+
+class EmbeddingEncoderConnectionError(ConnectionError):
+    error_string = "Error in connecting to the embedding model provider: {}"
+    status_code: Optional[int] = 400
+
+
+class UserError(UnstructuredIngestError):
+    error_string = "User error: {}"
+    status_code: Optional[int] = 401
+
+
+class UserAuthError(UserError):
+    error_string = "User authentication error: {}"
+    status_code: Optional[int] = 401
+
+
+class RateLimitError(UserError):
+    error_string = "Rate limit error: {}"
+    status_code: Optional[int] = 429
+
+
+class NotFoundError(UnstructuredIngestError):
+    error_string = "Not found error: {}"
+    status_code: Optional[int] = 404
+
+
+class TimeoutError(UnstructuredIngestError):
+    error_string = "Timeout error: {}"
+    status_code: Optional[int] = 408
+
+
+class ResponseError(UnstructuredIngestError):
+    error_string = "Response error: {}"
+    status_code: Optional[int] = 400
+
+
+class WriteError(UnstructuredIngestError):
+    error_string = "Error in writing to downstream data source: {}"
+    status_code: Optional[int] = 400
+
+
+class ProviderError(UnstructuredIngestError):
+    error_string = "Provider error: {}"
+    status_code: Optional[int] = 500
+
+
+class ValueError(UnstructuredIngestError):
+    error_string = "Value error: {}"
+
+
+class PartitionError(UnstructuredIngestError):
+    error_string = "Error in partitioning content: {}"
+
+
+class QuotaError(UserError):
+    error_string = "Quota error: {}"
+
+
+class MissingCategoryError(UnstructuredIngestError):
+    error_string = "Missing category error: {}"
+
+
+class ValidationError(UnstructuredIngestError):
+    error_string = "Validation error: {}"
+
+
+class KeyError(UnstructuredIngestError):
+    error_string = "Key error: {}"
+
+
+class FileExistsError(UnstructuredIngestError):
+    error_string = "File exists error: {}"
+
+
+class TypeError(UnstructuredIngestError):
+    error_string = "Type error: {}"
+
+
+class IcebergCommitFailedException(UnstructuredIngestError):
+    error_string = "Failed to commit changes to the iceberg table"
+
+
+recognized_errors = [
+    UserError,
+    UserAuthError,
+    RateLimitError,
+    QuotaError,
+    ProviderError,
+    NotFoundError,
+    TypeError,
+    ValueError,
+    FileExistsError,
+    TimeoutError,
+    KeyError,
+    ResponseError,
+    ValidationError,
+    PartitionError,
+    WriteError,
+    ConnectionError,
+    SourceConnectionError,
+    SourceConnectionNetworkError,
+    DestinationConnectionError,
+    EmbeddingEncoderConnectionError,
+]
+
+
+def is_internal_error(e: Exception) -> bool:
+    return any(isinstance(e, recognized_error) for recognized_error in recognized_errors)

unstructured_ingest/errors_v2.py

@@ -0,0 +1,156 @@
+from abc import ABC
+from functools import wraps
+from typing import Optional
+
+
+class UnstructuredIngestError(Exception, ABC):
+    error_string: str
+    status_code: Optional[int] = None
+
+    @classmethod
+    def wrap(cls, f):
+        """
+        Provides a wrapper for a function that catches any exception and
+        re-raises it as the customer error. If the exception itself is already an instance
+        of the custom error, re-raises original error.
+        """
+
+        @wraps(f)
+        def wrapper(*args, **kwargs):
+            try:
+                return f(*args, **kwargs)
+            except BaseException as error:
+                if not isinstance(error, cls) and not issubclass(type(error), cls):
+                    raise cls(cls.error_string.format(str(error))) from error
+                raise
+
+        return wrapper
+
+
+class ConnectionError(UnstructuredIngestError):
+    error_string = "Connection error: {}"
+    status_code: Optional[int] = 400
+
+
+class SourceConnectionError(ConnectionError):
+    error_string = "Error in getting data from upstream data source: {}"
+    status_code: Optional[int] = 400
+
+
+class SourceConnectionNetworkError(SourceConnectionError):
+    error_string = "Error in connecting to upstream data source: {}"
+    status_code: Optional[int] = 400
+
+
+class DestinationConnectionError(ConnectionError):
+    error_string = "Error in connecting to downstream data source: {}"
+    status_code: Optional[int] = 400
+
+
+class EmbeddingEncoderConnectionError(ConnectionError):
+    error_string = "Error in connecting to the embedding model provider: {}"
+    status_code: Optional[int] = 400
+
+
+class UserError(UnstructuredIngestError):
+    error_string = "User error: {}"
+    status_code: Optional[int] = 401
+
+
+class UserAuthError(UserError):
+    error_string = "User authentication error: {}"
+    status_code: Optional[int] = 401
+
+
+class RateLimitError(UserError):
+    error_string = "Rate limit error: {}"
+    status_code: Optional[int] = 429
+
+
+class NotFoundError(UnstructuredIngestError):
+    error_string = "Not found error: {}"
+    status_code: Optional[int] = 404
+
+
+class TimeoutError(UnstructuredIngestError):
+    error_string = "Timeout error: {}"
+    status_code: Optional[int] = 408
+
+
+class ResponseError(UnstructuredIngestError):
+    error_string = "Response error: {}"
+    status_code: Optional[int] = 400
+
+
+class WriteError(UnstructuredIngestError):
+    error_string = "Error in writing to downstream data source: {}"
+    status_code: Optional[int] = 400
+
+
+class ProviderError(UnstructuredIngestError):
+    error_string = "Provider error: {}"
+    status_code: Optional[int] = 500
+
+
+class ValueError(UnstructuredIngestError):
+    error_string = "Value error: {}"
+
+
+class PartitionError(UnstructuredIngestError):
+    error_string = "Error in partitioning content: {}"
+
+
+class QuotaError(UserError):
+    error_string = "Quota error: {}"
+
+
+class MissingCategoryError(UnstructuredIngestError):
+    error_string = "Missing category error: {}"
+
+
+class ValidationError(UnstructuredIngestError):
+    error_string = "Validation error: {}"
+
+
+class KeyError(UnstructuredIngestError):
+    error_string = "Key error: {}"
+
+
+class FileExistsError(UnstructuredIngestError):
+    error_string = "File exists error: {}"
+
+
+class TypeError(UnstructuredIngestError):
+    error_string = "Type error: {}"
+
+
+class IcebergCommitFailedException(UnstructuredIngestError):
+    error_string = "Failed to commit changes to the iceberg table"
+
+
+recognized_errors = [
+    UserError,
+    UserAuthError,
+    RateLimitError,
+    QuotaError,
+    ProviderError,
+    NotFoundError,
+    TypeError,
+    ValueError,
+    FileExistsError,
+    TimeoutError,
+    KeyError,
+    ResponseError,
+    ValidationError,
+    PartitionError,
+    WriteError,
+    ConnectionError,
+    SourceConnectionError,
+    SourceConnectionNetworkError,
+    DestinationConnectionError,
+    EmbeddingEncoderConnectionError,
+]
+
+
+def is_internal_error(e: Exception) -> bool:
+    return any(isinstance(e, recognized_error) for recognized_error in recognized_errors)

unstructured_ingest/interfaces/__init__.py

@@ -0,0 +1,27 @@
+from .connector import AccessConfig, BaseConnector, ConnectionConfig
+from .downloader import Downloader, DownloaderConfig, DownloadResponse, download_responses
+from .indexer import Indexer, IndexerConfig
+from .process import BaseProcess
+from .processor import ProcessorConfig
+from .upload_stager import UploadStager, UploadStagerConfig
+from .uploader import UploadContent, Uploader, UploaderConfig, VectorDBUploader
+
+__all__ = [
+    "DownloadResponse",
+    "download_responses",
+    "Downloader",
+    "DownloaderConfig",
+    "Indexer",
+    "IndexerConfig",
+    "BaseProcess",
+    "ProcessorConfig",
+    "UploadStager",
+    "UploadStagerConfig",
+    "Uploader",
+    "UploaderConfig",
+    "UploadContent",
+    "AccessConfig",
+    "ConnectionConfig",
+    "BaseConnector",
+    "VectorDBUploader",
+]

unstructured_ingest/interfaces/connector.py

@@ -0,0 +1,56 @@
+from abc import ABC
+from dataclasses import dataclass
+from typing import Any, TypeVar, Union
+
+from pydantic import BaseModel, Secret, model_validator
+from pydantic.types import _SecretBase
+
+from unstructured_ingest.processes.utils.logging.connector import ConnectorLoggingMixin
+
+
+class AccessConfig(BaseModel):
+    """Meant to designate holding any sensitive information associated with other configs
+    and also for access specific configs."""
+
+
+AccessConfigT = TypeVar("AccessConfigT", bound=AccessConfig)
+
+
+class ConnectionConfig(BaseModel):
+    access_config: Secret[AccessConfigT]
+
+    def get_access_config(self) -> dict[str, Any]:
+        if not self.access_config:
+            return {}
+        return self.access_config.get_secret_value().model_dump()
+
+    @model_validator(mode="after")
+    def check_access_config(self):
+        access_config = self.access_config
+        if self._is_access_config_optional() and access_config is None:
+            return self
+        if not isinstance(access_config, _SecretBase):
+            raise ValueError("access_config must be an instance of SecretBase")
+        return self
+
+    def _is_access_config_optional(self) -> bool:
+        access_config_type = self.model_fields["access_config"].annotation
+        return (
+            hasattr(access_config_type, "__origin__")
+            and hasattr(access_config_type, "__args__")
+            and access_config_type.__origin__ is Union
+            and len(access_config_type.__args__) == 2
+            and type(None) in access_config_type.__args__
+        )
+
+
+ConnectionConfigT = TypeVar("ConnectionConfigT", bound=ConnectionConfig)
+
+
+@dataclass
+class BaseConnector(ABC, ConnectorLoggingMixin):
+    connection_config: ConnectionConfigT
+
+    def __post_init__(self):
+        """Initialize the logging mixin after dataclass initialization."""
+        ConnectorLoggingMixin.__init__(self)

unstructured_ingest/interfaces/downloader.py

@@ -0,0 +1,90 @@
+import os
+from abc import ABC
+from pathlib import Path
+from typing import Any, Optional, TypedDict, TypeVar, Union
+
+from pydantic import BaseModel, Field
+
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.interfaces.connector import BaseConnector
+from unstructured_ingest.interfaces.process import BaseProcess
+
+
+class DownloaderConfig(BaseModel):
+    download_dir: Optional[Path] = Field(
+        default=None,
+        description="Where files are downloaded to, defaults to a location at"
+        "`$HOME/.cache/unstructured/ingest/<connector name>/<SHA256>`.",
+    )
+
+
+DownloaderConfigT = TypeVar("DownloaderConfigT", bound=DownloaderConfig)
+
+
+class DownloadResponse(TypedDict):
+    file_data: FileData
+    path: Path
+
+
+download_responses = Union[list[DownloadResponse], DownloadResponse]
+
+
+class Downloader(BaseProcess, BaseConnector, ABC):
+    connector_type: str
+    download_config: DownloaderConfigT
+
+    def get_download_path(self, file_data: FileData) -> Optional[Path]:
+        if not file_data.source_identifiers:
+            return None
+
+        rel_path = file_data.source_identifiers.relative_path
+        if not rel_path:
+            return None
+
+        rel_path = rel_path[1:] if rel_path.startswith("/") else rel_path
+        return self.download_dir / Path(rel_path)
+
+    @staticmethod
+    def is_float(value: str):
+        try:
+            float(value)
+            return True
+        except ValueError:
+            return False
+
+    def generate_download_response(
+        self, file_data: FileData, download_path: Path
+    ) -> DownloadResponse:
+        if (
+            file_data.metadata.date_modified
+            and self.is_float(file_data.metadata.date_modified)
+            and file_data.metadata.date_created
+            and self.is_float(file_data.metadata.date_created)
+        ):
+            date_modified = float(file_data.metadata.date_modified)
+            date_created = float(file_data.metadata.date_created)
+            os.utime(download_path, times=(date_created, date_modified))
+        file_data.local_download_path = str(download_path.resolve())
+        return DownloadResponse(file_data=file_data, path=download_path)
+
+    @property
+    def download_dir(self) -> Path:
+        if self.download_config.download_dir is None:
+            self.download_config.download_dir = (
+                Path.home()
+                / ".cache"
+                / "unstructured"
+                / "ingest"
+                / "download"
+                / self.connector_type
+            ).resolve()
+        return self.download_config.download_dir
+
+    def is_async(self) -> bool:
+        return True
+
+    def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        raise NotImplementedError()
+
+    async def run_async(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        return self.run(file_data=file_data, **kwargs)

unstructured_ingest/interfaces/indexer.py

@@ -0,0 +1,29 @@
+from abc import ABC
+from typing import Any, AsyncGenerator, Generator, Optional, TypeVar
+
+from pydantic import BaseModel
+
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.interfaces.connector import BaseConnector
+from unstructured_ingest.interfaces.process import BaseProcess
+
+
+class IndexerConfig(BaseModel):
+    pass
+
+
+IndexerConfigT = TypeVar("IndexerConfigT", bound=IndexerConfig)
+
+
+class Indexer(BaseProcess, BaseConnector, ABC):
+    connector_type: str
+    index_config: Optional[IndexerConfigT] = None
+
+    def is_async(self) -> bool:
+        return False
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        raise NotImplementedError()
+
+    async def run_async(self, **kwargs: Any) -> AsyncGenerator[FileData, None]:
+        raise NotImplementedError()

unstructured_ingest/interfaces/process.py

@@ -0,0 +1,22 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class BaseProcess(ABC):
+    def is_async(self) -> bool:
+        return False
+
+    def init(self, **kwargs: Any) -> None:
+        pass
+
+    def precheck(self) -> None:
+        pass
+
+    @abstractmethod
+    def run(self, **kwargs: Any) -> Any:
+        pass
+
+    async def run_async(self, **kwargs: Any) -> Any:
+        return self.run(**kwargs)

unstructured_ingest/interfaces/processor.py

@@ -0,0 +1,88 @@
+import os
+from asyncio import Semaphore
+from pathlib import Path
+from typing import Any, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+DEFAULT_WORK_DIR = str((Path.home() / ".cache" / "unstructured" / "ingest" / "pipeline").resolve())
+
+
+class ProcessorConfig(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
+    reprocess: bool = Field(
+        default=False,
+        description="Reprocess a downloaded file even if the relevant structured "
+        "output .json file in output directory already exists.",
+    )
+    verbose: bool = Field(default=False)
+    tqdm: bool = Field(default=False, description="Display tqdm progress bar")
+    work_dir: str = Field(
+        default_factory=lambda: DEFAULT_WORK_DIR,
+        description="Where to place working files when processing each step",
+    )
+    num_processes: int = Field(
+        default=2, description="Number of parallel processes with which to process docs"
+    )
+    max_connections: Optional[int] = Field(
+        default=None, description="Limit of concurrent connectionts"
+    )
+    raise_on_error: bool = Field(
+        default=False,
+        description="Is set, will raise error if any doc in the pipeline fail. "
+        "Otherwise will log error and continue with other docs",
+    )
+    disable_parallelism: bool = Field(
+        default_factory=lambda: os.getenv("INGEST_DISABLE_PARALLELISM", "false").lower() == "true",
+    )
+    preserve_downloads: bool = Field(
+        default=False, description="Don't delete downloaded files after process completes"
+    )
+    download_only: bool = Field(
+        default=False, description="skip the rest of the process after files are downloaded"
+    )
+    re_download: bool = Field(
+        default=False,
+        description="If set, will re-download downloaded files "
+        "regardless of if they already exist locally",
+    )
+    uncompress: bool = Field(
+        default=False,
+        description="Uncompress any archived files. Currently supporting "
+        "zip and tar files based on file extension.",
+    )
+    iter_delete: bool = Field(
+        default=False,
+        description="If limited on memory, this can be enabled to delete "
+        "cached content as it's used and no longer needed in the pipeline.",
+    )
+    delete_cache: bool = Field(
+        default=False,
+        description="If set, will delete the cache work directory when process finishes",
+    )
+
+    # OTEL support
+    otel_endpoint: Optional[str] = Field(
+        default=None, description="OTEL endpoint to publish trace data to"
+    )
+
+    # Used to keep track of state in pipeline
+    status: dict = Field(default_factory=dict)
+    semaphore: Optional[Semaphore] = Field(init=False, default=None, exclude=True)
+
+    def model_post_init(self, __context: Any) -> None:
+        if self.max_connections is not None:
+            self.semaphore = Semaphore(self.max_connections)
+
+    @property
+    def mp_supported(self) -> bool:
+        return not self.disable_parallelism and self.num_processes > 1
+
+    @property
+    def async_supported(self) -> bool:
+        if self.disable_parallelism:
+            return False
+        if self.max_connections is not None and isinstance(self.max_connections, int):
+            return self.max_connections > 1
+        return True