unstructured-ingest 0.3.13__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/fsspec/__init__.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from unstructured_ingest.v2.processes.connector_registry import (
+    add_destination_entry,
+    add_source_entry,
+)
+
+from .azure import CONNECTOR_TYPE as AZURE_CONNECTOR_TYPE
+from .azure import azure_destination_entry, azure_source_entry
+from .box import CONNECTOR_TYPE as BOX_CONNECTOR_TYPE
+from .box import box_destination_entry, box_source_entry
+from .dropbox import CONNECTOR_TYPE as DROPBOX_CONNECTOR_TYPE
+from .dropbox import dropbox_destination_entry, dropbox_source_entry
+from .gcs import CONNECTOR_TYPE as GCS_CONNECTOR_TYPE
+from .gcs import gcs_destination_entry, gcs_source_entry
+from .s3 import CONNECTOR_TYPE as S3_CONNECTOR_TYPE
+from .s3 import s3_destination_entry, s3_source_entry
+from .sftp import CONNECTOR_TYPE as SFTP_CONNECTOR_TYPE
+from .sftp import sftp_destination_entry, sftp_source_entry
+
+add_source_entry(source_type=AZURE_CONNECTOR_TYPE, entry=azure_source_entry)
+add_destination_entry(destination_type=AZURE_CONNECTOR_TYPE, entry=azure_destination_entry)
+
+add_source_entry(source_type=BOX_CONNECTOR_TYPE, entry=box_source_entry)
+add_destination_entry(destination_type=BOX_CONNECTOR_TYPE, entry=box_destination_entry)
+
+add_source_entry(source_type=DROPBOX_CONNECTOR_TYPE, entry=dropbox_source_entry)
+add_destination_entry(destination_type=DROPBOX_CONNECTOR_TYPE, entry=dropbox_destination_entry)
+
+add_source_entry(source_type=GCS_CONNECTOR_TYPE, entry=gcs_source_entry)
+add_destination_entry(destination_type=GCS_CONNECTOR_TYPE, entry=gcs_destination_entry)
+
+add_source_entry(source_type=S3_CONNECTOR_TYPE, entry=s3_source_entry)
+add_destination_entry(destination_type=S3_CONNECTOR_TYPE, entry=s3_destination_entry)
+
+add_source_entry(source_type=SFTP_CONNECTOR_TYPE, entry=sftp_source_entry)
+add_destination_entry(destination_type=SFTP_CONNECTOR_TYPE, entry=sftp_destination_entry)

unstructured_ingest/v2/processes/connectors/fsspec/azure.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from time import time
+from typing import TYPE_CHECKING, Any, Generator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserError
+from unstructured_ingest.v2.interfaces import FileDataSourceMetadata
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
+    FsspecAccessConfig,
+    FsspecConnectionConfig,
+    FsspecDownloader,
+    FsspecDownloaderConfig,
+    FsspecIndexer,
+    FsspecIndexerConfig,
+    FsspecUploader,
+    FsspecUploaderConfig,
+)
+from unstructured_ingest.v2.processes.connectors.fsspec.utils import json_serial, sterilize_dict
+
+if TYPE_CHECKING:
+    from adlfs import AzureBlobFileSystem
+
+CONNECTOR_TYPE = "azure"
+
+
+def azure_json_serial(obj):
+    from azure.storage.blob._models import ContentSettings
+
+    if isinstance(obj, ContentSettings):
+        return dict(obj)
+    if isinstance(obj, bytearray):
+        return str(obj)
+    return json_serial(obj)
+
+
+class AzureIndexerConfig(FsspecIndexerConfig):
+    pass
+
+
+class AzureAccessConfig(FsspecAccessConfig):
+    account_name: Optional[str] = Field(
+        default=None,
+        description="The storage account name. This is used to authenticate "
+        "requests signed with an account key and to construct "
+        "the storage endpoint. It is required unless a connection "
+        "string is given, or if a custom domain is used with "
+        "anonymous authentication.",
+    )
+    account_key: Optional[str] = Field(
+        default=None,
+        description="The storage account key. This is used for shared key "
+        "authentication. If any of account key, sas token or "
+        "client_id are not specified, anonymous access will be used.",
+    )
+    connection_string: Optional[str] = Field(
+        default=None,
+        description="If specified, this will override all other parameters. See "
+        "http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ "  # noqa: E501
+        "for the connection string format.",
+    )
+    sas_token: Optional[str] = Field(
+        default=None,
+        description="A shared access signature token to use to authenticate "
+        "requests instead of the account key. If account key and "
+        "sas token are both specified, account key will be used "
+        "to sign. If any of account key, sas token or client_id "
+        "are not specified, anonymous access will be used.",
+    )
+
+    def model_post_init(self, __context: Any) -> None:
+        if self.connection_string is None and self.account_name is None:
+            raise ValueError("either connection_string or account_name must be set")
+
+
+class AzureConnectionConfig(FsspecConnectionConfig):
+    supported_protocols: list[str] = field(default_factory=lambda: ["az"], init=False)
+    access_config: Secret[AzureAccessConfig]
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+
+    def get_access_config(self) -> dict[str, Any]:
+        # Avoid injecting None by filtering out k,v pairs where the value is None
+        access_configs: dict[str, Any] = {
+            k: v for k, v in self.access_config.get_secret_value().model_dump().items() if v
+        }
+        return access_configs
+
+    @requires_dependencies(["adlfs", "fsspec"], extras="azure")
+    @contextmanager
+    def get_client(self, protocol: str) -> Generator["AzureBlobFileSystem", None, None]:
+        with super().get_client(protocol=protocol) as client:
+            yield client
+
+    def wrap_error(self, e: Exception) -> Exception:
+        from azure.core.exceptions import ClientAuthenticationError, HttpResponseError
+
+        if not isinstance(e, HttpResponseError):
+            logger.error(f"unhandled exception from azure ({type(e)}): {e}", exc_info=True)
+            return e
+        if isinstance(e, ClientAuthenticationError):
+            return UserAuthError(e.reason)
+        status_code = e.status_code
+        message = e.reason
+        if status_code is not None:
+            if 400 <= status_code < 500:
+                return UserError(message)
+            if status_code >= 500:
+                return ProviderError(message)
+        logger.error(f"unhandled exception from azure ({type(e)}): {e}", exc_info=True)
+        return e
+
+
+@dataclass
+class AzureIndexer(FsspecIndexer):
+    connection_config: AzureConnectionConfig
+    index_config: AzureIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def sterilize_info(self, file_data: dict) -> dict:
+        return sterilize_dict(data=file_data, default=azure_json_serial)
+
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"]
+        date_created = (
+            str(file_data.get("creation_time").timestamp())
+            if "creation_time" in file_data
+            else None
+        )
+        date_modified = (
+            str(file_data.get("last_modified").timestamp())
+            if "last_modified" in file_data
+            else None
+        )
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        version = file_data.get("etag")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
+
+class AzureDownloaderConfig(FsspecDownloaderConfig):
+    pass
+
+
+@dataclass
+class AzureDownloader(FsspecDownloader):
+    protocol: str = "az"
+    connection_config: AzureConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+    download_config: Optional[AzureDownloaderConfig] = field(default_factory=AzureDownloaderConfig)
+
+
+class AzureUploaderConfig(FsspecUploaderConfig):
+    pass
+
+
+@dataclass
+class AzureUploader(FsspecUploader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: AzureConnectionConfig
+    upload_config: AzureUploaderConfig = field(default=None)
+
+
+azure_source_entry = SourceRegistryEntry(
+    indexer=AzureIndexer,
+    indexer_config=AzureIndexerConfig,
+    downloader=AzureDownloader,
+    downloader_config=AzureDownloaderConfig,
+    connection_config=AzureConnectionConfig,
+)
+
+azure_destination_entry = DestinationRegistryEntry(
+    uploader=AzureUploader,
+    uploader_config=AzureUploaderConfig,
+    connection_config=AzureConnectionConfig,
+)

unstructured_ingest/v2/processes/connectors/fsspec/box.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from time import time
+from typing import TYPE_CHECKING, Annotated, Any, Generator, Optional
+
+from dateutil import parser
+from pydantic import Field, Secret
+from pydantic.functional_validators import BeforeValidator
+
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.errors import ProviderError, UserAuthError, UserError
+from unstructured_ingest.v2.interfaces import FileDataSourceMetadata
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
+    FsspecAccessConfig,
+    FsspecConnectionConfig,
+    FsspecDownloader,
+    FsspecDownloaderConfig,
+    FsspecIndexer,
+    FsspecIndexerConfig,
+    FsspecUploader,
+    FsspecUploaderConfig,
+)
+from unstructured_ingest.v2.processes.connectors.utils import conform_string_to_dict
+
+if TYPE_CHECKING:
+    from boxfs import BoxFileSystem
+
+CONNECTOR_TYPE = "box"
+
+
+class BoxIndexerConfig(FsspecIndexerConfig):
+    pass
+
+
+class BoxAccessConfig(FsspecAccessConfig):
+    box_app_config: Annotated[dict, BeforeValidator(conform_string_to_dict)] = Field(
+        description="Box app credentials as a JSON string."
+    )
+
+
+class BoxConnectionConfig(FsspecConnectionConfig):
+    supported_protocols: list[str] = field(default_factory=lambda: ["box"], init=False)
+    access_config: Secret[BoxAccessConfig]
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+
+    def get_access_config(self) -> dict[str, Any]:
+        from boxsdk import JWTAuth
+
+        ac = self.access_config.get_secret_value()
+        settings_dict = ac.box_app_config
+
+        # Create and authenticate the JWTAuth object
+        oauth = JWTAuth.from_settings_dictionary(settings_dict)
+        oauth.authenticate_instance()
+
+        # if not oauth.access_token:
+        #     raise SourceConnectionError("Authentication failed: No access token generated.")
+
+        # Prepare the access configuration with the authenticated oauth
+        access_kwargs_with_oauth: dict[str, Any] = {
+            "oauth": oauth,
+        }
+        access_config: dict[str, Any] = ac.model_dump()
+        access_config.pop("box_app_config", None)
+        access_kwargs_with_oauth.update(access_config)
+
+        return access_kwargs_with_oauth
+
+    def wrap_error(self, e: Exception) -> Exception:
+        from boxsdk.exception import BoxAPIException, BoxOAuthException
+
+        if isinstance(e, BoxOAuthException):
+            return UserAuthError(e.message)
+        if not isinstance(e, BoxAPIException):
+            logger.error(f"unhandled exception from box ({type(e)}): {e}", exc_info=True)
+            return e
+        message = e.message or e
+        if error_code_status := e.status:
+            if 400 <= error_code_status < 500:
+                return UserError(message)
+            if error_code_status >= 500:
+                return ProviderError(message)
+
+        logger.error(f"unhandled exception from box ({type(e)}): {e}", exc_info=True)
+        return e
+
+    @requires_dependencies(["boxfs"], extras="box")
+    @contextmanager
+    def get_client(self, protocol: str) -> Generator["BoxFileSystem", None, None]:
+        with super().get_client(protocol=protocol) as client:
+            yield client
+
+
+@dataclass
+class BoxIndexer(FsspecIndexer):
+    connection_config: BoxConnectionConfig
+    index_config: BoxIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"]
+        date_created = None
+        date_modified = None
+        if modified_at_str := file_data.get("modified_at"):
+            date_modified = str(parser.parse(modified_at_str).timestamp())
+        if created_at_str := file_data.get("created_at"):
+            date_created = str(parser.parse(created_at_str).timestamp())
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        version = file_data.get("id")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+            "file_id": file_data.get("id"),
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
+
+class BoxDownloaderConfig(FsspecDownloaderConfig):
+    pass
+
+
+@dataclass
+class BoxDownloader(FsspecDownloader):
+    protocol: str = "box"
+    connection_config: BoxConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+    download_config: Optional[BoxDownloaderConfig] = field(default_factory=BoxDownloaderConfig)
+
+
+class BoxUploaderConfig(FsspecUploaderConfig):
+    pass
+
+
+@dataclass
+class BoxUploader(FsspecUploader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: BoxConnectionConfig
+    upload_config: BoxUploaderConfig = field(default=None)
+
+
+box_source_entry = SourceRegistryEntry(
+    indexer=BoxIndexer,
+    indexer_config=BoxIndexerConfig,
+    downloader=BoxDownloader,
+    downloader_config=BoxDownloaderConfig,
+    connection_config=BoxConnectionConfig,
+)
+
+box_destination_entry = DestinationRegistryEntry(
+    uploader=BoxUploader,
+    uploader_config=BoxUploaderConfig,
+    connection_config=BoxConnectionConfig,
+)

unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from time import time
+from typing import TYPE_CHECKING, Generator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.errors import (
+    ProviderError,
+    UserAuthError,
+    UserError,
+)
+from unstructured_ingest.v2.errors import (
+    RateLimitError as CustomRateLimitError,
+)
+from unstructured_ingest.v2.interfaces import FileDataSourceMetadata
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
+    FsspecAccessConfig,
+    FsspecConnectionConfig,
+    FsspecDownloader,
+    FsspecDownloaderConfig,
+    FsspecIndexer,
+    FsspecIndexerConfig,
+    FsspecUploader,
+    FsspecUploaderConfig,
+)
+
+if TYPE_CHECKING:
+    from dropboxdrivefs import DropboxDriveFileSystem
+
+CONNECTOR_TYPE = "dropbox"
+
+
+class DropboxIndexerConfig(FsspecIndexerConfig):
+    def model_post_init(self, __context):
+        if not self.path_without_protocol.startswith("/"):
+            self.path_without_protocol = "/" + self.path_without_protocol
+
+
+class DropboxAccessConfig(FsspecAccessConfig):
+    token: Optional[str] = Field(default=None, description="Dropbox access token.")
+
+
+class DropboxConnectionConfig(FsspecConnectionConfig):
+    supported_protocols: list[str] = field(default_factory=lambda: ["dropbox"], init=False)
+    access_config: Secret[DropboxAccessConfig] = Field(
+        default=DropboxAccessConfig(), validate_default=True
+    )
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+
+    @requires_dependencies(["dropboxdrivefs", "fsspec"], extras="dropbox")
+    @contextmanager
+    def get_client(self, protocol: str) -> Generator["DropboxDriveFileSystem", None, None]:
+        with super().get_client(protocol=protocol) as client:
+            yield client
+
+    def wrap_error(self, e: Exception) -> Exception:
+        from dropbox.exceptions import AuthError, HttpError, RateLimitError
+
+        if not isinstance(e, HttpError):
+            logger.error(f"unhandled exception from dropbox ({type(e)}): {e}", exc_info=True)
+            return e
+        if isinstance(e, AuthError):
+            raise UserAuthError(e.error)
+        if isinstance(e, RateLimitError):
+            return CustomRateLimitError(e.error)
+        status_code = e.status_code
+        if 400 <= status_code < 500:
+            if body := getattr(e, "body", None):
+                return UserError(body)
+            else:
+                return UserError(e.body)
+        if status_code >= 500:
+            if body := getattr(e, "body", None):
+                return ProviderError(body)
+            else:
+                return ProviderError(e.body)
+        logger.error(f"unhandled exception from dropbox ({type(e)}): {e}", exc_info=True)
+        return e
+
+
+@dataclass
+class DropboxIndexer(FsspecIndexer):
+    connection_config: DropboxConnectionConfig
+    index_config: DropboxIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def get_path(self, file_data: dict) -> str:
+        return file_data["name"]
+
+    def get_metadata(self, file_data: dict) -> FileDataSourceMetadata:
+        path = file_data["name"].lstrip("/")
+        date_created = None
+        date_modified = None
+        server_modified = file_data.get("server_modified")
+        client_modified = file_data.get("client_modified")
+        if server_modified and client_modified and server_modified > client_modified:
+            date_created = str(client_modified.timestamp())
+            date_modified = str(server_modified.timestamp())
+        elif server_modified and client_modified and server_modified < client_modified:
+            date_created = str(server_modified.timestamp())
+            date_modified = str(client_modified.timestamp())
+
+        file_size = file_data.get("size") if "size" in file_data else None
+
+        version = file_data.get("content_hash")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+            "file_id": file_data.get("id"),
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
+
+class DropboxDownloaderConfig(FsspecDownloaderConfig):
+    pass
+
+
+@dataclass
+class DropboxDownloader(FsspecDownloader):
+    protocol: str = "dropbox"
+    connection_config: DropboxConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+    download_config: Optional[DropboxDownloaderConfig] = field(
+        default_factory=DropboxDownloaderConfig
+    )
+
+
+class DropboxUploaderConfig(FsspecUploaderConfig):
+    pass
+
+
+@dataclass
+class DropboxUploader(FsspecUploader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: DropboxConnectionConfig
+    upload_config: DropboxUploaderConfig = field(default=None)
+
+
+dropbox_source_entry = SourceRegistryEntry(
+    indexer=DropboxIndexer,
+    indexer_config=DropboxIndexerConfig,
+    downloader=DropboxDownloader,
+    downloader_config=DropboxDownloaderConfig,
+    connection_config=DropboxConnectionConfig,
+)
+
+dropbox_destination_entry = DestinationRegistryEntry(
+    uploader=DropboxUploader,
+    uploader_config=DropboxUploaderConfig,
+    connection_config=DropboxConnectionConfig,
+)