unstructured-ingest 1.2.32__py3-none-any.whl

unstructured_ingest/processes/connectors/fsspec/azure.py

@@ -0,0 +1,203 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from time import time
+from typing import TYPE_CHECKING, Any, Generator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.data_types.file_data import FileDataSourceMetadata
+from unstructured_ingest.error import ProviderError, UserAuthError, UserError, ValueError
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.processes.connectors.fsspec.fsspec import (
+    FsspecAccessConfig,
+    FsspecConnectionConfig,
+    FsspecDownloader,
+    FsspecDownloaderConfig,
+    FsspecIndexer,
+    FsspecIndexerConfig,
+    FsspecUploader,
+    FsspecUploaderConfig,
+)
+from unstructured_ingest.processes.connectors.fsspec.utils import json_serial, sterilize_dict
+from unstructured_ingest.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from adlfs import AzureBlobFileSystem
+
+CONNECTOR_TYPE = "azure"
+
+
+def azure_json_serial(obj):
+    from azure.storage.blob._models import ContentSettings
+
+    if isinstance(obj, ContentSettings):
+        return dict(obj)
+    if isinstance(obj, bytearray):
+        return str(obj)
+    return json_serial(obj)
+
+
+class AzureIndexerConfig(FsspecIndexerConfig):
+    pass
+
+
+class AzureAccessConfig(FsspecAccessConfig):
+    account_name: Optional[str] = Field(
+        default=None,
+        description="The storage account name. This is used to authenticate "
+        "requests signed with an account key and to construct "
+        "the storage endpoint. It is required unless a connection "
+        "string is given, or if a custom domain is used with "
+        "anonymous authentication.",
+    )
+    account_key: Optional[str] = Field(
+        default=None,
+        description="The storage account key. This is used for shared key "
+        "authentication. If any of account key, sas token or "
+        "client_id are not specified, anonymous access will be used.",
+    )
+    connection_string: Optional[str] = Field(
+        default=None,
+        description="If specified, this will override all other parameters. See "
+        "http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ "  # noqa: E501
+        "for the connection string format.",
+    )
+    sas_token: Optional[str] = Field(
+        default=None,
+        description="A shared access signature token to use to authenticate "
+        "requests instead of the account key. If account key and "
+        "sas token are both specified, account key will be used "
+        "to sign. If any of account key, sas token or client_id "
+        "are not specified, anonymous access will be used.",
+    )
+
+    def model_post_init(self, __context: Any) -> None:
+        if self.connection_string is None and self.account_name is None:
+            raise ValueError("either connection_string or account_name must be set")
+
+
+class AzureConnectionConfig(FsspecConnectionConfig):
+    supported_protocols: list[str] = field(default_factory=lambda: ["az"], init=False)
+    access_config: Secret[AzureAccessConfig]
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+
+    def get_access_config(self) -> dict[str, Any]:
+        # Avoid injecting None by filtering out k,v pairs where the value is None
+        access_configs: dict[str, Any] = {
+            k: v for k, v in self.access_config.get_secret_value().model_dump().items() if v
+        }
+        return access_configs
+
+    @requires_dependencies(["adlfs", "fsspec"], extras="azure")
+    @contextmanager
+    def get_client(self, protocol: str) -> Generator["AzureBlobFileSystem", None, None]:
+        with super().get_client(protocol=protocol) as client:
+            yield client
+
+    def wrap_error(self, e: Exception) -> Exception:
+        from azure.core.exceptions import ClientAuthenticationError, HttpResponseError
+
+        if not isinstance(e, HttpResponseError):
+            logger.error(f"unhandled exception from azure ({type(e)}): {e}", exc_info=True)
+            return e
+        if isinstance(e, ClientAuthenticationError):
+            return UserAuthError(e.reason)
+        status_code = e.status_code
+        message = e.reason
+        if status_code is not None:
+            if 400 <= status_code < 500:
+                return UserError(message)
+            if status_code >= 500:
+                return ProviderError(message)
+        logger.error(f"unhandled exception from azure ({type(e)}): {e}", exc_info=True)
+        return e
+
+
+@dataclass
+class AzureIndexer(FsspecIndexer):
+    connection_config: AzureConnectionConfig
+    index_config: AzureIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def sterilize_info(self, file_data: dict) -> dict:
+        return sterilize_dict(data=file_data, default=azure_json_serial)
+
+    def get_metadata(self, file_info: dict) -> FileDataSourceMetadata:
+        path = file_info["name"]
+        date_created = (
+            str(file_info.get("creation_time").timestamp())
+            if "creation_time" in file_info
+            else None
+        )
+        date_modified = (
+            str(file_info.get("last_modified").timestamp())
+            if "last_modified" in file_info
+            else None
+        )
+
+        file_size = file_info.get("size") if "size" in file_info else None
+
+        version = file_info.get("etag")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
+
+class AzureDownloaderConfig(FsspecDownloaderConfig):
+    pass
+
+
+@dataclass
+class AzureDownloader(FsspecDownloader):
+    protocol: str = "az"
+    connection_config: AzureConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+    download_config: Optional[AzureDownloaderConfig] = field(default_factory=AzureDownloaderConfig)
+
+
+class AzureUploaderConfig(FsspecUploaderConfig):
+    pass
+
+
+@dataclass
+class AzureUploader(FsspecUploader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: AzureConnectionConfig
+    upload_config: AzureUploaderConfig = field(default=None)
+
+
+azure_source_entry = SourceRegistryEntry(
+    indexer=AzureIndexer,
+    indexer_config=AzureIndexerConfig,
+    downloader=AzureDownloader,
+    downloader_config=AzureDownloaderConfig,
+    connection_config=AzureConnectionConfig,
+)
+
+azure_destination_entry = DestinationRegistryEntry(
+    uploader=AzureUploader,
+    uploader_config=AzureUploaderConfig,
+    connection_config=AzureConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
+)

unstructured_ingest/processes/connectors/fsspec/box.py

@@ -0,0 +1,176 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from time import time
+from typing import TYPE_CHECKING, Annotated, Any, Generator, Optional
+
+from dateutil import parser
+from pydantic import Field, Secret
+from pydantic.functional_validators import BeforeValidator
+
+from unstructured_ingest.data_types.file_data import FileDataSourceMetadata
+from unstructured_ingest.error import ProviderError, UserAuthError, UserError
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.processes.connectors.fsspec.fsspec import (
+    FsspecAccessConfig,
+    FsspecConnectionConfig,
+    FsspecDownloader,
+    FsspecDownloaderConfig,
+    FsspecIndexer,
+    FsspecIndexerConfig,
+    FsspecUploader,
+    FsspecUploaderConfig,
+)
+from unstructured_ingest.processes.connectors.utils import conform_string_to_dict
+from unstructured_ingest.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from boxfs import BoxFileSystem
+
+CONNECTOR_TYPE = "box"
+
+
+class BoxIndexerConfig(FsspecIndexerConfig):
+    pass
+
+
+class BoxAccessConfig(FsspecAccessConfig):
+    box_app_config: Annotated[dict, BeforeValidator(conform_string_to_dict)] = Field(
+        description="Box app credentials as a JSON string."
+    )
+
+
+class BoxConnectionConfig(FsspecConnectionConfig):
+    supported_protocols: list[str] = field(default_factory=lambda: ["box"], init=False)
+    access_config: Secret[BoxAccessConfig]
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+
+    def get_access_config(self) -> dict[str, Any]:
+        from boxsdk import JWTAuth
+
+        ac = self.access_config.get_secret_value()
+        settings_dict = ac.box_app_config
+
+        # Create and authenticate the JWTAuth object
+        oauth = JWTAuth.from_settings_dictionary(settings_dict)
+        oauth.authenticate_instance()
+
+        # if not oauth.access_token:
+        #     raise SourceConnectionError("Authentication failed: No access token generated.")
+
+        # Prepare the access configuration with the authenticated oauth
+        access_kwargs_with_oauth: dict[str, Any] = {
+            "oauth": oauth,
+        }
+        access_config: dict[str, Any] = ac.model_dump()
+        access_config.pop("box_app_config", None)
+        access_kwargs_with_oauth.update(access_config)
+
+        return access_kwargs_with_oauth
+
+    def wrap_error(self, e: Exception) -> Exception:
+        from boxsdk.exception import BoxAPIException, BoxOAuthException
+
+        if isinstance(e, BoxOAuthException):
+            return UserAuthError(e.message)
+        if not isinstance(e, BoxAPIException):
+            logger.error(f"unhandled exception from box ({type(e)}): {e}", exc_info=True)
+            return e
+        message = e.message or e
+        if error_code_status := e.status:
+            if 400 <= error_code_status < 500:
+                return UserError(message)
+            if error_code_status >= 500:
+                return ProviderError(message)
+
+        logger.error(f"unhandled exception from box ({type(e)}): {e}", exc_info=True)
+        return e
+
+    @requires_dependencies(["boxfs"], extras="box")
+    @contextmanager
+    def get_client(self, protocol: str) -> Generator["BoxFileSystem", None, None]:
+        with super().get_client(protocol=protocol) as client:
+            yield client
+
+
+@dataclass
+class BoxIndexer(FsspecIndexer):
+    connection_config: BoxConnectionConfig
+    index_config: BoxIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def get_metadata(self, file_info: dict) -> FileDataSourceMetadata:
+        path = file_info["name"]
+        date_created = None
+        date_modified = None
+        if modified_at_str := file_info.get("modified_at"):
+            date_modified = str(parser.parse(modified_at_str).timestamp())
+        if created_at_str := file_info.get("created_at"):
+            date_created = str(parser.parse(created_at_str).timestamp())
+
+        file_size = file_info.get("size") if "size" in file_info else None
+
+        version = file_info.get("id")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+            "file_id": file_info.get("id"),
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
+
+class BoxDownloaderConfig(FsspecDownloaderConfig):
+    pass
+
+
+@dataclass
+class BoxDownloader(FsspecDownloader):
+    protocol: str = "box"
+    connection_config: BoxConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+    download_config: Optional[BoxDownloaderConfig] = field(default_factory=BoxDownloaderConfig)
+
+
+class BoxUploaderConfig(FsspecUploaderConfig):
+    pass
+
+
+@dataclass
+class BoxUploader(FsspecUploader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: BoxConnectionConfig
+    upload_config: BoxUploaderConfig = field(default=None)
+
+
+box_source_entry = SourceRegistryEntry(
+    indexer=BoxIndexer,
+    indexer_config=BoxIndexerConfig,
+    downloader=BoxDownloader,
+    downloader_config=BoxDownloaderConfig,
+    connection_config=BoxConnectionConfig,
+)
+
+box_destination_entry = DestinationRegistryEntry(
+    uploader=BoxUploader,
+    uploader_config=BoxUploaderConfig,
+    connection_config=BoxConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
+)

unstructured_ingest/processes/connectors/fsspec/dropbox.py

@@ -0,0 +1,238 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from time import time
+from typing import TYPE_CHECKING, Any, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.data_types.file_data import FileDataSourceMetadata
+from unstructured_ingest.error import (
+    ProviderError,
+    UserAuthError,
+    UserError,
+    ValueError,
+)
+from unstructured_ingest.error import (
+    RateLimitError as CustomRateLimitError,
+)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.processes.connectors.fsspec.fsspec import (
+    FsspecAccessConfig,
+    FsspecConnectionConfig,
+    FsspecDownloader,
+    FsspecDownloaderConfig,
+    FsspecIndexer,
+    FsspecIndexerConfig,
+    FsspecUploader,
+    FsspecUploaderConfig,
+)
+from unstructured_ingest.processes.utils.blob_storage import (
+    BlobStoreUploadStager,
+    BlobStoreUploadStagerConfig,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    pass
+
+CONNECTOR_TYPE = "dropbox"
+
+
+class DropboxIndexerConfig(FsspecIndexerConfig):
+    def model_post_init(self, __context):
+        if not self.path_without_protocol.startswith("/"):
+            self.path_without_protocol = "/" + self.path_without_protocol
+
+
+class DropboxAccessConfig(FsspecAccessConfig):
+    token: Optional[str] = Field(
+        default=None, description="Dropbox access token."
+    )  # This is the short lived (4h) token that needs to be generated anew each time.
+    app_key: Optional[str] = Field(default=None, description="Dropbox app key.")
+    app_secret: Optional[str] = Field(default=None, description="Dropbox app secret.")
+    refresh_token: Optional[str] = Field(
+        default=None, description="Dropbox refresh token."
+    )  # This is the long lived token that doesn't expire
+
+
+class DropboxConnectionConfig(FsspecConnectionConfig):
+    access_config: Secret[DropboxAccessConfig] = Field(
+        default=DropboxAccessConfig(), validate_default=True
+    )
+    connector_type: str = Field(default=CONNECTOR_TYPE)
+
+    @requires_dependencies(["dropbox"])
+    def get_dropbox_access_token_from_refresh(
+        self,
+        refresh_token: str,
+        app_key: str,
+        app_secret: str,
+    ) -> str:
+        """
+        Uses the Dropbox Python SDK to exchange a long-lived refresh token for an access token.
+        """
+        import dropbox
+
+        dbx = dropbox.Dropbox(
+            oauth2_access_token=None,
+            oauth2_refresh_token=refresh_token,
+            app_key=app_key,
+            app_secret=app_secret,
+        )
+
+        # This call fetches a new short-lived token and auto-updates dbx._oauth2_access_token
+        dbx.check_and_refresh_access_token()
+        short_lived_token = dbx._oauth2_access_token  # Private attr, but standard usage
+        return short_lived_token
+
+    def get_access_config(self) -> dict[str, Any]:
+        """
+        Overrides the parent FsspecConnectionConfig.get_access_config() to ensure
+        that we always provide an access token if refresh credentials exist.
+        """
+        base_conf = super().get_access_config()
+
+        refresh_token = base_conf.get("refresh_token")
+        app_key = base_conf.get("app_key")
+        app_secret = base_conf.get("app_secret")
+
+        # Standard scenario - we have refresh a token and creds provided
+        # which we're going to use to retrieve access token
+        if refresh_token and app_key and app_secret:
+            logger.debug("Attempting to generate access token from refresh token...")
+            new_token = self.get_dropbox_access_token_from_refresh(
+                refresh_token=refresh_token,
+                app_key=app_key,
+                app_secret=app_secret,
+            )
+            if not new_token:
+                raise ValueError(
+                    "Unable to retrieve an access token from Dropbox. "
+                    "Please check that your refresh token, app key, and secret are valid."
+                )
+            base_conf["token"] = new_token
+        elif not base_conf.get("token"):  # we might already have an access token from outside
+            # We have neither an existing short?lived token nor refresh credentials
+            raise ValueError(
+                "No valid token or refresh_token with app credentials was found. "
+                "Please check that your refresh token, app key, and secret are valid "
+                "or provide a valid short-lived token"
+            )
+
+        return base_conf
+
+    @requires_dependencies(["dropbox"])
+    def wrap_error(self, e: Exception) -> Exception:
+        from dropbox.exceptions import AuthError, HttpError, RateLimitError
+
+        if not isinstance(e, HttpError):
+            logger.error(f"Unhandled Dropbox exception: {repr(e)}", exc_info=True)
+            return e
+
+        if isinstance(e, AuthError):
+            raise UserAuthError(e.error)
+        elif isinstance(e, RateLimitError):
+            return CustomRateLimitError(e.error)
+
+        status_code = e.status_code
+        if 400 <= status_code < 500:
+            if body := getattr(e, "body", None):
+                return UserError(body)
+            else:
+                return UserError(e.body)
+        if status_code >= 500:
+            if body := getattr(e, "body", None):
+                return ProviderError(body)
+            else:
+                return ProviderError(e.body)
+
+        logger.error(f"Unhandled Dropbox HttpError: {repr(e)}", exc_info=True)
+        return e
+
+
+@dataclass
+class DropboxIndexer(FsspecIndexer):
+    connection_config: DropboxConnectionConfig
+    index_config: DropboxIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def get_path(self, file_info: dict) -> str:
+        return file_info["name"]
+
+    def get_metadata(self, file_info: dict) -> FileDataSourceMetadata:
+        path = file_info["name"].lstrip("/")
+        date_created = None
+        date_modified = None
+        server_modified = file_info.get("server_modified")
+        client_modified = file_info.get("client_modified")
+        if server_modified and client_modified and server_modified > client_modified:
+            date_created = str(client_modified.timestamp())
+            date_modified = str(server_modified.timestamp())
+        elif server_modified and client_modified and server_modified < client_modified:
+            date_created = str(server_modified.timestamp())
+            date_modified = str(client_modified.timestamp())
+
+        file_size = file_info.get("size") if "size" in file_info else None
+
+        version = file_info.get("content_hash")
+        record_locator = {
+            "protocol": self.index_config.protocol,
+            "remote_file_path": self.index_config.remote_url,
+            "file_id": file_info.get("id"),
+        }
+        return FileDataSourceMetadata(
+            date_created=date_created,
+            date_modified=date_modified,
+            date_processed=str(time()),
+            version=version,
+            url=f"{self.index_config.protocol}://{path}",
+            record_locator=record_locator,
+            filesize_bytes=file_size,
+        )
+
+
+class DropboxDownloaderConfig(FsspecDownloaderConfig):
+    pass
+
+
+@dataclass
+class DropboxDownloader(FsspecDownloader):
+    protocol: str = "dropbox"
+    connection_config: DropboxConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+    download_config: Optional[DropboxDownloaderConfig] = field(
+        default_factory=DropboxDownloaderConfig
+    )
+
+
+class DropboxUploaderConfig(FsspecUploaderConfig):
+    pass
+
+
+@dataclass
+class DropboxUploader(FsspecUploader):
+    connector_type: str = CONNECTOR_TYPE
+    connection_config: DropboxConnectionConfig
+    upload_config: DropboxUploaderConfig = field(default=None)
+
+
+dropbox_source_entry = SourceRegistryEntry(
+    indexer=DropboxIndexer,
+    indexer_config=DropboxIndexerConfig,
+    downloader=DropboxDownloader,
+    downloader_config=DropboxDownloaderConfig,
+    connection_config=DropboxConnectionConfig,
+)
+
+dropbox_destination_entry = DestinationRegistryEntry(
+    uploader=DropboxUploader,
+    uploader_config=DropboxUploaderConfig,
+    connection_config=DropboxConnectionConfig,
+    upload_stager_config=BlobStoreUploadStagerConfig,
+    upload_stager=BlobStoreUploadStager,
+)