unstructured-ingest 0.0.2.dev0__py3-none-any.whl → 0.0.4__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/fsspec/azure.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Generator, Optional
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import Field, Secret
+
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
 from unstructured_ingest.v2.processes.connector_registry import (
@@ -36,35 +37,59 @@ def azure_json_serial(obj):
     return json_serial(obj)
 
 
-@dataclass
 class AzureIndexerConfig(FsspecIndexerConfig):
     pass
 
 
-@dataclass
 class AzureAccessConfig(FsspecAccessConfig):
-    account_name: Optional[str] = None
-    account_key: Optional[str] = None
-    connection_string: Optional[str] = None
-    sas_token: Optional[str] = None
+    account_name: Optional[str] = Field(
+        default=None,
+        description="The storage account name. This is used to authenticate "
+        "requests signed with an account key and to construct "
+        "the storage endpoint. It is required unless a connection "
+        "string is given, or if a custom domain is used with "
+        "anonymous authentication.",
+    )
+    account_key: Optional[str] = Field(
+        default=None,
+        description="The storage account key. This is used for shared key "
+        "authentication. If any of account key, sas token or "
+        "client_id are not specified, anonymous access will be used.",
+    )
+    connection_string: Optional[str] = Field(
+        default=None,
+        description="If specified, this will override all other parameters. See "
+        "http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ "  # noqa: E501
+        "for the connection string format.",
+    )
+    sas_token: Optional[str] = Field(
+        default=None,
+        description="A shared access signature token to use to authenticate "
+        "requests instead of the account key. If account key and "
+        "sas token are both specified, account key will be used "
+        "to sign. If any of account key, sas token or client_id "
+        "are not specified, anonymous access will be used.",
+    )
 
-    def __post_init__(self):
+    def model_post_init(self, __context: Any) -> None:
         if self.connection_string is None and self.account_name is None:
             raise ValueError("either connection_string or account_name must be set")
 
 
-@dataclass
+SecretAzureAccessConfig = Secret[AzureAccessConfig]
+
+
 class AzureConnectionConfig(FsspecConnectionConfig):
-    supported_protocols: list[str] = field(default_factory=lambda: ["az"])
-    access_config: AzureAccessConfig = enhanced_field(
-        sensitive=True, default_factory=lambda: AzureAccessConfig()
+    supported_protocols: list[str] = field(default_factory=lambda: ["az"], init=False)
+    access_config: SecretAzureAccessConfig = Field(
+        default_factory=lambda: SecretAzureAccessConfig(secret_value=AzureAccessConfig())
     )
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     def get_access_config(self) -> dict[str, Any]:
         # Avoid injecting None by filtering out k,v pairs where the value is None
         access_configs: dict[str, Any] = {
-            k: v for k, v in self.access_config.to_dict().items() if v
+            k: v for k, v in self.access_config.get_secret_value().dict().items() if v
         }
         return access_configs
 
@@ -75,6 +100,10 @@ class AzureIndexer(FsspecIndexer):
     index_config: AzureIndexerConfig
     connector_type: str = CONNECTOR_TYPE
 
+    @requires_dependencies(["adlfs", "fsspec"], extras="azure")
+    def precheck(self) -> None:
+        super().precheck()
+
     def sterilize_info(self, path) -> dict:
         info = self.fs.info(path=path)
         return sterilize_dict(data=info, default=azure_json_serial)
@@ -84,7 +113,6 @@ class AzureIndexer(FsspecIndexer):
         return super().run(**kwargs)
 
 
-@dataclass
 class AzureDownloaderConfig(FsspecDownloaderConfig):
     pass
 
@@ -105,7 +133,6 @@ class AzureDownloader(FsspecDownloader):
         return await super().run_async(file_data=file_data, **kwargs)
 
 
-@dataclass
 class AzureUploaderConfig(FsspecUploaderConfig):
     pass
 
@@ -120,6 +147,10 @@ class AzureUploader(FsspecUploader):
     def __post_init__(self):
         super().__post_init__()
 
+    @requires_dependencies(["adlfs", "fsspec"], extras="azure")
+    def precheck(self) -> None:
+        super().precheck()
+
     @requires_dependencies(["adlfs", "fsspec"], extras="azure")
     def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
         return super().run(contents=contents, **kwargs)

unstructured_ingest/v2/processes/connectors/fsspec/box.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Generator, Optional
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import Field, Secret
+
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
 from unstructured_ingest.v2.processes.connector_registry import (
@@ -25,35 +26,38 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
 CONNECTOR_TYPE = "box"
 
 
-@dataclass
 class BoxIndexerConfig(FsspecIndexerConfig):
     pass
 
 
-@dataclass
 class BoxAccessConfig(FsspecAccessConfig):
-    box_app_config: Optional[str] = None
+    box_app_config: Optional[str] = Field(
+        default=None, description="Path to Box app credentials as json file."
+    )
+
+
+SecretBoxAccessConfig = Secret[BoxAccessConfig]
 
 
-@dataclass
 class BoxConnectionConfig(FsspecConnectionConfig):
-    supported_protocols: list[str] = field(default_factory=lambda: ["box"])
-    access_config: BoxAccessConfig = enhanced_field(
-        sensitive=True, default_factory=lambda: BoxAccessConfig()
+    supported_protocols: list[str] = field(default_factory=lambda: ["box"], init=False)
+    access_config: SecretBoxAccessConfig = Field(
+        default_factory=lambda: SecretBoxAccessConfig(secret_value=BoxAccessConfig())
     )
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     def get_access_config(self) -> dict[str, Any]:
         # Return access_kwargs with oauth. The oauth object can not be stored directly in the config
         # because it is not serializable.
         from boxsdk import JWTAuth
 
+        ac = self.access_config.get_secret_value()
         access_kwargs_with_oauth: dict[str, Any] = {
             "oauth": JWTAuth.from_settings_file(
-                self.access_config.box_app_config,
+                ac.box_app_config,
             ),
         }
-        access_config: dict[str, Any] = self.access_config.to_dict()
+        access_config: dict[str, Any] = ac.dict()
         access_config.pop("box_app_config", None)
         access_kwargs_with_oauth.update(access_config)
 
@@ -70,8 +74,11 @@ class BoxIndexer(FsspecIndexer):
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         return super().run(**kwargs)
 
+    @requires_dependencies(["boxfs"], extras="box")
+    def precheck(self) -> None:
+        super().precheck()
+
 
-@dataclass
 class BoxDownloaderConfig(FsspecDownloaderConfig):
     pass
 
@@ -92,7 +99,6 @@ class BoxDownloader(FsspecDownloader):
         return await super().run_async(file_data=file_data, **kwargs)
 
 
-@dataclass
 class BoxUploaderConfig(FsspecUploaderConfig):
     pass
 
@@ -107,6 +113,10 @@ class BoxUploader(FsspecUploader):
     def __post_init__(self):
         super().__post_init__()
 
+    @requires_dependencies(["boxfs"], extras="box")
+    def precheck(self) -> None:
+        super().precheck()
+
     @requires_dependencies(["boxfs"], extras="box")
     def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
         return super().run(contents=contents, **kwargs)

unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Generator, Optional
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import Field, Secret
+
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
 from unstructured_ingest.v2.processes.connector_registry import (
@@ -26,23 +27,23 @@ from unstructured_ingest.v2.processes.connectors.fsspec.utils import sterilize_d
 CONNECTOR_TYPE = "dropbox"
 
 
-@dataclass
 class DropboxIndexerConfig(FsspecIndexerConfig):
     pass
 
 
-@dataclass
 class DropboxAccessConfig(FsspecAccessConfig):
-    token: Optional[str] = None
+    token: Optional[str] = Field(default=None, description="Dropbox access token.")
+
+
+SecretDropboxAccessConfig = Secret[DropboxAccessConfig]
 
 
-@dataclass
 class DropboxConnectionConfig(FsspecConnectionConfig):
-    supported_protocols: list[str] = field(default_factory=lambda: ["dropbox"])
-    access_config: DropboxAccessConfig = enhanced_field(
-        sensitive=True, default_factory=lambda: DropboxAccessConfig()
+    supported_protocols: list[str] = field(default_factory=lambda: ["dropbox"], init=False)
+    access_config: SecretDropboxAccessConfig = Field(
+        default_factory=lambda: SecretDropboxAccessConfig(secret_value=DropboxAccessConfig())
     )
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
 
 @dataclass
@@ -57,6 +58,10 @@ class DropboxIndexer(FsspecIndexer):
         if not self.index_config.path_without_protocol.startswith("/"):
             self.index_config.path_without_protocol = "/" + self.index_config.path_without_protocol
 
+    @requires_dependencies(["dropboxdrivefs", "fsspec"], extras="dropbox")
+    def precheck(self) -> None:
+        super().precheck()
+
     @requires_dependencies(["dropboxdrivefs", "fsspec"], extras="dropbox")
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         return super().run(**kwargs)
@@ -68,7 +73,6 @@ class DropboxIndexer(FsspecIndexer):
         return sterilize_dict(data=info)
 
 
-@dataclass
 class DropboxDownloaderConfig(FsspecDownloaderConfig):
     pass
 
@@ -91,7 +95,6 @@ class DropboxDownloader(FsspecDownloader):
         return await super().run_async(file_data=file_data, **kwargs)
 
 
-@dataclass
 class DropboxUploaderConfig(FsspecUploaderConfig):
     pass
 
@@ -106,6 +109,10 @@ class DropboxUploader(FsspecUploader):
     def __post_init__(self):
         super().__post_init__()
 
+    @requires_dependencies(["dropboxdrivefs", "fsspec"], extras="dropbox")
+    def precheck(self) -> None:
+        super().precheck()
+
     @requires_dependencies(["dropboxdrivefs", "fsspec"], extras="dropbox")
     def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
         return super().run(contents=contents, **kwargs)

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
 import contextlib
-import fnmatch
 from dataclasses import dataclass, field
 from datetime import datetime
 from pathlib import Path
@@ -9,10 +8,13 @@ from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional, TypeVar
 from uuid import NAMESPACE_DNS, uuid5
 
-from unstructured.documents.elements import DataSourceMetadata
+from pydantic import BaseModel, Field, Secret
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
+from unstructured_ingest.error import (
+    DestinationConnectionError,
+    SourceConnectionError,
+    SourceConnectionNetworkError,
+)
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -20,6 +22,7 @@ from unstructured_ingest.v2.interfaces import (
     DownloaderConfig,
     DownloadResponse,
     FileData,
+    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
     SourceIdentifiers,
@@ -36,17 +39,12 @@ if TYPE_CHECKING:
 CONNECTOR_TYPE = "fsspec"
 
 
-class Base(object):
-    def __post_init__(self):
-        pass
-
-
-@dataclass
-class FileConfig(Base):
-    remote_url: str
-    protocol: str = field(init=False)
-    path_without_protocol: str = field(init=False)
-    supported_protocols: list[str] = field(
+class FileConfig(BaseModel):
+    remote_url: str = Field(description="Remote fsspec URL formatted as `protocol://dir/path`")
+    protocol: str = Field(init=False)
+    path_without_protocol: str = Field(init=False)
+    supported_protocols: list[str] = Field(
+        init=False,
         default_factory=lambda: [
             "s3",
             "s3a",
@@ -57,38 +55,27 @@ class FileConfig(Base):
             "box",
             "dropbox",
             "sftp",
-        ]
+        ],
     )
 
-    def __post_init__(self):
-        super().__post_init__()
-        self.protocol, self.path_without_protocol = self.remote_url.split("://")
-        if self.protocol not in self.supported_protocols:
-            raise ValueError(
-                "Protocol {} not supported yet, only {} are supported.".format(
-                    self.protocol, ", ".join(self.supported_protocols)
-                ),
-            )
+    def __init__(self, **data):
+        protocol, path_without_protocol = data["remote_url"].split("://")
+        data["protocol"] = protocol
+        data["path_without_protocol"] = path_without_protocol
+        super().__init__(**data)
 
 
-@dataclass
 class FsspecIndexerConfig(FileConfig, IndexerConfig):
     recursive: bool = False
-    file_glob: Optional[list[str]] = None
 
 
-@dataclass
 class FsspecAccessConfig(AccessConfig):
     pass
 
 
-FsspecAccessConfigT = TypeVar("FsspecAccessConfigT", bound=FsspecAccessConfig)
-
-
-@dataclass
 class FsspecConnectionConfig(ConnectionConfig):
-    access_config: FsspecAccessConfigT = enhanced_field(sensitive=True, default=None)
-    connector_type: str = CONNECTOR_TYPE
+    access_config: Secret[FsspecAccessConfig]
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
 
 FsspecIndexerConfigT = TypeVar("FsspecIndexerConfigT", bound=FsspecIndexerConfig)
@@ -99,7 +86,7 @@ FsspecConnectionConfigT = TypeVar("FsspecConnectionConfigT", bound=FsspecConnect
 class FsspecIndexer(Indexer):
     connection_config: FsspecConnectionConfigT
     index_config: FsspecIndexerConfigT
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     @property
     def fs(self) -> "AbstractFileSystem":
@@ -109,17 +96,7 @@ class FsspecIndexer(Indexer):
             **self.connection_config.get_access_config(),
         )
 
-    def does_path_match_glob(self, path: str) -> bool:
-        if self.index_config.file_glob is None:
-            return True
-        patterns = self.index_config.file_glob
-        for pattern in patterns:
-            if fnmatch.filter([path], pattern):
-                return True
-        logger.debug(f"The file {path!r} is discarded as it does not match any given glob.")
-        return False
-
-    def check_connection(self):
+    def precheck(self) -> None:
         from fsspec import get_filesystem_class
 
         try:
@@ -157,10 +134,10 @@ class FsspecIndexer(Indexer):
             else:
                 raise TypeError(f"unhandled response type from find: {type(found)}")
 
-    def get_metadata(self, path: str) -> DataSourceMetadata:
+    def get_metadata(self, path: str) -> FileDataSourceMetadata:
         date_created = None
         date_modified = None
-
+        file_size = None
         try:
             created: Optional[Any] = self.fs.created(path)
             if created:
@@ -180,6 +157,8 @@ class FsspecIndexer(Indexer):
                     date_modified = str(modified)
         except NotImplementedError:
             pass
+        with contextlib.suppress(AttributeError):
+            file_size = self.fs.size(path)
 
         version = self.fs.checksum(path)
         metadata: dict[str, str] = {}
@@ -189,15 +168,19 @@ class FsspecIndexer(Indexer):
             "protocol": self.index_config.protocol,
             "remote_file_path": self.index_config.remote_url,
         }
+        file_stat = self.fs.stat(path=path)
+        if file_id := file_stat.get("id"):
+            record_locator["file_id"] = file_id
         if metadata:
             record_locator["metadata"] = metadata
-        return DataSourceMetadata(
+        return FileDataSourceMetadata(
             date_created=date_created,
             date_modified=date_modified,
             date_processed=str(time()),
             version=str(version),
             url=f"{self.index_config.protocol}://{path}",
             record_locator=record_locator,
+            filesize_bytes=file_size,
         )
 
     def sterilize_info(self, path) -> dict:
@@ -205,8 +188,7 @@ class FsspecIndexer(Indexer):
         return sterilize_dict(data=info)
 
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
-        raw_files = self.list_files()
-        files = [f for f in raw_files if self.does_path_match_glob(f)]
+        files = self.list_files()
         for file in files:
             # Note: we remove any remaining leading slashes (Box introduces these)
             # to get a valid relative path
@@ -227,7 +209,6 @@ class FsspecIndexer(Indexer):
             )
 
 
-@dataclass
 class FsspecDownloaderConfig(DownloaderConfig):
     pass
 
@@ -255,13 +236,6 @@ class FsspecDownloader(Downloader):
             **self.connection_config.get_access_config(),
         )
 
-    def get_download_path(self, file_data: FileData) -> Path:
-        return (
-            self.download_dir / Path(file_data.source_identifiers.relative_path)
-            if self.download_config
-            else Path(file_data.source_identifiers.rel_path)
-        )
-
     def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
         download_path = self.get_download_path(file_data=file_data)
         download_path.parent.mkdir(parents=True, exist_ok=True)
@@ -285,9 +259,10 @@ class FsspecDownloader(Downloader):
         return self.generate_download_response(file_data=file_data, download_path=download_path)
 
 
-@dataclass
 class FsspecUploaderConfig(FileConfig, UploaderConfig):
-    overwrite: bool = False
+    overwrite: bool = Field(
+        default=False, description="If true, an existing file will be overwritten."
+    )
 
 
 FsspecUploaderConfigT = TypeVar("FsspecUploaderConfigT", bound=FsspecUploaderConfig)
@@ -315,6 +290,19 @@ class FsspecUploader(Uploader):
                 f"missing 1 required positional argument: 'upload_config'"
             )
 
+    def precheck(self) -> None:
+        from fsspec import get_filesystem_class
+
+        try:
+            fs = get_filesystem_class(self.upload_config.protocol)(
+                **self.connection_config.get_access_config(),
+            )
+            root_dir = self.upload_config.path_without_protocol.split("/")[0]
+            fs.ls(path=root_dir, detail=False)
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
+
     def get_upload_path(self, file_data: FileData) -> Path:
         upload_path = (
             Path(self.upload_config.path_without_protocol)

unstructured_ingest/v2/processes/connectors/fsspec/gcs.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Generator, Optional, Union
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import Field, Secret
+
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.utils.string_and_date_utils import json_to_dict
 from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
@@ -26,17 +27,41 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
 CONNECTOR_TYPE = "gcs"
 
 
-@dataclass
 class GcsIndexerConfig(FsspecIndexerConfig):
     pass
 
 
-@dataclass
+service_account_key_description = """
+  Options:
+  - ``None``, GCSFS will attempt to guess your credentials in the
+following order: gcloud CLI default, gcsfs cached token, google compute
+metadata service, anonymous.
+  - ``'google_default'``, your default gcloud credentials will be used,
+    which are typically established by doing ``gcloud login`` in a terminal.
+  - ``'cache'``, credentials from previously successful gcsfs
+    authentication will be used (use this after "browser" auth succeeded)
+  - ``'anon'``, no authentication is performed, and you can only
+    access data which is accessible to allUsers (in this case, the project and
+    access level parameters are meaningless)
+  - ``'browser'``, you get an access code with which you can
+    authenticate via a specially provided URL
+  - if ``'cloud'``, we assume we are running within google compute
+    or google container engine, and query the internal metadata directly for
+    a token.
+  - you may supply a token generated by the
+    [gcloud](https://cloud.google.com/sdk/docs/)
+    utility; this is either a python dictionary or the name of a file
+    containing the JSON returned by logging in with the gcloud CLI tool.
+  """
+
+
 class GcsAccessConfig(FsspecAccessConfig):
-    service_account_key: Optional[str] = None
-    token: Union[str, dict, None] = field(init=False, default=None)
+    service_account_key: Optional[str] = Field(
+        default=None, description=service_account_key_description
+    )
+    token: Union[str, dict, None] = Field(init=False, default=None)
 
-    def __post_init__(self):
+    def model_post_init(self, __context: Any) -> None:
         ALLOWED_AUTH_VALUES = "google_default", "cache", "anon", "browser", "cloud"
 
         # Case: null value
@@ -61,13 +86,15 @@ class GcsAccessConfig(FsspecAccessConfig):
         raise ValueError("Invalid auth token value")
 
 
-@dataclass
+SecretGcsAccessConfig = Secret[GcsAccessConfig]
+
+
 class GcsConnectionConfig(FsspecConnectionConfig):
-    supported_protocols: list[str] = field(default_factory=lambda: ["gs", "gcs"])
-    access_config: GcsAccessConfig = enhanced_field(
-        sensitive=True, default_factory=lambda: GcsAccessConfig()
+    supported_protocols: list[str] = field(default_factory=lambda: ["gs", "gcs"], init=False)
+    access_config: SecretGcsAccessConfig = Field(
+        default_factory=lambda: SecretGcsAccessConfig(secret_value=GcsAccessConfig())
     )
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
 
 @dataclass
@@ -80,8 +107,11 @@ class GcsIndexer(FsspecIndexer):
     def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
         return super().run(**kwargs)
 
+    @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
+    def precheck(self) -> None:
+        super().precheck()
+
 
-@dataclass
 class GcsDownloaderConfig(FsspecDownloaderConfig):
     pass
 
@@ -102,7 +132,6 @@ class GcsDownloader(FsspecDownloader):
         return await super().run_async(file_data=file_data, **kwargs)
 
 
-@dataclass
 class GcsUploaderConfig(FsspecUploaderConfig):
     pass
 
@@ -117,6 +146,10 @@ class GcsUploader(FsspecUploader):
     def __post_init__(self):
         super().__post_init__()
 
+    @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
+    def precheck(self) -> None:
+        super().precheck()
+
     @requires_dependencies(["gcsfs", "fsspec"], extras="gcs")
     def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
         return super().run(contents=contents, **kwargs)