unstructured-ingest 0.0.3__py3-none-any.whl → 0.0.5__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/fsspec/azure.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Generator, Optional
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import Field, Secret
+
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
 from unstructured_ingest.v2.processes.connector_registry import (
@@ -36,35 +37,59 @@ def azure_json_serial(obj):
     return json_serial(obj)
 
 
-@dataclass
 class AzureIndexerConfig(FsspecIndexerConfig):
     pass
 
 
-@dataclass
 class AzureAccessConfig(FsspecAccessConfig):
-    account_name: Optional[str] = None
-    account_key: Optional[str] = None
-    connection_string: Optional[str] = None
-    sas_token: Optional[str] = None
+    account_name: Optional[str] = Field(
+        default=None,
+        description="The storage account name. This is used to authenticate "
+        "requests signed with an account key and to construct "
+        "the storage endpoint. It is required unless a connection "
+        "string is given, or if a custom domain is used with "
+        "anonymous authentication.",
+    )
+    account_key: Optional[str] = Field(
+        default=None,
+        description="The storage account key. This is used for shared key "
+        "authentication. If any of account key, sas token or "
+        "client_id are not specified, anonymous access will be used.",
+    )
+    connection_string: Optional[str] = Field(
+        default=None,
+        description="If specified, this will override all other parameters. See "
+        "http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ "  # noqa: E501
+        "for the connection string format.",
+    )
+    sas_token: Optional[str] = Field(
+        default=None,
+        description="A shared access signature token to use to authenticate "
+        "requests instead of the account key. If account key and "
+        "sas token are both specified, account key will be used "
+        "to sign. If any of account key, sas token or client_id "
+        "are not specified, anonymous access will be used.",
+    )
 
-    def __post_init__(self):
+    def model_post_init(self, __context: Any) -> None:
         if self.connection_string is None and self.account_name is None:
             raise ValueError("either connection_string or account_name must be set")
 
 
-@dataclass
+SecretAzureAccessConfig = Secret[AzureAccessConfig]
+
+
 class AzureConnectionConfig(FsspecConnectionConfig):
-    supported_protocols: list[str] = field(default_factory=lambda: ["az"])
-    access_config: AzureAccessConfig = enhanced_field(
-        sensitive=True, default_factory=lambda: AzureAccessConfig()
+    supported_protocols: list[str] = field(default_factory=lambda: ["az"], init=False)
+    access_config: SecretAzureAccessConfig = Field(
+        default_factory=lambda: SecretAzureAccessConfig(secret_value=AzureAccessConfig())
     )
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     def get_access_config(self) -> dict[str, Any]:
         # Avoid injecting None by filtering out k,v pairs where the value is None
         access_configs: dict[str, Any] = {
-            k: v for k, v in self.access_config.to_dict().items() if v
+            k: v for k, v in self.access_config.get_secret_value().dict().items() if v
         }
         return access_configs
 
@@ -88,7 +113,6 @@ class AzureIndexer(FsspecIndexer):
         return super().run(**kwargs)
 
 
-@dataclass
 class AzureDownloaderConfig(FsspecDownloaderConfig):
     pass
 
@@ -109,7 +133,6 @@ class AzureDownloader(FsspecDownloader):
         return await super().run_async(file_data=file_data, **kwargs)
 
 
-@dataclass
 class AzureUploaderConfig(FsspecUploaderConfig):
     pass
 

unstructured_ingest/v2/processes/connectors/fsspec/box.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Generator, Optional
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import Field, Secret
+
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
 from unstructured_ingest.v2.processes.connector_registry import (
@@ -25,35 +26,38 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
 CONNECTOR_TYPE = "box"
 
 
-@dataclass
 class BoxIndexerConfig(FsspecIndexerConfig):
     pass
 
 
-@dataclass
 class BoxAccessConfig(FsspecAccessConfig):
-    box_app_config: Optional[str] = None
+    box_app_config: Optional[str] = Field(
+        default=None, description="Path to Box app credentials as json file."
+    )
+
+
+SecretBoxAccessConfig = Secret[BoxAccessConfig]
 
 
-@dataclass
 class BoxConnectionConfig(FsspecConnectionConfig):
-    supported_protocols: list[str] = field(default_factory=lambda: ["box"])
-    access_config: BoxAccessConfig = enhanced_field(
-        sensitive=True, default_factory=lambda: BoxAccessConfig()
+    supported_protocols: list[str] = field(default_factory=lambda: ["box"], init=False)
+    access_config: SecretBoxAccessConfig = Field(
+        default_factory=lambda: SecretBoxAccessConfig(secret_value=BoxAccessConfig())
     )
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     def get_access_config(self) -> dict[str, Any]:
         # Return access_kwargs with oauth. The oauth object can not be stored directly in the config
         # because it is not serializable.
         from boxsdk import JWTAuth
 
+        ac = self.access_config.get_secret_value()
         access_kwargs_with_oauth: dict[str, Any] = {
             "oauth": JWTAuth.from_settings_file(
-                self.access_config.box_app_config,
+                ac.box_app_config,
             ),
         }
-        access_config: dict[str, Any] = self.access_config.to_dict()
+        access_config: dict[str, Any] = ac.dict()
         access_config.pop("box_app_config", None)
         access_kwargs_with_oauth.update(access_config)
 
@@ -75,7 +79,6 @@ class BoxIndexer(FsspecIndexer):
         super().precheck()
 
 
-@dataclass
 class BoxDownloaderConfig(FsspecDownloaderConfig):
     pass
 
@@ -96,7 +99,6 @@ class BoxDownloader(FsspecDownloader):
         return await super().run_async(file_data=file_data, **kwargs)
 
 
-@dataclass
 class BoxUploaderConfig(FsspecUploaderConfig):
     pass
 

unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Generator, Optional
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import Field, Secret
+
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
 from unstructured_ingest.v2.processes.connector_registry import (
@@ -26,23 +27,23 @@ from unstructured_ingest.v2.processes.connectors.fsspec.utils import sterilize_d
 CONNECTOR_TYPE = "dropbox"
 
 
-@dataclass
 class DropboxIndexerConfig(FsspecIndexerConfig):
     pass
 
 
-@dataclass
 class DropboxAccessConfig(FsspecAccessConfig):
-    token: Optional[str] = None
+    token: Optional[str] = Field(default=None, description="Dropbox access token.")
+
+
+SecretDropboxAccessConfig = Secret[DropboxAccessConfig]
 
 
-@dataclass
 class DropboxConnectionConfig(FsspecConnectionConfig):
-    supported_protocols: list[str] = field(default_factory=lambda: ["dropbox"])
-    access_config: DropboxAccessConfig = enhanced_field(
-        sensitive=True, default_factory=lambda: DropboxAccessConfig()
+    supported_protocols: list[str] = field(default_factory=lambda: ["dropbox"], init=False)
+    access_config: SecretDropboxAccessConfig = Field(
+        default_factory=lambda: SecretDropboxAccessConfig(secret_value=DropboxAccessConfig())
     )
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
 
 @dataclass
@@ -72,7 +73,6 @@ class DropboxIndexer(FsspecIndexer):
         return sterilize_dict(data=info)
 
 
-@dataclass
 class DropboxDownloaderConfig(FsspecDownloaderConfig):
     pass
 
@@ -95,7 +95,6 @@ class DropboxDownloader(FsspecDownloader):
         return await super().run_async(file_data=file_data, **kwargs)
 
 
-@dataclass
 class DropboxUploaderConfig(FsspecUploaderConfig):
     pass
 

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -8,7 +8,8 @@ from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional, TypeVar
 from uuid import NAMESPACE_DNS, uuid5
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import BaseModel, Field, Secret
+
 from unstructured_ingest.error import (
     DestinationConnectionError,
     SourceConnectionError,
@@ -38,17 +39,12 @@ if TYPE_CHECKING:
 CONNECTOR_TYPE = "fsspec"
 
 
-class Base(object):
-    def __post_init__(self):
-        pass
-
-
-@dataclass
-class FileConfig(Base):
-    remote_url: str
-    protocol: str = field(init=False)
-    path_without_protocol: str = field(init=False)
-    supported_protocols: list[str] = field(
+class FileConfig(BaseModel):
+    remote_url: str = Field(description="Remote fsspec URL formatted as `protocol://dir/path`")
+    protocol: str = Field(init=False)
+    path_without_protocol: str = Field(init=False)
+    supported_protocols: list[str] = Field(
+        init=False,
         default_factory=lambda: [
             "s3",
             "s3a",
@@ -59,37 +55,27 @@ class FileConfig(Base):
             "box",
             "dropbox",
             "sftp",
-        ]
+        ],
     )
 
-    def __post_init__(self):
-        super().__post_init__()
-        self.protocol, self.path_without_protocol = self.remote_url.split("://")
-        if self.protocol not in self.supported_protocols:
-            raise ValueError(
-                "Protocol {} not supported yet, only {} are supported.".format(
-                    self.protocol, ", ".join(self.supported_protocols)
-                ),
-            )
+    def __init__(self, **data):
+        protocol, path_without_protocol = data["remote_url"].split("://")
+        data["protocol"] = protocol
+        data["path_without_protocol"] = path_without_protocol
+        super().__init__(**data)
 
 
-@dataclass
 class FsspecIndexerConfig(FileConfig, IndexerConfig):
     recursive: bool = False
 
 
-@dataclass
 class FsspecAccessConfig(AccessConfig):
     pass
 
 
-FsspecAccessConfigT = TypeVar("FsspecAccessConfigT", bound=FsspecAccessConfig)
-
-
-@dataclass
 class FsspecConnectionConfig(ConnectionConfig):
-    access_config: FsspecAccessConfigT = enhanced_field(sensitive=True, default=None)
-    connector_type: str = CONNECTOR_TYPE
+    access_config: Secret[FsspecAccessConfig]
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
 
 FsspecIndexerConfigT = TypeVar("FsspecIndexerConfigT", bound=FsspecIndexerConfig)
@@ -100,7 +86,7 @@ FsspecConnectionConfigT = TypeVar("FsspecConnectionConfigT", bound=FsspecConnect
 class FsspecIndexer(Indexer):
     connection_config: FsspecConnectionConfigT
     index_config: FsspecIndexerConfigT
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     @property
     def fs(self) -> "AbstractFileSystem":
@@ -223,7 +209,6 @@ class FsspecIndexer(Indexer):
             )
 
 
-@dataclass
 class FsspecDownloaderConfig(DownloaderConfig):
     pass
 
@@ -274,9 +259,10 @@ class FsspecDownloader(Downloader):
         return self.generate_download_response(file_data=file_data, download_path=download_path)
 
 
-@dataclass
 class FsspecUploaderConfig(FileConfig, UploaderConfig):
-    overwrite: bool = False
+    overwrite: bool = Field(
+        default=False, description="If true, an existing file will be overwritten."
+    )
 
 
 FsspecUploaderConfigT = TypeVar("FsspecUploaderConfigT", bound=FsspecUploaderConfig)

unstructured_ingest/v2/processes/connectors/fsspec/gcs.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Generator, Optional, Union
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import Field, Secret
+
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.utils.string_and_date_utils import json_to_dict
 from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
@@ -26,17 +27,41 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
 CONNECTOR_TYPE = "gcs"
 
 
-@dataclass
 class GcsIndexerConfig(FsspecIndexerConfig):
     pass
 
 
-@dataclass
+service_account_key_description = """
+  Options:
+  - ``None``, GCSFS will attempt to guess your credentials in the
+following order: gcloud CLI default, gcsfs cached token, google compute
+metadata service, anonymous.
+  - ``'google_default'``, your default gcloud credentials will be used,
+    which are typically established by doing ``gcloud login`` in a terminal.
+  - ``'cache'``, credentials from previously successful gcsfs
+    authentication will be used (use this after "browser" auth succeeded)
+  - ``'anon'``, no authentication is performed, and you can only
+    access data which is accessible to allUsers (in this case, the project and
+    access level parameters are meaningless)
+  - ``'browser'``, you get an access code with which you can
+    authenticate via a specially provided URL
+  - if ``'cloud'``, we assume we are running within google compute
+    or google container engine, and query the internal metadata directly for
+    a token.
+  - you may supply a token generated by the
+    [gcloud](https://cloud.google.com/sdk/docs/)
+    utility; this is either a python dictionary or the name of a file
+    containing the JSON returned by logging in with the gcloud CLI tool.
+  """
+
+
 class GcsAccessConfig(FsspecAccessConfig):
-    service_account_key: Optional[str] = None
-    token: Union[str, dict, None] = field(init=False, default=None)
+    service_account_key: Optional[str] = Field(
+        default=None, description=service_account_key_description
+    )
+    token: Union[str, dict, None] = Field(init=False, default=None)
 
-    def __post_init__(self):
+    def model_post_init(self, __context: Any) -> None:
         ALLOWED_AUTH_VALUES = "google_default", "cache", "anon", "browser", "cloud"
 
         # Case: null value
@@ -61,13 +86,15 @@ class GcsAccessConfig(FsspecAccessConfig):
         raise ValueError("Invalid auth token value")
 
 
-@dataclass
+SecretGcsAccessConfig = Secret[GcsAccessConfig]
+
+
 class GcsConnectionConfig(FsspecConnectionConfig):
-    supported_protocols: list[str] = field(default_factory=lambda: ["gs", "gcs"])
-    access_config: GcsAccessConfig = enhanced_field(
-        sensitive=True, default_factory=lambda: GcsAccessConfig()
+    supported_protocols: list[str] = field(default_factory=lambda: ["gs", "gcs"], init=False)
+    access_config: SecretGcsAccessConfig = Field(
+        default_factory=lambda: SecretGcsAccessConfig(secret_value=GcsAccessConfig())
     )
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
 
 @dataclass
@@ -85,7 +112,6 @@ class GcsIndexer(FsspecIndexer):
         super().precheck()
 
 
-@dataclass
 class GcsDownloaderConfig(FsspecDownloaderConfig):
     pass
 
@@ -106,7 +132,6 @@ class GcsDownloader(FsspecDownloader):
         return await super().run_async(file_data=file_data, **kwargs)
 
 
-@dataclass
 class GcsUploaderConfig(FsspecUploaderConfig):
     pass
 

unstructured_ingest/v2/processes/connectors/fsspec/s3.py

@@ -5,9 +5,9 @@ from pathlib import Path
 from time import time
 from typing import Any, Generator, Optional
 
-from unstructured.utils import requires_dependencies
+from pydantic import Field, Secret
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     DownloadResponse,
     FileData,
@@ -32,27 +32,41 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
 CONNECTOR_TYPE = "s3"
 
 
-@dataclass
 class S3IndexerConfig(FsspecIndexerConfig):
     pass
 
 
-@dataclass
 class S3AccessConfig(FsspecAccessConfig):
-    key: Optional[str] = None
-    secret: Optional[str] = None
-    token: Optional[str] = None
+    key: Optional[str] = Field(
+        default=None,
+        description="If not anonymous, use this access key ID, if specified. Takes precedence "
+        "over `aws_access_key_id` in client_kwargs.",
+    )
+    secret: Optional[str] = Field(
+        default=None, description="If not anonymous, use this secret access key, if specified."
+    )
+    token: Optional[str] = Field(
+        default=None, description="If not anonymous, use this security token, if specified."
+    )
+
+
+SecretS3AccessConfig = Secret[S3AccessConfig]
 
 
-@dataclass
 class S3ConnectionConfig(FsspecConnectionConfig):
-    supported_protocols: list[str] = field(default_factory=lambda: ["s3", "s3a"])
-    access_config: S3AccessConfig = enhanced_field(
-        sensitive=True, default_factory=lambda: S3AccessConfig()
+    supported_protocols: list[str] = field(default_factory=lambda: ["s3", "s3a"], init=False)
+    access_config: SecretS3AccessConfig = Field(
+        default_factory=lambda: SecretS3AccessConfig(secret_value=S3AccessConfig())
     )
-    endpoint_url: Optional[str] = None
-    anonymous: bool = False
-    connector_type: str = CONNECTOR_TYPE
+    endpoint_url: Optional[str] = Field(
+        default=None,
+        description="Use this endpoint_url, if specified. Needed for "
+        "connecting to non-AWS S3 buckets.",
+    )
+    anonymous: bool = Field(
+        default=False, description="Connect to s3 without local AWS credentials."
+    )
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     def get_access_config(self) -> dict[str, Any]:
         access_configs: dict[str, Any] = {"anon": self.anonymous}
@@ -60,7 +74,9 @@ class S3ConnectionConfig(FsspecConnectionConfig):
             access_configs["endpoint_url"] = self.endpoint_url
 
         # Avoid injecting None by filtering out k,v pairs where the value is None
-        access_configs.update({k: v for k, v in self.access_config.to_dict().items() if v})
+        access_configs.update(
+            {k: v for k, v in self.access_config.get_secret_value().dict().items() if v}
+        )
         return access_configs
 
 
@@ -116,7 +132,6 @@ class S3Indexer(FsspecIndexer):
         super().precheck()
 
 
-@dataclass
 class S3DownloaderConfig(FsspecDownloaderConfig):
     pass
 
@@ -137,7 +152,6 @@ class S3Downloader(FsspecDownloader):
         return await super().run_async(file_data=file_data, **kwargs)
 
 
-@dataclass
 class S3UploaderConfig(FsspecUploaderConfig):
     pass
 

unstructured_ingest/v2/processes/connectors/fsspec/sftp.py

@@ -6,7 +6,8 @@ from pathlib import Path
 from typing import Any, Generator, Optional
 from urllib.parse import urlparse
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import Field, Secret
+
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
 from unstructured_ingest.v2.processes.connector_registry import (
@@ -27,10 +28,10 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
 CONNECTOR_TYPE = "sftp"
 
 
-@dataclass
 class SftpIndexerConfig(FsspecIndexerConfig):
-    def __post_init__(self):
-        super().__post_init__()
+
+    def model_post_init(self, __context: Any) -> None:
+        super().model_post_init(__context)
         _, ext = os.path.splitext(self.remote_url)
         parsed_url = urlparse(self.remote_url)
         if ext:
@@ -39,21 +40,21 @@ class SftpIndexerConfig(FsspecIndexerConfig):
             self.path_without_protocol = parsed_url.path.lstrip("/")
 
 
-@dataclass
 class SftpAccessConfig(FsspecAccessConfig):
-    password: str
+    password: str = Field(description="Password for sftp connection")
 
 
-@dataclass
 class SftpConnectionConfig(FsspecConnectionConfig):
-    supported_protocols: list[str] = field(default_factory=lambda: ["sftp"])
-    access_config: SftpAccessConfig = enhanced_field(sensitive=True)
-    connector_type: str = CONNECTOR_TYPE
-    username: Optional[str] = None
-    host: Optional[str] = None
-    port: int = 22
-    look_for_keys: bool = False
-    allow_agent: bool = False
+    supported_protocols: list[str] = Field(default_factory=lambda: ["sftp"], init=False)
+    access_config: Secret[SftpAccessConfig]
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+    username: str = Field(description="Username for sftp connection")
+    host: Optional[str] = Field(default=None, description="Hostname for sftp connection")
+    port: int = Field(default=22, description="Port for sftp connection")
+    look_for_keys: bool = Field(
+        default=False, description="Whether to search for private key files in ~/.ssh/"
+    )
+    allow_agent: bool = Field(default=False, description="Whether to connect to the SSH agent.")
 
     def get_access_config(self) -> dict[str, Any]:
         access_config = {
@@ -62,7 +63,7 @@ class SftpConnectionConfig(FsspecConnectionConfig):
             "port": self.port,
             "look_for_keys": self.look_for_keys,
             "allow_agent": self.allow_agent,
-            "password": self.access_config.password,
+            "password": self.access_config.get_secret_value().password,
         }
         return access_config
 
@@ -96,24 +97,15 @@ class SftpIndexer(FsspecIndexer):
         super().precheck()
 
 
-@dataclass
 class SftpDownloaderConfig(FsspecDownloaderConfig):
-    remote_url: Optional[str] = None
-
-    def __post_init__(self):
-        # TODO once python3.9 no longer supported and kw_only is allowed in dataclasses, remove:
-        if not self.remote_url:
-            raise TypeError(
-                f"{self.__class__.__name__}.__init__() "
-                f"missing 1 required positional argument: 'remote_url'"
-            )
+    remote_url: str = Field(description="Remote fsspec URL formatted as `protocol://dir/path`")
 
 
 @dataclass
 class SftpDownloader(FsspecDownloader):
     protocol: str = "sftp"
     connection_config: SftpConnectionConfig
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
     download_config: Optional[SftpDownloaderConfig] = field(default_factory=SftpDownloaderConfig)
 
     @requires_dependencies(["paramiko", "fsspec"], extras="sftp")
@@ -131,7 +123,6 @@ class SftpDownloader(FsspecDownloader):
         return await super().run_async(file_data=file_data, **kwargs)
 
 
-@dataclass
 class SftpUploaderConfig(FsspecUploaderConfig):
     pass