unstructured-ingest 0.0.3__py3-none-any.whl → 0.0.4__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/elasticsearch.py

@@ -5,9 +5,10 @@ import uuid
 from dataclasses import dataclass, field
 from pathlib import Path
 from time import time
-from typing import TYPE_CHECKING, Any, Generator, Optional
+from typing import TYPE_CHECKING, Any, Generator, Optional, Union
+
+from pydantic import BaseModel, Field, Secret, SecretStr
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin, enhanced_field
 from unstructured_ingest.error import (
     DestinationConnectionError,
     SourceConnectionError,
@@ -44,57 +45,74 @@ if TYPE_CHECKING:
 CONNECTOR_TYPE = "elasticsearch"
 
 
-@dataclass
 class ElasticsearchAccessConfig(AccessConfig):
-    password: Optional[str] = None
-    api_key: Optional[str] = enhanced_field(default=None, overload_name="es_api_key")
-    bearer_auth: Optional[str] = None
-    ssl_assert_fingerprint: Optional[str] = None
-
-
-@dataclass
-class ElasticsearchClientInput(EnhancedDataClassJsonMixin):
+    password: Optional[str] = Field(
+        default=None, description="password when using basic auth or connecting to a cloud instance"
+    )
+    es_api_key: Optional[str] = Field(default=None, description="api key used for authentication")
+    bearer_auth: Optional[str] = Field(
+        default=None, description="bearer token used for HTTP bearer authentication"
+    )
+    ssl_assert_fingerprint: Optional[str] = Field(
+        default=None, description="SHA256 fingerprint value"
+    )
+
+
+class ElasticsearchClientInput(BaseModel):
     hosts: Optional[list[str]] = None
     cloud_id: Optional[str] = None
-    ca_certs: Optional[str] = None
-    basic_auth: Optional[tuple[str, str]] = enhanced_field(sensitive=True, default=None)
-    api_key: Optional[str] = enhanced_field(sensitive=True, default=None)
+    ca_certs: Optional[Path] = None
+    basic_auth: Optional[Secret[tuple[str, str]]] = None
+    api_key: Optional[Union[Secret[tuple[str, str]], SecretStr]] = None
 
 
-@dataclass
 class ElasticsearchConnectionConfig(ConnectionConfig):
-    hosts: Optional[list[str]] = None
-    username: Optional[str] = None
-    cloud_id: Optional[str] = None
-    api_key_id: Optional[str] = None
-    ca_certs: Optional[str] = None
-    access_config: ElasticsearchAccessConfig = enhanced_field(sensitive=True)
+    hosts: Optional[list[str]] = Field(
+        default=None,
+        description="list of the Elasticsearch hosts to connect to",
+        examples=["http://localhost:9200"],
+    )
+    username: Optional[str] = Field(default=None, description="username when using basic auth")
+    cloud_id: Optional[str] = Field(default=None, description="id used to connect to Elastic Cloud")
+    api_key_id: Optional[str] = Field(
+        default=None,
+        description="id associated with api key used for authentication: "
+        "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html",  # noqa: E501
+    )
+    ca_certs: Optional[Path] = None
+    access_config: Secret[ElasticsearchAccessConfig]
 
     def get_client_kwargs(self) -> dict:
         # Update auth related fields to conform to what the SDK expects based on the
         # supported methods:
         # https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/connecting.html
-        client_input = ElasticsearchClientInput()
+        client_input_kwargs: dict[str, Any] = {}
+        access_config = self.access_config.get_secret_value()
         if self.hosts:
-            client_input.hosts = self.hosts
+            client_input_kwargs["hosts"] = self.hosts
         if self.cloud_id:
-            client_input.cloud_id = self.cloud_id
+            client_input_kwargs["cloud_id"] = self.cloud_id
         if self.ca_certs:
-            client_input.ca_certs = self.ca_certs
-        if self.access_config.password and (
-            self.cloud_id or self.ca_certs or self.access_config.ssl_assert_fingerprint
+            client_input_kwargs["ca_certs"] = self.ca_certs
+        if access_config.password and (
+            self.cloud_id or self.ca_certs or access_config.ssl_assert_fingerprint
         ):
-            client_input.basic_auth = ("elastic", self.access_config.password)
-        elif not self.cloud_id and self.username and self.access_config.password:
-            client_input.basic_auth = (self.username, self.access_config.password)
-        elif self.access_config.api_key and self.api_key_id:
-            client_input.api_key = (self.api_key_id, self.access_config.api_key)
-        elif self.access_config.api_key:
-            client_input.api_key = self.access_config.api_key
-        logger.debug(
-            f"Elasticsearch client inputs mapped to: {client_input.to_dict(redact_sensitive=True)}"
+            client_input_kwargs["basic_auth"] = ("elastic", access_config.password)
+        elif not self.cloud_id and self.username and access_config.password:
+            client_input_kwargs["basic_auth"] = (self.username, access_config.password)
+        elif access_config.es_api_key and self.api_key_id:
+            client_input_kwargs["api_key"] = (self.api_key_id, access_config.es_api_key)
+        elif access_config.es_api_key:
+            client_input_kwargs["api_key"] = access_config.es_api_key
+        client_input = ElasticsearchClientInput(**client_input_kwargs)
+        logger.debug(f"Elasticsearch client inputs mapped to: {client_input.dict()}")
+        client_kwargs = client_input.dict()
+        client_kwargs["basic_auth"] = (
+            client_input.basic_auth.get_secret_value() if client_input.basic_auth else None
+        )
+        client_kwargs["api_key"] = (
+            client_input.api_key.get_secret_value() if client_input.api_key else None
         )
-        client_kwargs = client_input.to_dict(redact_sensitive=False)
         client_kwargs = {k: v for k, v in client_kwargs.items() if v is not None}
         return client_kwargs
 
@@ -114,7 +132,6 @@ class ElasticsearchConnectionConfig(ConnectionConfig):
             raise SourceConnectionError(f"failed to validate connection: {e}")
 
 
-@dataclass
 class ElasticsearchIndexerConfig(IndexerConfig):
     index_name: str
     batch_size: int = 100
@@ -186,7 +203,6 @@ class ElasticsearchIndexer(Indexer):
             )
 
 
-@dataclass
 class ElasticsearchDownloaderConfig(DownloaderConfig):
     fields: list[str] = field(default_factory=list)
 
@@ -292,9 +308,10 @@ class ElasticsearchDownloader(Downloader):
         return download_responses
 
 
-@dataclass
 class ElasticsearchUploadStagerConfig(UploadStagerConfig):
-    index_name: str
+    index_name: str = Field(
+        description="Name of the Elasticsearch index to pull data from, or upload data to."
+    )
 
 
 @dataclass
@@ -333,11 +350,19 @@ class ElasticsearchUploadStager(UploadStager):
         return output_path
 
 
-@dataclass
 class ElasticsearchUploaderConfig(UploaderConfig):
-    index_name: str
-    batch_size_bytes: int = 15_000_000
-    num_threads: int = 4
+    index_name: str = Field(
+        description="Name of the Elasticsearch index to pull data from, or upload data to."
+    )
+    batch_size_bytes: int = Field(
+        default=15_000_000,
+        description="Size limit (in bytes) for each batch of items to be uploaded. Check"
+        " https://www.elastic.co/guide/en/elasticsearch/guide/current/bulk.html"
+        "#_how_big_is_too_big for more information.",
+    )
+    num_threads: int = Field(
+        default=4, description="Number of threads to be used while uploading content"
+    )
 
 
 @dataclass

unstructured_ingest/v2/processes/connectors/fsspec/azure.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Generator, Optional
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import Field, Secret
+
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
 from unstructured_ingest.v2.processes.connector_registry import (
@@ -36,35 +37,59 @@ def azure_json_serial(obj):
     return json_serial(obj)
 
 
-@dataclass
 class AzureIndexerConfig(FsspecIndexerConfig):
     pass
 
 
-@dataclass
 class AzureAccessConfig(FsspecAccessConfig):
-    account_name: Optional[str] = None
-    account_key: Optional[str] = None
-    connection_string: Optional[str] = None
-    sas_token: Optional[str] = None
+    account_name: Optional[str] = Field(
+        default=None,
+        description="The storage account name. This is used to authenticate "
+        "requests signed with an account key and to construct "
+        "the storage endpoint. It is required unless a connection "
+        "string is given, or if a custom domain is used with "
+        "anonymous authentication.",
+    )
+    account_key: Optional[str] = Field(
+        default=None,
+        description="The storage account key. This is used for shared key "
+        "authentication. If any of account key, sas token or "
+        "client_id are not specified, anonymous access will be used.",
+    )
+    connection_string: Optional[str] = Field(
+        default=None,
+        description="If specified, this will override all other parameters. See "
+        "http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/ "  # noqa: E501
+        "for the connection string format.",
+    )
+    sas_token: Optional[str] = Field(
+        default=None,
+        description="A shared access signature token to use to authenticate "
+        "requests instead of the account key. If account key and "
+        "sas token are both specified, account key will be used "
+        "to sign. If any of account key, sas token or client_id "
+        "are not specified, anonymous access will be used.",
+    )
 
-    def __post_init__(self):
+    def model_post_init(self, __context: Any) -> None:
         if self.connection_string is None and self.account_name is None:
             raise ValueError("either connection_string or account_name must be set")
 
 
-@dataclass
+SecretAzureAccessConfig = Secret[AzureAccessConfig]
+
+
 class AzureConnectionConfig(FsspecConnectionConfig):
-    supported_protocols: list[str] = field(default_factory=lambda: ["az"])
-    access_config: AzureAccessConfig = enhanced_field(
-        sensitive=True, default_factory=lambda: AzureAccessConfig()
+    supported_protocols: list[str] = field(default_factory=lambda: ["az"], init=False)
+    access_config: SecretAzureAccessConfig = Field(
+        default_factory=lambda: SecretAzureAccessConfig(secret_value=AzureAccessConfig())
     )
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     def get_access_config(self) -> dict[str, Any]:
         # Avoid injecting None by filtering out k,v pairs where the value is None
         access_configs: dict[str, Any] = {
-            k: v for k, v in self.access_config.to_dict().items() if v
+            k: v for k, v in self.access_config.get_secret_value().dict().items() if v
         }
         return access_configs
 
@@ -88,7 +113,6 @@ class AzureIndexer(FsspecIndexer):
         return super().run(**kwargs)
 
 
-@dataclass
 class AzureDownloaderConfig(FsspecDownloaderConfig):
     pass
 
@@ -109,7 +133,6 @@ class AzureDownloader(FsspecDownloader):
         return await super().run_async(file_data=file_data, **kwargs)
 
 
-@dataclass
 class AzureUploaderConfig(FsspecUploaderConfig):
     pass
 

unstructured_ingest/v2/processes/connectors/fsspec/box.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Generator, Optional
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import Field, Secret
+
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
 from unstructured_ingest.v2.processes.connector_registry import (
@@ -25,35 +26,38 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
 CONNECTOR_TYPE = "box"
 
 
-@dataclass
 class BoxIndexerConfig(FsspecIndexerConfig):
     pass
 
 
-@dataclass
 class BoxAccessConfig(FsspecAccessConfig):
-    box_app_config: Optional[str] = None
+    box_app_config: Optional[str] = Field(
+        default=None, description="Path to Box app credentials as json file."
+    )
+
+
+SecretBoxAccessConfig = Secret[BoxAccessConfig]
 
 
-@dataclass
 class BoxConnectionConfig(FsspecConnectionConfig):
-    supported_protocols: list[str] = field(default_factory=lambda: ["box"])
-    access_config: BoxAccessConfig = enhanced_field(
-        sensitive=True, default_factory=lambda: BoxAccessConfig()
+    supported_protocols: list[str] = field(default_factory=lambda: ["box"], init=False)
+    access_config: SecretBoxAccessConfig = Field(
+        default_factory=lambda: SecretBoxAccessConfig(secret_value=BoxAccessConfig())
     )
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     def get_access_config(self) -> dict[str, Any]:
         # Return access_kwargs with oauth. The oauth object can not be stored directly in the config
         # because it is not serializable.
         from boxsdk import JWTAuth
 
+        ac = self.access_config.get_secret_value()
         access_kwargs_with_oauth: dict[str, Any] = {
             "oauth": JWTAuth.from_settings_file(
-                self.access_config.box_app_config,
+                ac.box_app_config,
             ),
         }
-        access_config: dict[str, Any] = self.access_config.to_dict()
+        access_config: dict[str, Any] = ac.dict()
         access_config.pop("box_app_config", None)
         access_kwargs_with_oauth.update(access_config)
 
@@ -75,7 +79,6 @@ class BoxIndexer(FsspecIndexer):
         super().precheck()
 
 
-@dataclass
 class BoxDownloaderConfig(FsspecDownloaderConfig):
     pass
 
@@ -96,7 +99,6 @@ class BoxDownloader(FsspecDownloader):
         return await super().run_async(file_data=file_data, **kwargs)
 
 
-@dataclass
 class BoxUploaderConfig(FsspecUploaderConfig):
     pass
 

unstructured_ingest/v2/processes/connectors/fsspec/dropbox.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Generator, Optional
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import Field, Secret
+
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
 from unstructured_ingest.v2.processes.connector_registry import (
@@ -26,23 +27,23 @@ from unstructured_ingest.v2.processes.connectors.fsspec.utils import sterilize_d
 CONNECTOR_TYPE = "dropbox"
 
 
-@dataclass
 class DropboxIndexerConfig(FsspecIndexerConfig):
     pass
 
 
-@dataclass
 class DropboxAccessConfig(FsspecAccessConfig):
-    token: Optional[str] = None
+    token: Optional[str] = Field(default=None, description="Dropbox access token.")
+
+
+SecretDropboxAccessConfig = Secret[DropboxAccessConfig]
 
 
-@dataclass
 class DropboxConnectionConfig(FsspecConnectionConfig):
-    supported_protocols: list[str] = field(default_factory=lambda: ["dropbox"])
-    access_config: DropboxAccessConfig = enhanced_field(
-        sensitive=True, default_factory=lambda: DropboxAccessConfig()
+    supported_protocols: list[str] = field(default_factory=lambda: ["dropbox"], init=False)
+    access_config: SecretDropboxAccessConfig = Field(
+        default_factory=lambda: SecretDropboxAccessConfig(secret_value=DropboxAccessConfig())
     )
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
 
 @dataclass
@@ -72,7 +73,6 @@ class DropboxIndexer(FsspecIndexer):
         return sterilize_dict(data=info)
 
 
-@dataclass
 class DropboxDownloaderConfig(FsspecDownloaderConfig):
     pass
 
@@ -95,7 +95,6 @@ class DropboxDownloader(FsspecDownloader):
         return await super().run_async(file_data=file_data, **kwargs)
 
 
-@dataclass
 class DropboxUploaderConfig(FsspecUploaderConfig):
     pass
 

unstructured_ingest/v2/processes/connectors/fsspec/fsspec.py

@@ -8,7 +8,8 @@ from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional, TypeVar
 from uuid import NAMESPACE_DNS, uuid5
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import BaseModel, Field, Secret
+
 from unstructured_ingest.error import (
     DestinationConnectionError,
     SourceConnectionError,
@@ -38,17 +39,12 @@ if TYPE_CHECKING:
 CONNECTOR_TYPE = "fsspec"
 
 
-class Base(object):
-    def __post_init__(self):
-        pass
-
-
-@dataclass
-class FileConfig(Base):
-    remote_url: str
-    protocol: str = field(init=False)
-    path_without_protocol: str = field(init=False)
-    supported_protocols: list[str] = field(
+class FileConfig(BaseModel):
+    remote_url: str = Field(description="Remote fsspec URL formatted as `protocol://dir/path`")
+    protocol: str = Field(init=False)
+    path_without_protocol: str = Field(init=False)
+    supported_protocols: list[str] = Field(
+        init=False,
         default_factory=lambda: [
             "s3",
             "s3a",
@@ -59,37 +55,27 @@ class FileConfig(Base):
             "box",
             "dropbox",
             "sftp",
-        ]
+        ],
     )
 
-    def __post_init__(self):
-        super().__post_init__()
-        self.protocol, self.path_without_protocol = self.remote_url.split("://")
-        if self.protocol not in self.supported_protocols:
-            raise ValueError(
-                "Protocol {} not supported yet, only {} are supported.".format(
-                    self.protocol, ", ".join(self.supported_protocols)
-                ),
-            )
+    def __init__(self, **data):
+        protocol, path_without_protocol = data["remote_url"].split("://")
+        data["protocol"] = protocol
+        data["path_without_protocol"] = path_without_protocol
+        super().__init__(**data)
 
 
-@dataclass
 class FsspecIndexerConfig(FileConfig, IndexerConfig):
     recursive: bool = False
 
 
-@dataclass
 class FsspecAccessConfig(AccessConfig):
     pass
 
 
-FsspecAccessConfigT = TypeVar("FsspecAccessConfigT", bound=FsspecAccessConfig)
-
-
-@dataclass
 class FsspecConnectionConfig(ConnectionConfig):
-    access_config: FsspecAccessConfigT = enhanced_field(sensitive=True, default=None)
-    connector_type: str = CONNECTOR_TYPE
+    access_config: Secret[FsspecAccessConfig]
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
 
 FsspecIndexerConfigT = TypeVar("FsspecIndexerConfigT", bound=FsspecIndexerConfig)
@@ -100,7 +86,7 @@ FsspecConnectionConfigT = TypeVar("FsspecConnectionConfigT", bound=FsspecConnect
 class FsspecIndexer(Indexer):
     connection_config: FsspecConnectionConfigT
     index_config: FsspecIndexerConfigT
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
     @property
     def fs(self) -> "AbstractFileSystem":
@@ -223,7 +209,6 @@ class FsspecIndexer(Indexer):
             )
 
 
-@dataclass
 class FsspecDownloaderConfig(DownloaderConfig):
     pass
 
@@ -274,9 +259,10 @@ class FsspecDownloader(Downloader):
         return self.generate_download_response(file_data=file_data, download_path=download_path)
 
 
-@dataclass
 class FsspecUploaderConfig(FileConfig, UploaderConfig):
-    overwrite: bool = False
+    overwrite: bool = Field(
+        default=False, description="If true, an existing file will be overwritten."
+    )
 
 
 FsspecUploaderConfigT = TypeVar("FsspecUploaderConfigT", bound=FsspecUploaderConfig)

unstructured_ingest/v2/processes/connectors/fsspec/gcs.py

@@ -4,7 +4,8 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Generator, Optional, Union
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import Field, Secret
+
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.utils.string_and_date_utils import json_to_dict
 from unstructured_ingest.v2.interfaces import DownloadResponse, FileData, UploadContent
@@ -26,17 +27,41 @@ from unstructured_ingest.v2.processes.connectors.fsspec.fsspec import (
 CONNECTOR_TYPE = "gcs"
 
 
-@dataclass
 class GcsIndexerConfig(FsspecIndexerConfig):
     pass
 
 
-@dataclass
+service_account_key_description = """
+  Options:
+  - ``None``, GCSFS will attempt to guess your credentials in the
+following order: gcloud CLI default, gcsfs cached token, google compute
+metadata service, anonymous.
+  - ``'google_default'``, your default gcloud credentials will be used,
+    which are typically established by doing ``gcloud login`` in a terminal.
+  - ``'cache'``, credentials from previously successful gcsfs
+    authentication will be used (use this after "browser" auth succeeded)
+  - ``'anon'``, no authentication is performed, and you can only
+    access data which is accessible to allUsers (in this case, the project and
+    access level parameters are meaningless)
+  - ``'browser'``, you get an access code with which you can
+    authenticate via a specially provided URL
+  - if ``'cloud'``, we assume we are running within google compute
+    or google container engine, and query the internal metadata directly for
+    a token.
+  - you may supply a token generated by the
+    [gcloud](https://cloud.google.com/sdk/docs/)
+    utility; this is either a python dictionary or the name of a file
+    containing the JSON returned by logging in with the gcloud CLI tool.
+  """
+
+
 class GcsAccessConfig(FsspecAccessConfig):
-    service_account_key: Optional[str] = None
-    token: Union[str, dict, None] = field(init=False, default=None)
+    service_account_key: Optional[str] = Field(
+        default=None, description=service_account_key_description
+    )
+    token: Union[str, dict, None] = Field(init=False, default=None)
 
-    def __post_init__(self):
+    def model_post_init(self, __context: Any) -> None:
         ALLOWED_AUTH_VALUES = "google_default", "cache", "anon", "browser", "cloud"
 
         # Case: null value
@@ -61,13 +86,15 @@ class GcsAccessConfig(FsspecAccessConfig):
         raise ValueError("Invalid auth token value")
 
 
-@dataclass
+SecretGcsAccessConfig = Secret[GcsAccessConfig]
+
+
 class GcsConnectionConfig(FsspecConnectionConfig):
-    supported_protocols: list[str] = field(default_factory=lambda: ["gs", "gcs"])
-    access_config: GcsAccessConfig = enhanced_field(
-        sensitive=True, default_factory=lambda: GcsAccessConfig()
+    supported_protocols: list[str] = field(default_factory=lambda: ["gs", "gcs"], init=False)
+    access_config: SecretGcsAccessConfig = Field(
+        default_factory=lambda: SecretGcsAccessConfig(secret_value=GcsAccessConfig())
     )
-    connector_type: str = CONNECTOR_TYPE
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
 
 @dataclass
@@ -85,7 +112,6 @@ class GcsIndexer(FsspecIndexer):
         super().precheck()
 
 
-@dataclass
 class GcsDownloaderConfig(FsspecDownloaderConfig):
     pass
 
@@ -106,7 +132,6 @@ class GcsDownloader(FsspecDownloader):
         return await super().run_async(file_data=file_data, **kwargs)
 
 
-@dataclass
 class GcsUploaderConfig(FsspecUploaderConfig):
     pass