unstructured-ingest 0.0.3__py3-none-any.whl → 0.0.4__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/opensearch.py

@@ -1,7 +1,9 @@
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import TYPE_CHECKING, Optional
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin, enhanced_field
+from pydantic import BaseModel, Field, Secret
+
 from unstructured_ingest.error import (
     DestinationConnectionError,
 )
@@ -35,20 +37,28 @@ CONNECTOR_TYPE = "opensearch"
 heavily on the Elasticsearch connector code, inheriting the functionality as much as possible."""
 
 
-@dataclass
 class OpenSearchAccessConfig(AccessConfig):
-    password: Optional[str] = enhanced_field(default=None, sensitive=True)
-    use_ssl: bool = False
-    verify_certs: bool = False
-    ssl_show_warn: bool = False
-    ca_certs: Optional[str] = None
-    client_cert: Optional[str] = None
-    client_key: Optional[str] = None
-
-
-@dataclass
-class OpenSearchClientInput(EnhancedDataClassJsonMixin):
-    http_auth: Optional[tuple[str, str]] = enhanced_field(sensitive=True, default=None)
+    password: Optional[str] = Field(default=None, description="password when using basic auth")
+    use_ssl: bool = Field(default=False, description="use ssl for the connection")
+    verify_certs: bool = Field(default=False, description="whether to verify SSL certificates")
+    ssl_show_warn: bool = Field(
+        default=False, description="show warning when verify certs is disabled"
+    )
+    ca_certs: Optional[Path] = Field(default=None, description="path to CA bundle")
+    client_cert: Optional[Path] = Field(
+        default=None,
+        description="path to the file containing the private key and the certificate,"
+        " or cert only if using client_key",
+    )
+    client_key: Optional[Path] = Field(
+        default=None,
+        description="path to the file containing the private key"
+        " if using separate cert and key files",
+    )
+
+
+class OpenSearchClientInput(BaseModel):
+    http_auth: Secret[Optional[tuple[str, str]]] = None
     hosts: Optional[list[str]] = None
     use_ssl: bool = False
     verify_certs: bool = False
@@ -58,37 +68,41 @@ class OpenSearchClientInput(EnhancedDataClassJsonMixin):
     client_key: Optional[str] = None
 
 
-@dataclass
 class OpenSearchConnectionConfig(ConnectionConfig):
-    hosts: Optional[list[str]] = None
-    username: Optional[str] = None
-    access_config: OpenSearchAccessConfig = enhanced_field(sensitive=True)
+    hosts: Optional[list[str]] = Field(
+        default=None,
+        description="List of the OpenSearch hosts to connect",
+        examples=["http://localhost:9200"],
+    )
+    username: Optional[str] = Field(default=None, description="username when using basic auth")
+    access_config: Secret[OpenSearchAccessConfig]
 
     def get_client_kwargs(self) -> dict:
         # Update auth related fields to conform to what the SDK expects based on the
         # supported methods:
         # https://github.com/opensearch-project/opensearch-py/blob/main/opensearchpy/client/__init__.py
-        client_input = OpenSearchClientInput()
+        access_config = self.access_config.get_secret_value()
+        client_input_kwargs = {}
         if self.hosts:
-            client_input.hosts = self.hosts
-        if self.access_config.use_ssl:
-            client_input.use_ssl = self.access_config.use_ssl
-        if self.access_config.verify_certs:
-            client_input.verify_certs = self.access_config.verify_certs
-        if self.access_config.ssl_show_warn:
-            client_input.ssl_show_warn = self.access_config.ssl_show_warn
-        if self.access_config.ca_certs:
-            client_input.ca_certs = self.access_config.ca_certs
-        if self.access_config.client_cert:
-            client_input.client_cert = self.access_config.client_cert
-        if self.access_config.client_key:
-            client_input.client_key = self.access_config.client_key
-        if self.username and self.access_config.password:
-            client_input.http_auth = (self.username, self.access_config.password)
-        logger.debug(
-            f"OpenSearch client inputs mapped to: {client_input.to_dict(redact_sensitive=True)}"
-        )
-        client_kwargs = client_input.to_dict(redact_sensitive=False)
+            client_input_kwargs["hosts"] = self.hosts
+        if access_config.use_ssl:
+            client_input_kwargs["use_ssl"] = access_config.use_ssl
+        if access_config.verify_certs:
+            client_input_kwargs["verify_certs"] = access_config.verify_certs
+        if access_config.ssl_show_warn:
+            client_input_kwargs["ssl_show_warn"] = access_config.ssl_show_warn
+        if access_config.ca_certs:
+            client_input_kwargs["ca_certs"] = str(access_config.ca_certs)
+        if access_config.client_cert:
+            client_input_kwargs["client_cert"] = str(access_config.client_cert)
+        if access_config.client_key:
+            client_input_kwargs["client_key"] = str(access_config.client_key)
+        if self.username and access_config.password:
+            client_input_kwargs["http_auth"] = (self.username, access_config.password)
+        client_input = OpenSearchClientInput(**client_input_kwargs)
+        logger.debug(f"OpenSearch client inputs mapped to: {client_input.dict()}")
+        client_kwargs = client_input.dict()
+        client_kwargs["http_auth"] = client_input.http_auth.get_secret_value()
         client_kwargs = {k: v for k, v in client_kwargs.items() if v is not None}
         return client_kwargs
 
@@ -100,15 +114,14 @@ class OpenSearchConnectionConfig(ConnectionConfig):
         return OpenSearch(**self.get_client_kwargs())
 
 
-@dataclass
-class OpensearchIndexerConfig(ElasticsearchIndexerConfig):
+class OpenSearchIndexerConfig(ElasticsearchIndexerConfig):
     pass
 
 
 @dataclass
 class OpenSearchIndexer(ElasticsearchIndexer):
     connection_config: OpenSearchConnectionConfig
-    index_config: OpensearchIndexerConfig
+    index_config: OpenSearchIndexerConfig
     client: "OpenSearch" = field(init=False)
 
     @requires_dependencies(["opensearchpy"], extras="opensearch")
@@ -118,15 +131,14 @@ class OpenSearchIndexer(ElasticsearchIndexer):
         return scan
 
 
-@dataclass
-class OpensearchDownloaderConfig(ElasticsearchDownloaderConfig):
+class OpenSearchDownloaderConfig(ElasticsearchDownloaderConfig):
     pass
 
 
 @dataclass
 class OpenSearchDownloader(ElasticsearchDownloader):
     connection_config: OpenSearchConnectionConfig
-    download_config: OpensearchDownloaderConfig
+    download_config: OpenSearchDownloaderConfig
     connector_type: str = CONNECTOR_TYPE
 
     @requires_dependencies(["opensearchpy"], extras="opensearch")
@@ -137,15 +149,14 @@ class OpenSearchDownloader(ElasticsearchDownloader):
         return AsyncOpenSearch, async_scan
 
 
-@dataclass
-class OpensearchUploaderConfig(ElasticsearchUploaderConfig):
+class OpenSearchUploaderConfig(ElasticsearchUploaderConfig):
     pass
 
 
 @dataclass
 class OpenSearchUploader(ElasticsearchUploader):
     connection_config: OpenSearchConnectionConfig
-    upload_config: OpensearchUploaderConfig
+    upload_config: OpenSearchUploaderConfig
     connector_type: str = CONNECTOR_TYPE
 
     @requires_dependencies(["opensearchpy"], extras="opensearch")
@@ -155,29 +166,28 @@ class OpenSearchUploader(ElasticsearchUploader):
         return parallel_bulk
 
 
-@dataclass
-class OpensearchUploadStagerConfig(ElasticsearchUploadStagerConfig):
+class OpenSearchUploadStagerConfig(ElasticsearchUploadStagerConfig):
     pass
 
 
 @dataclass
-class OpensearchUploadStager(ElasticsearchUploadStager):
-    upload_stager_config: OpensearchUploadStagerConfig
+class OpenSearchUploadStager(ElasticsearchUploadStager):
+    upload_stager_config: OpenSearchUploadStagerConfig
 
 
 opensearch_source_entry = SourceRegistryEntry(
     connection_config=OpenSearchConnectionConfig,
     indexer=OpenSearchIndexer,
-    indexer_config=OpensearchIndexerConfig,
+    indexer_config=OpenSearchIndexerConfig,
     downloader=OpenSearchDownloader,
-    downloader_config=OpensearchDownloaderConfig,
+    downloader_config=OpenSearchDownloaderConfig,
 )
 
 
 opensearch_destination_entry = DestinationRegistryEntry(
     connection_config=OpenSearchConnectionConfig,
-    upload_stager_config=OpensearchUploadStagerConfig,
-    upload_stager=OpensearchUploadStager,
-    uploader_config=OpensearchUploaderConfig,
+    upload_stager_config=OpenSearchUploadStagerConfig,
+    upload_stager=OpenSearchUploadStager,
+    uploader_config=OpenSearchUploaderConfig,
     uploader=OpenSearchUploader,
 )

unstructured_ingest/v2/processes/connectors/pinecone.py

@@ -5,12 +5,11 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 
-from unstructured.staging.base import flatten_dict
-from unstructured.utils import requires_dependencies
+from pydantic import Field, Secret
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
 from unstructured_ingest.error import DestinationConnectionError
-from unstructured_ingest.utils.data_prep import batch_generator
+from unstructured_ingest.utils.data_prep import batch_generator, flatten_dict
+from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -32,25 +31,31 @@ if TYPE_CHECKING:
 CONNECTOR_TYPE = "pinecone"
 
 
-@dataclass
 class PineconeAccessConfig(AccessConfig):
-    api_key: Optional[str] = enhanced_field(default=None, overload_name="pinecone_api_key")
+    pinecone_api_key: Optional[str] = Field(
+        default=None, description="API key for Pinecone.", alias="api_key"
+    )
+
+
+SecretPineconeAccessConfig = Secret[PineconeAccessConfig]
 
 
-@dataclass
 class PineconeConnectionConfig(ConnectionConfig):
-    index_name: str
-    environment: str
-    access_config: PineconeAccessConfig = enhanced_field(sensitive=True)
+    index_name: str = Field(description="Name of the index to connect to.")
+    environment: str = Field(description="Environment to connect to.")
+    access_config: SecretPineconeAccessConfig = Field(
+        default_factory=lambda: SecretPineconeAccessConfig(secret_value=PineconeAccessConfig())
+    )
 
     @requires_dependencies(["pinecone"], extras="pinecone")
     def get_index(self) -> "PineconeIndex":
         from pinecone import Pinecone
-        from unstructured import __version__ as unstructured_version
+
+        from unstructured_ingest import __version__ as unstructured_version
 
         pc = Pinecone(
-            api_key=self.access_config.api_key,
-            source_tag=f"unstructured=={unstructured_version}",
+            api_key=self.access_config.get_secret_value().pinecone_api_key,
+            source_tag=f"unstructured_ingest=={unstructured_version}",
         )
 
         index = pc.Index(self.index_name)
@@ -58,15 +63,13 @@ class PineconeConnectionConfig(ConnectionConfig):
         return index
 
 
-@dataclass
 class PineconeUploadStagerConfig(UploadStagerConfig):
     pass
 
 
-@dataclass
 class PineconeUploaderConfig(UploaderConfig):
-    batch_size: int = 100
-    num_of_processes: int = 4
+    batch_size: int = Field(default=100, description="Number of records per batch")
+    num_processes: int = Field(default=4, description="Number of processes to use for uploading")
 
 
 @dataclass
@@ -154,18 +157,18 @@ class PineconeUploader(Uploader):
             f" index named {self.connection_config.index_name}"
             f" environment named {self.connection_config.environment}"
             f" with batch size {self.upload_config.batch_size}"
-            f" with {self.upload_config.num_of_processes} (number of) processes"
+            f" with {self.upload_config.num_processes} (number of) processes"
         )
 
         pinecone_batch_size = self.upload_config.batch_size
 
-        if self.upload_config.num_of_processes == 1:
+        if self.upload_config.num_processes == 1:
             for batch in batch_generator(elements_dict, pinecone_batch_size):
                 self.upsert_batch(batch)  # noqa: E203
 
         else:
             with mp.Pool(
-                processes=self.upload_config.num_of_processes,
+                processes=self.upload_config.num_processes,
             ) as pool:
                 pool.map(
                     self.upsert_batch, list(batch_generator(elements_dict, pinecone_batch_size))

unstructured_ingest/v2/processes/connectors/salesforce.py

@@ -15,11 +15,11 @@ from email.utils import formatdate
 from pathlib import Path
 from string import Template
 from textwrap import dedent
-from typing import TYPE_CHECKING, Any, Generator, Type
+from typing import TYPE_CHECKING, Any, Generator, Optional, Type
 
 from dateutil import parser
+from pydantic import Field, Secret
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
 from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
@@ -75,49 +75,58 @@ $htmlbody
 )
 
 
-@dataclass
 class SalesforceAccessConfig(AccessConfig):
     consumer_key: str
-    private_key: str
+    private_key_path: Optional[Path] = Field(
+        default=None,
+        description="Path to the private key file. " "Key file is usually named server.key.",
+    )
+    private_key: Optional[str] = Field(default=None, description="Contents of the private key")
+
+    def model_post_init(self, __context: Any) -> None:
+        if self.private_key_path is None and self.private_key is None:
+            raise ValueError("either private_key or private_key_path must be set")
+        if self.private_key is not None and self.private_key_path is not None:
+            raise ValueError("only one of private_key or private_key_path must be set")
 
     @requires_dependencies(["cryptography"])
     def get_private_key_value_and_type(self) -> tuple[str, Type]:
         from cryptography.hazmat.primitives import serialization
 
-        try:
-            serialization.load_pem_private_key(data=self.private_key.encode("utf-8"), password=None)
-        except ValueError:
-            pass
-        else:
+        if self.private_key_path and self.private_key_path.is_file():
+            return str(self.private_key_path), Path
+        if self.private_key:
+            try:
+                serialization.load_pem_private_key(
+                    data=str(self.private_key).encode("utf-8"), password=None
+                )
+            except Exception as e:
+                raise ValueError(f"failed to validate private key data: {e}") from e
             return self.private_key, str
 
-        if Path(self.private_key).is_file():
-            return self.private_key, Path
-
         raise ValueError("private_key does not contain PEM private key or path")
 
 
-@dataclass
 class SalesforceConnectionConfig(ConnectionConfig):
     username: str
-    access_config: SalesforceAccessConfig = enhanced_field(sensitive=True)
+    access_config: Secret[SalesforceAccessConfig]
 
     @requires_dependencies(["simple_salesforce"], extras="salesforce")
     def get_client(self) -> "Salesforce":
         from simple_salesforce import Salesforce
 
-        pkey_value, pkey_type = self.access_config.get_private_key_value_and_type()
+        access_config = self.access_config.get_secret_value()
+        pkey_value, pkey_type = access_config.get_private_key_value_and_type()
 
         return Salesforce(
             username=self.username,
-            consumer_key=self.access_config.consumer_key,
+            consumer_key=access_config.consumer_key,
             privatekey_file=pkey_value if pkey_type is Path else None,
             privatekey=pkey_value if pkey_type is str else None,
             version=SALESFORCE_API_VERSION,
         )
 
 
-@dataclass
 class SalesforceIndexerConfig(IndexerConfig):
     categories: list[str]
 
@@ -201,7 +210,6 @@ class SalesforceIndexer(Indexer):
             yield f
 
 
-@dataclass
 class SalesforceDownloaderConfig(DownloaderConfig):
     pass
 

unstructured_ingest/v2/processes/connectors/sharepoint.py

@@ -6,7 +6,8 @@ from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional
 from urllib.parse import quote
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin, enhanced_field
+from pydantic import BaseModel, Field, Secret, SecretStr
+
 from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
@@ -54,24 +55,38 @@ class SharepointContentType(Enum):
     LIST = "list"
 
 
-@dataclass
 class SharepointAccessConfig(AccessConfig):
-    client_cred: str
-
-
-@dataclass
-class SharepointPermissionsConfig(EnhancedDataClassJsonMixin):
-    permissions_application_id: str
-    permissions_tenant: str
-    permissions_client_cred: str = enhanced_field(sensitive=True)
-    authority_url: Optional[str] = field(repr=False, default="https://login.microsoftonline.com")
+    client_cred: str = Field(description="Sharepoint app secret")
+
+
+class SharepointPermissionsConfig(BaseModel):
+    permissions_application_id: str = Field(description="Microsoft Graph API application id")
+    permissions_tenant: str = Field(
+        description="url to get permissions data within tenant.",
+        examples=["https://contoso.onmicrosoft.com"],
+    )
+    permissions_client_cred: SecretStr = Field(
+        description="Microsoft Graph API application credentials"
+    )
+    authority_url: Optional[SecretStr] = Field(
+        repr=False,
+        default_factory=lambda: SecretStr(secret_value="https://login.microsoftonline.com"),
+        description="Permissions authority url",
+        examples=["https://login.microsoftonline.com"],
+    )
 
 
-@dataclass
 class SharepointConnectionConfig(ConnectionConfig):
-    client_id: str
-    site: str
-    access_config: SharepointAccessConfig = enhanced_field(sensitive=True)
+    client_id: str = Field(description="Sharepoint app client ID")
+    site: str = Field(
+        description="Sharepoint site url. Process either base url e.g \
+                    https://[tenant].sharepoint.com  or relative sites \
+                    https://[tenant].sharepoint.com/sites/<site_name>. \
+                    To process all sites within the tenant pass a site url as \
+                    https://[tenant]-admin.sharepoint.com.\
+                    This requires the app to be registered at a tenant level"
+    )
+    access_config: Secret[SharepointAccessConfig]
     permissions_config: Optional[SharepointPermissionsConfig] = None
 
     @requires_dependencies(["office365"], extras="sharepoint")
@@ -80,7 +95,9 @@ class SharepointConnectionConfig(ConnectionConfig):
         from office365.sharepoint.client_context import ClientContext
 
         try:
-            credentials = ClientCredential(self.client_id, self.access_config.client_cred)
+            credentials = ClientCredential(
+                self.client_id, self.access_config.get_secret_value().client_cred
+            )
             site_client = ClientContext(self.site).with_credentials(credentials)
         except Exception as e:
             logger.error(f"Couldn't set Sharepoint client: {e}")
@@ -92,11 +109,12 @@ class SharepointConnectionConfig(ConnectionConfig):
         from msal import ConfidentialClientApplication
 
         try:
+            client_credential = self.permissions_config.permissions_client_cred.get_secret_value()
             app = ConfidentialClientApplication(
-                authority=f"{self.permissions_config.authority_url}/"
+                authority=f"{self.permissions_config.authority_url.get_secret_value()}/"
                 f"{self.permissions_config.permissions_tenant}",
                 client_id=self.permissions_config.permissions_application_id,
-                client_credential=self.permissions_config.permissions_client_cred,
+                client_credential=client_credential,
             )
             token = app.acquire_token_for_client(scopes=["https://graph.microsoft.com/.default"])
         except ValueError as exc:
@@ -119,13 +137,21 @@ class SharepointConnectionConfig(ConnectionConfig):
         return client
 
 
-@dataclass
 class SharepointIndexerConfig(IndexerConfig):
-    path: Optional[str] = None
-    recursive: bool = False
-    omit_files: bool = False
-    omit_pages: bool = False
-    omit_lists: bool = False
+    path: Optional[str] = Field(
+        defaul=None,
+        description="Path from which to start parsing files. If the connector is to \
+                process all sites within the tenant this filter will be applied to \
+                all sites document libraries.",
+    )
+    recursive: bool = Field(
+        default=False,
+        description="Recursively download files in their respective folders "
+        "otherwise stop at the files in provided folder level.",
+    )
+    omit_files: bool = Field(default=False, description="Don't process files.")
+    omit_pages: bool = Field(default=False, description="Don't process site pages.")
+    omit_lists: bool = Field(default=False, description="Don't process lists.")
 
 
 @dataclass
@@ -310,7 +336,7 @@ class SharepointIndexer(Indexer):
     def process_permissions(self) -> bool:
         return (
             self.connection_config.permissions_config.permissions_tenant
-            and self.connection_config.permissions_config.permissions_client_cred
+            and self.connection_config.permissions_config.permissions_client_cred.get_secret_value()
             and self.connection_config.permissions_config.permissions_application_id
         )
 
@@ -335,7 +361,6 @@ class SharepointIndexer(Indexer):
                 yield file_data
 
 
-@dataclass
 class SharepointDownloaderConfig(DownloaderConfig):
     pass
 

unstructured_ingest/v2/processes/connectors/singlestore.py

@@ -7,8 +7,8 @@ from typing import TYPE_CHECKING, Any, Optional
 import numpy as np
 import pandas as pd
 from dateutil import parser
+from pydantic import Field, Secret
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
 from unstructured_ingest.utils.data_prep import batch_generator
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.utils.table import convert_to_pandas_dataframe
@@ -33,18 +33,16 @@ if TYPE_CHECKING:
 CONNECTOR_TYPE = "singlestore"
 
 
-@dataclass
 class SingleStoreAccessConfig(AccessConfig):
-    password: Optional[str] = None
+    password: Optional[str] = Field(default=None, description="SingleStore password")
 
 
-@dataclass
 class SingleStoreConnectionConfig(ConnectionConfig):
-    host: Optional[str] = None
-    port: Optional[int] = None
-    user: Optional[str] = None
-    database: Optional[str] = None
-    access_config: SingleStoreAccessConfig = enhanced_field(sensitive=True)
+    host: Optional[str] = Field(default=None, description="SingleStore host")
+    port: Optional[int] = Field(default=None, description="SingleStore port")
+    user: Optional[str] = Field(default=None, description="SingleStore user")
+    database: Optional[str] = Field(default=None, description="SingleStore database")
+    access_config: Secret[SingleStoreAccessConfig]
 
     @requires_dependencies(["singlestoredb"], extras="singlestore")
     def get_connection(self) -> "Connection":
@@ -55,14 +53,13 @@ class SingleStoreConnectionConfig(ConnectionConfig):
             port=self.port,
             database=self.database,
             user=self.user,
-            password=self.access_config.password,
+            password=self.access_config.get_secret_value().password,
         )
         return conn
 
 
-@dataclass
 class SingleStoreUploadStagerConfig(UploadStagerConfig):
-    drop_empty_cols: bool = False
+    drop_empty_cols: bool = Field(default=False, description="Drop any columns that have no data")
 
 
 @dataclass
@@ -112,10 +109,9 @@ class SingleStoreUploadStager(UploadStager):
         return output_path
 
 
-@dataclass
 class SingleStoreUploaderConfig(UploaderConfig):
-    table_name: str
-    batch_size: int = 100
+    table_name: str = Field(description="SingleStore table to write contents to")
+    batch_size: int = Field(default=100, description="Batch size when writing to SingleStore")
 
 
 @dataclass