unstructured-ingest 0.0.2.dev0__py3-none-any.whl → 0.0.4__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/mongodb.py

@@ -3,9 +3,10 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 
-from unstructured.__version__ import __version__ as unstructured_version
+from pydantic import Field, Secret
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest.__version__ import __version__ as unstructured_version
+from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.data_prep import batch_generator
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
@@ -30,25 +31,28 @@ CONNECTOR_TYPE = "mongodb"
 SERVER_API_VERSION = "1"
 
 
-@dataclass
 class MongoDBAccessConfig(AccessConfig):
-    uri: Optional[str] = None
+    uri: Optional[str] = Field(default=None, description="URI to user when connecting")
+
+
+SecretMongoDBAccessConfig = Secret[MongoDBAccessConfig]
 
 
-@dataclass
 class MongoDBConnectionConfig(ConnectionConfig):
-    access_config: MongoDBAccessConfig = enhanced_field(
-        sensitive=True, default_factory=MongoDBAccessConfig
+    access_config: SecretMongoDBAccessConfig = Field(
+        default_factory=lambda: SecretMongoDBAccessConfig(secret_value=MongoDBAccessConfig())
     )
-    host: Optional[str] = None
-    database: Optional[str] = None
-    collection: Optional[str] = None
-    port: int = 27017
-    batch_size: int = 100
-    connector_type: str = CONNECTOR_TYPE
+    host: Optional[str] = Field(
+        default=None,
+        description="hostname or IP address or Unix domain socket path of a single mongod or "
+        "mongos instance to connect to, or a list of hostnames",
+    )
+    database: Optional[str] = Field(default=None, description="database name to connect to")
+    collection: Optional[str] = Field(default=None, description="collection name to connect to")
+    port: int = Field(default=27017)
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
 
 
-@dataclass
 class MongoDBUploadStagerConfig(UploadStagerConfig):
     pass
 
@@ -76,20 +80,23 @@ class MongoDBUploadStager(UploadStager):
         return output_path
 
 
-@dataclass
 class MongoDBUploaderConfig(UploaderConfig):
-    batch_size: int = 100
+    batch_size: int = Field(default=100, description="Number of records per batch")
 
 
 @dataclass
 class MongoDBUploader(Uploader):
     upload_config: MongoDBUploaderConfig
     connection_config: MongoDBConnectionConfig
-    client: Optional["MongoClient"] = field(init=False)
     connector_type: str = CONNECTOR_TYPE
 
-    def __post_init__(self):
-        self.client = self.create_client()
+    def precheck(self) -> None:
+        try:
+            client = self.create_client()
+            client.admin.command("ping")
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
 
     @requires_dependencies(["pymongo"], extras="mongodb")
     def create_client(self) -> "MongoClient":
@@ -97,9 +104,11 @@ class MongoDBUploader(Uploader):
         from pymongo.driver_info import DriverInfo
         from pymongo.server_api import ServerApi
 
-        if self.connection_config.access_config.uri:
+        access_config = self.connection_config.access_config.get_secret_value()
+
+        if access_config.uri:
             return MongoClient(
-                self.connection_config.access_config.uri,
+                access_config.uri,
                 server_api=ServerApi(version=SERVER_API_VERSION),
                 driver=DriverInfo(name="unstructured", version=unstructured_version),
             )
@@ -123,7 +132,8 @@ class MongoDBUploader(Uploader):
             f"collection {self.connection_config.collection} "
             f"at {self.connection_config.host}",
         )
-        db = self.client[self.connection_config.database]
+        client = self.create_client()
+        db = client[self.connection_config.database]
         collection = db[self.connection_config.collection]
         for chunk in batch_generator(elements_dict, self.upload_config.batch_size):
             collection.insert_many(chunk)

unstructured_ingest/v2/processes/connectors/onedrive.py

@@ -1,13 +1,12 @@
 import json
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from pathlib import Path
 from time import time
 from typing import TYPE_CHECKING, Any, Generator, Optional
 
 from dateutil import parser
-from unstructured.documents.elements import DataSourceMetadata
+from pydantic import Field, Secret
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
 from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
@@ -17,6 +16,7 @@ from unstructured_ingest.v2.interfaces import (
     DownloaderConfig,
     DownloadResponse,
     FileData,
+    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
     SourceIdentifiers,
@@ -35,18 +35,23 @@ CONNECTOR_TYPE = "onedrive"
 MAX_MB_SIZE = 512_000_000
 
 
-@dataclass
 class OnedriveAccessConfig(AccessConfig):
-    client_cred: str
+    client_cred: str = Field(description="Microsoft App client secret")
 
 
-@dataclass
 class OnedriveConnectionConfig(ConnectionConfig):
-    client_id: str
-    user_pname: str
-    tenant: str = field(repr=False)
-    authority_url: Optional[str] = field(repr=False, default="https://login.microsoftonline.com")
-    access_config: OnedriveAccessConfig = enhanced_field(sensitive=True)
+    client_id: str = Field(description="Microsoft app client ID")
+    user_pname: str = Field(description="User principal name, usually is your Azure AD email.")
+    tenant: str = Field(
+        repr=False, description="ID or domain name associated with your Azure AD instance"
+    )
+    authority_url: Optional[str] = Field(
+        repr=False,
+        default="https://login.microsoftonline.com",
+        examples=["https://login.microsoftonline.com"],
+        description="Authentication token provider for Microsoft apps",
+    )
+    access_config: Secret[OnedriveAccessConfig]
 
     @requires_dependencies(["msal"], extras="onedrive")
     def get_token(self):
@@ -56,7 +61,7 @@ class OnedriveConnectionConfig(ConnectionConfig):
             app = ConfidentialClientApplication(
                 authority=f"{self.authority_url}/{self.tenant}",
                 client_id=self.client_id,
-                client_credential=self.access_config.client_cred,
+                client_credential=self.access_config.get_secret_value().client_cred,
             )
             token = app.acquire_token_for_client(scopes=["https://graph.microsoft.com/.default"])
         except ValueError as exc:
@@ -76,9 +81,8 @@ class OnedriveConnectionConfig(ConnectionConfig):
         return client
 
 
-@dataclass
 class OnedriveIndexerConfig(IndexerConfig):
-    path: Optional[str] = field(default="")
+    path: Optional[str] = Field(default="")
     recursive: bool = False
 
 
@@ -87,6 +91,18 @@ class OnedriveIndexer(Indexer):
     connection_config: OnedriveConnectionConfig
     index_config: OnedriveIndexerConfig
 
+    def precheck(self) -> None:
+        try:
+            token_resp: dict = self.connection_config.get_token()
+            if error := token_resp.get("error"):
+                raise SourceConnectionError(
+                    "{} ({})".format(error, token_resp.get("error_description"))
+                )
+            self.connection_config.get_client()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+
     def list_objects(self, folder, recursive) -> list["DriveItem"]:
         drive_items = folder.children.get().execute_query()
         files = [d for d in drive_items if d.is_file]
@@ -136,7 +152,7 @@ class OnedriveIndexer(Indexer):
             source_identifiers=SourceIdentifiers(
                 fullpath=server_path, filename=drive_item.name, rel_path=rel_path
             ),
-            metadata=DataSourceMetadata(
+            metadata=FileDataSourceMetadata(
                 url=drive_item.parent_reference.path + "/" + drive_item.name,
                 version=drive_item.etag,
                 date_modified=str(date_modified_dt.timestamp()) if date_modified_dt else None,
@@ -159,7 +175,6 @@ class OnedriveIndexer(Indexer):
             yield file_data
 
 
-@dataclass
 class OnedriveDownloaderConfig(DownloaderConfig):
     pass
 

unstructured_ingest/v2/processes/connectors/opensearch.py

@@ -1,7 +1,9 @@
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import TYPE_CHECKING, Optional
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin, enhanced_field
+from pydantic import BaseModel, Field, Secret
+
 from unstructured_ingest.error import (
     DestinationConnectionError,
 )
@@ -35,20 +37,28 @@ CONNECTOR_TYPE = "opensearch"
 heavily on the Elasticsearch connector code, inheriting the functionality as much as possible."""
 
 
-@dataclass
 class OpenSearchAccessConfig(AccessConfig):
-    password: Optional[str] = enhanced_field(default=None, sensitive=True)
-    use_ssl: bool = False
-    verify_certs: bool = False
-    ssl_show_warn: bool = False
-    ca_certs: Optional[str] = None
-    client_cert: Optional[str] = None
-    client_key: Optional[str] = None
-
-
-@dataclass
-class OpenSearchClientInput(EnhancedDataClassJsonMixin):
-    http_auth: Optional[tuple[str, str]] = enhanced_field(sensitive=True, default=None)
+    password: Optional[str] = Field(default=None, description="password when using basic auth")
+    use_ssl: bool = Field(default=False, description="use ssl for the connection")
+    verify_certs: bool = Field(default=False, description="whether to verify SSL certificates")
+    ssl_show_warn: bool = Field(
+        default=False, description="show warning when verify certs is disabled"
+    )
+    ca_certs: Optional[Path] = Field(default=None, description="path to CA bundle")
+    client_cert: Optional[Path] = Field(
+        default=None,
+        description="path to the file containing the private key and the certificate,"
+        " or cert only if using client_key",
+    )
+    client_key: Optional[Path] = Field(
+        default=None,
+        description="path to the file containing the private key"
+        " if using separate cert and key files",
+    )
+
+
+class OpenSearchClientInput(BaseModel):
+    http_auth: Secret[Optional[tuple[str, str]]] = None
     hosts: Optional[list[str]] = None
     use_ssl: bool = False
     verify_certs: bool = False
@@ -58,37 +68,41 @@ class OpenSearchClientInput(EnhancedDataClassJsonMixin):
     client_key: Optional[str] = None
 
 
-@dataclass
 class OpenSearchConnectionConfig(ConnectionConfig):
-    hosts: Optional[list[str]] = None
-    username: Optional[str] = None
-    access_config: OpenSearchAccessConfig = enhanced_field(sensitive=True)
+    hosts: Optional[list[str]] = Field(
+        default=None,
+        description="List of the OpenSearch hosts to connect",
+        examples=["http://localhost:9200"],
+    )
+    username: Optional[str] = Field(default=None, description="username when using basic auth")
+    access_config: Secret[OpenSearchAccessConfig]
 
     def get_client_kwargs(self) -> dict:
         # Update auth related fields to conform to what the SDK expects based on the
         # supported methods:
         # https://github.com/opensearch-project/opensearch-py/blob/main/opensearchpy/client/__init__.py
-        client_input = OpenSearchClientInput()
+        access_config = self.access_config.get_secret_value()
+        client_input_kwargs = {}
         if self.hosts:
-            client_input.hosts = self.hosts
-        if self.access_config.use_ssl:
-            client_input.use_ssl = self.access_config.use_ssl
-        if self.access_config.verify_certs:
-            client_input.verify_certs = self.access_config.verify_certs
-        if self.access_config.ssl_show_warn:
-            client_input.ssl_show_warn = self.access_config.ssl_show_warn
-        if self.access_config.ca_certs:
-            client_input.ca_certs = self.access_config.ca_certs
-        if self.access_config.client_cert:
-            client_input.client_cert = self.access_config.client_cert
-        if self.access_config.client_key:
-            client_input.client_key = self.access_config.client_key
-        if self.username and self.access_config.password:
-            client_input.http_auth = (self.username, self.access_config.password)
-        logger.debug(
-            f"OpenSearch client inputs mapped to: {client_input.to_dict(redact_sensitive=True)}"
-        )
-        client_kwargs = client_input.to_dict(redact_sensitive=False)
+            client_input_kwargs["hosts"] = self.hosts
+        if access_config.use_ssl:
+            client_input_kwargs["use_ssl"] = access_config.use_ssl
+        if access_config.verify_certs:
+            client_input_kwargs["verify_certs"] = access_config.verify_certs
+        if access_config.ssl_show_warn:
+            client_input_kwargs["ssl_show_warn"] = access_config.ssl_show_warn
+        if access_config.ca_certs:
+            client_input_kwargs["ca_certs"] = str(access_config.ca_certs)
+        if access_config.client_cert:
+            client_input_kwargs["client_cert"] = str(access_config.client_cert)
+        if access_config.client_key:
+            client_input_kwargs["client_key"] = str(access_config.client_key)
+        if self.username and access_config.password:
+            client_input_kwargs["http_auth"] = (self.username, access_config.password)
+        client_input = OpenSearchClientInput(**client_input_kwargs)
+        logger.debug(f"OpenSearch client inputs mapped to: {client_input.dict()}")
+        client_kwargs = client_input.dict()
+        client_kwargs["http_auth"] = client_input.http_auth.get_secret_value()
         client_kwargs = {k: v for k, v in client_kwargs.items() if v is not None}
         return client_kwargs
 
@@ -100,9 +114,14 @@ class OpenSearchConnectionConfig(ConnectionConfig):
         return OpenSearch(**self.get_client_kwargs())
 
 
+class OpenSearchIndexerConfig(ElasticsearchIndexerConfig):
+    pass
+
+
 @dataclass
 class OpenSearchIndexer(ElasticsearchIndexer):
     connection_config: OpenSearchConnectionConfig
+    index_config: OpenSearchIndexerConfig
     client: "OpenSearch" = field(init=False)
 
     @requires_dependencies(["opensearchpy"], extras="opensearch")
@@ -112,9 +131,14 @@ class OpenSearchIndexer(ElasticsearchIndexer):
         return scan
 
 
+class OpenSearchDownloaderConfig(ElasticsearchDownloaderConfig):
+    pass
+
+
 @dataclass
 class OpenSearchDownloader(ElasticsearchDownloader):
     connection_config: OpenSearchConnectionConfig
+    download_config: OpenSearchDownloaderConfig
     connector_type: str = CONNECTOR_TYPE
 
     @requires_dependencies(["opensearchpy"], extras="opensearch")
@@ -125,9 +149,14 @@ class OpenSearchDownloader(ElasticsearchDownloader):
         return AsyncOpenSearch, async_scan
 
 
+class OpenSearchUploaderConfig(ElasticsearchUploaderConfig):
+    pass
+
+
 @dataclass
 class OpenSearchUploader(ElasticsearchUploader):
     connection_config: OpenSearchConnectionConfig
+    upload_config: OpenSearchUploaderConfig
     connector_type: str = CONNECTOR_TYPE
 
     @requires_dependencies(["opensearchpy"], extras="opensearch")
@@ -137,19 +166,28 @@ class OpenSearchUploader(ElasticsearchUploader):
         return parallel_bulk
 
 
+class OpenSearchUploadStagerConfig(ElasticsearchUploadStagerConfig):
+    pass
+
+
+@dataclass
+class OpenSearchUploadStager(ElasticsearchUploadStager):
+    upload_stager_config: OpenSearchUploadStagerConfig
+
+
 opensearch_source_entry = SourceRegistryEntry(
     connection_config=OpenSearchConnectionConfig,
     indexer=OpenSearchIndexer,
-    indexer_config=ElasticsearchIndexerConfig,
+    indexer_config=OpenSearchIndexerConfig,
     downloader=OpenSearchDownloader,
-    downloader_config=ElasticsearchDownloaderConfig,
+    downloader_config=OpenSearchDownloaderConfig,
 )
 
 
 opensearch_destination_entry = DestinationRegistryEntry(
     connection_config=OpenSearchConnectionConfig,
-    upload_stager_config=ElasticsearchUploadStagerConfig,
-    upload_stager=ElasticsearchUploadStager,
-    uploader_config=ElasticsearchUploaderConfig,
+    upload_stager_config=OpenSearchUploadStagerConfig,
+    upload_stager=OpenSearchUploadStager,
+    uploader_config=OpenSearchUploaderConfig,
     uploader=OpenSearchUploader,
 )

unstructured_ingest/v2/processes/connectors/pinecone.py

@@ -5,12 +5,11 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 
-from unstructured.staging.base import flatten_dict
-from unstructured.utils import requires_dependencies
+from pydantic import Field, Secret
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
 from unstructured_ingest.error import DestinationConnectionError
-from unstructured_ingest.utils.data_prep import batch_generator
+from unstructured_ingest.utils.data_prep import batch_generator, flatten_dict
+from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -32,25 +31,31 @@ if TYPE_CHECKING:
 CONNECTOR_TYPE = "pinecone"
 
 
-@dataclass
 class PineconeAccessConfig(AccessConfig):
-    api_key: Optional[str] = enhanced_field(default=None, overload_name="pinecone_api_key")
+    pinecone_api_key: Optional[str] = Field(
+        default=None, description="API key for Pinecone.", alias="api_key"
+    )
+
+
+SecretPineconeAccessConfig = Secret[PineconeAccessConfig]
 
 
-@dataclass
 class PineconeConnectionConfig(ConnectionConfig):
-    index_name: str
-    environment: str
-    access_config: PineconeAccessConfig = enhanced_field(sensitive=True)
+    index_name: str = Field(description="Name of the index to connect to.")
+    environment: str = Field(description="Environment to connect to.")
+    access_config: SecretPineconeAccessConfig = Field(
+        default_factory=lambda: SecretPineconeAccessConfig(secret_value=PineconeAccessConfig())
+    )
 
     @requires_dependencies(["pinecone"], extras="pinecone")
     def get_index(self) -> "PineconeIndex":
         from pinecone import Pinecone
-        from unstructured import __version__ as unstructured_version
+
+        from unstructured_ingest import __version__ as unstructured_version
 
         pc = Pinecone(
-            api_key=self.access_config.api_key,
-            source_tag=f"unstructured=={unstructured_version}",
+            api_key=self.access_config.get_secret_value().pinecone_api_key,
+            source_tag=f"unstructured_ingest=={unstructured_version}",
         )
 
         index = pc.Index(self.index_name)
@@ -58,15 +63,13 @@ class PineconeConnectionConfig(ConnectionConfig):
         return index
 
 
-@dataclass
 class PineconeUploadStagerConfig(UploadStagerConfig):
     pass
 
 
-@dataclass
 class PineconeUploaderConfig(UploaderConfig):
-    batch_size: int = 100
-    num_of_processes: int = 4
+    batch_size: int = Field(default=100, description="Number of records per batch")
+    num_processes: int = Field(default=4, description="Number of processes to use for uploading")
 
 
 @dataclass
@@ -123,9 +126,12 @@ class PineconeUploader(Uploader):
     connection_config: PineconeConnectionConfig
     connector_type: str = CONNECTOR_TYPE
 
-    @DestinationConnectionError.wrap
-    def check_connection(self):
-        _ = self.connection_config.get_index()
+    def precheck(self):
+        try:
+            self.connection_config.get_index()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
 
     @requires_dependencies(["pinecone"], extras="pinecone")
     def upsert_batch(self, batch):
@@ -151,18 +157,18 @@ class PineconeUploader(Uploader):
             f" index named {self.connection_config.index_name}"
             f" environment named {self.connection_config.environment}"
             f" with batch size {self.upload_config.batch_size}"
-            f" with {self.upload_config.num_of_processes} (number of) processes"
+            f" with {self.upload_config.num_processes} (number of) processes"
         )
 
         pinecone_batch_size = self.upload_config.batch_size
 
-        if self.upload_config.num_of_processes == 1:
+        if self.upload_config.num_processes == 1:
             for batch in batch_generator(elements_dict, pinecone_batch_size):
                 self.upsert_batch(batch)  # noqa: E203
 
         else:
             with mp.Pool(
-                processes=self.upload_config.num_of_processes,
+                processes=self.upload_config.num_processes,
             ) as pool:
                 pool.map(
                     self.upsert_batch, list(batch_generator(elements_dict, pinecone_batch_size))

unstructured_ingest/v2/processes/connectors/salesforce.py

@@ -15,13 +15,12 @@ from email.utils import formatdate
 from pathlib import Path
 from string import Template
 from textwrap import dedent
-from typing import TYPE_CHECKING, Any, Generator, Type
+from typing import TYPE_CHECKING, Any, Generator, Optional, Type
 
 from dateutil import parser
-from unstructured.documents.elements import DataSourceMetadata
+from pydantic import Field, Secret
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
-from unstructured_ingest.error import SourceConnectionNetworkError
+from unstructured_ingest.error import SourceConnectionError, SourceConnectionNetworkError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
@@ -30,6 +29,7 @@ from unstructured_ingest.v2.interfaces import (
     DownloaderConfig,
     DownloadResponse,
     FileData,
+    FileDataSourceMetadata,
     Indexer,
     IndexerConfig,
     SourceIdentifiers,
@@ -75,49 +75,58 @@ $htmlbody
 )
 
 
-@dataclass
 class SalesforceAccessConfig(AccessConfig):
     consumer_key: str
-    private_key: str
+    private_key_path: Optional[Path] = Field(
+        default=None,
+        description="Path to the private key file. " "Key file is usually named server.key.",
+    )
+    private_key: Optional[str] = Field(default=None, description="Contents of the private key")
+
+    def model_post_init(self, __context: Any) -> None:
+        if self.private_key_path is None and self.private_key is None:
+            raise ValueError("either private_key or private_key_path must be set")
+        if self.private_key is not None and self.private_key_path is not None:
+            raise ValueError("only one of private_key or private_key_path must be set")
 
     @requires_dependencies(["cryptography"])
     def get_private_key_value_and_type(self) -> tuple[str, Type]:
         from cryptography.hazmat.primitives import serialization
 
-        try:
-            serialization.load_pem_private_key(data=self.private_key.encode("utf-8"), password=None)
-        except ValueError:
-            pass
-        else:
+        if self.private_key_path and self.private_key_path.is_file():
+            return str(self.private_key_path), Path
+        if self.private_key:
+            try:
+                serialization.load_pem_private_key(
+                    data=str(self.private_key).encode("utf-8"), password=None
+                )
+            except Exception as e:
+                raise ValueError(f"failed to validate private key data: {e}") from e
             return self.private_key, str
 
-        if Path(self.private_key).is_file():
-            return self.private_key, Path
-
         raise ValueError("private_key does not contain PEM private key or path")
 
 
-@dataclass
 class SalesforceConnectionConfig(ConnectionConfig):
     username: str
-    access_config: SalesforceAccessConfig = enhanced_field(sensitive=True)
+    access_config: Secret[SalesforceAccessConfig]
 
     @requires_dependencies(["simple_salesforce"], extras="salesforce")
     def get_client(self) -> "Salesforce":
         from simple_salesforce import Salesforce
 
-        pkey_value, pkey_type = self.access_config.get_private_key_value_and_type()
+        access_config = self.access_config.get_secret_value()
+        pkey_value, pkey_type = access_config.get_private_key_value_and_type()
 
         return Salesforce(
             username=self.username,
-            consumer_key=self.access_config.consumer_key,
+            consumer_key=access_config.consumer_key,
             privatekey_file=pkey_value if pkey_type is Path else None,
             privatekey=pkey_value if pkey_type is str else None,
             version=SALESFORCE_API_VERSION,
         )
 
 
-@dataclass
 class SalesforceIndexerConfig(IndexerConfig):
     categories: list[str]
 
@@ -132,6 +141,13 @@ class SalesforceIndexer(Indexer):
             if record_type not in ACCEPTED_CATEGORIES:
                 raise ValueError(f"{record_type} not currently an accepted Salesforce category")
 
+    def precheck(self) -> None:
+        try:
+            self.connection_config.get_client()
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+
     def get_file_extension(self, record_type) -> str:
         if record_type == "EmailMessage":
             extension = ".eml"
@@ -172,7 +188,7 @@ class SalesforceIndexer(Indexer):
                                 filename=record_with_extension,
                                 fullpath=f"{record['attributes']['type']}/{record_with_extension}",
                             ),
-                            metadata=DataSourceMetadata(
+                            metadata=FileDataSourceMetadata(
                                 url=record["attributes"]["url"],
                                 version=str(parser.parse(record["SystemModstamp"]).timestamp()),
                                 date_created=str(parser.parse(record["CreatedDate"]).timestamp()),
@@ -194,7 +210,6 @@ class SalesforceIndexer(Indexer):
             yield f
 
 
-@dataclass
 class SalesforceDownloaderConfig(DownloaderConfig):
     pass
 
@@ -207,11 +222,6 @@ class SalesforceDownloader(Downloader):
     )
     connector_type: str = CONNECTOR_TYPE
 
-    def get_download_path(self, file_data: FileData) -> Path:
-        rel_path = file_data.source_identifiers.relative_path
-        rel_path = rel_path[1:] if rel_path.startswith("/") else rel_path
-        return self.download_dir / Path(rel_path)
-
     def _xml_for_record(self, record: OrderedDict) -> str:
         """Creates partitionable xml file from a record"""
         import xml.etree.ElementTree as ET