unstructured-ingest 0.0.3__py3-none-any.whl → 0.0.5__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/couchbase.py

@@ -0,0 +1,333 @@
+import hashlib
+import json
+import sys
+import time
+from dataclasses import dataclass, field
+from datetime import timedelta
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Generator, List
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.error import (
+    DestinationConnectionError,
+    SourceConnectionError,
+    SourceConnectionNetworkError,
+)
+from unstructured_ingest.utils.data_prep import batch_generator, flatten_dict
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    DownloadResponse,
+    FileData,
+    FileDataSourceMetadata,
+    Indexer,
+    IndexerConfig,
+    UploadContent,
+    Uploader,
+    UploaderConfig,
+    UploadStager,
+    UploadStagerConfig,
+    download_responses,
+)
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+
+if TYPE_CHECKING:
+    from couchbase.cluster import Cluster
+
+CONNECTOR_TYPE = "couchbase"
+SERVER_API_VERSION = "1"
+
+
+class CouchbaseAccessConfig(AccessConfig):
+    password: str = Field(description="The password for the Couchbase server")
+
+
+class CouchbaseConnectionConfig(ConnectionConfig):
+    username: str = Field(description="The username for the Couchbase server")
+    bucket: str = Field(description="The bucket to connect to on the Couchbase server")
+    connection_string: str = Field(
+        default="couchbase://localhost", description="The connection string of the Couchbase server"
+    )
+    scope: str = Field(
+        default="_default", description="The scope to connect to on the Couchbase server"
+    )
+    collection: str = Field(
+        default="_default", description="The collection to connect to on the Couchbase server"
+    )
+    connector_type: str = Field(default=CONNECTOR_TYPE, init=False)
+    access_config: Secret[CouchbaseAccessConfig]
+
+    @requires_dependencies(["couchbase"], extras="couchbase")
+    def connect_to_couchbase(self) -> "Cluster":
+        from couchbase.auth import PasswordAuthenticator
+        from couchbase.cluster import Cluster
+        from couchbase.options import ClusterOptions
+
+        auth = PasswordAuthenticator(self.username, self.access_config.get_secret_value().password)
+        options = ClusterOptions(auth)
+        options.apply_profile("wan_development")
+        cluster = Cluster(self.connection_string, options)
+        cluster.wait_until_ready(timedelta(seconds=5))
+        return cluster
+
+
+class CouchbaseUploadStagerConfig(UploadStagerConfig):
+    pass
+
+
+@dataclass
+class CouchbaseUploadStager(UploadStager):
+    upload_stager_config: CouchbaseUploadStagerConfig = field(
+        default_factory=lambda: CouchbaseUploadStagerConfig()
+    )
+
+    def run(
+        self,
+        elements_filepath: Path,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any,
+    ) -> Path:
+        with open(elements_filepath) as elements_file:
+            elements_contents = json.load(elements_file)
+
+        output_elements = []
+        for element in elements_contents:
+            new_doc = {
+                element["element_id"]: {
+                    "embedding": element.get("embeddings", None),
+                    "text": element.get("text", None),
+                    "metadata": element.get("metadata", None),
+                    "type": element.get("type", None),
+                }
+            }
+            output_elements.append(new_doc)
+
+        output_path = Path(output_dir) / Path(f"{output_filename}.json")
+        with open(output_path, "w") as output_file:
+            json.dump(output_elements, output_file)
+        return output_path
+
+
+class CouchbaseUploaderConfig(UploaderConfig):
+    batch_size: int = Field(default=50, description="Number of documents to upload per batch")
+
+
+@dataclass
+class CouchbaseUploader(Uploader):
+    connection_config: CouchbaseConnectionConfig
+    upload_config: CouchbaseUploaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def precheck(self) -> None:
+        try:
+            self.connection_config.connect_to_couchbase()
+        except Exception as e:
+            logger.error(f"Failed to validate connection {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
+
+    def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
+        elements = []
+        for content in contents:
+            with open(content.path) as elements_file:
+                elements.extend(json.load(elements_file))
+
+        logger.info(
+            f"writing {len(elements)} objects to destination "
+            f"bucket, {self.connection_config.bucket} "
+            f"at {self.connection_config.connection_string}",
+        )
+        cluster = self.connection_config.connect_to_couchbase()
+        bucket = cluster.bucket(self.connection_config.bucket)
+        scope = bucket.scope(self.connection_config.scope)
+        collection = scope.collection(self.connection_config.collection)
+
+        for chunk in batch_generator(elements, self.upload_config.batch_size):
+            collection.upsert_multi({doc_id: doc for doc in chunk for doc_id, doc in doc.items()})
+
+
+class CouchbaseIndexerConfig(IndexerConfig):
+    batch_size: int = Field(default=50, description="Number of documents to index per batch")
+
+
+@dataclass
+class CouchbaseIndexer(Indexer):
+    connection_config: CouchbaseConnectionConfig
+    index_config: CouchbaseIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def precheck(self) -> None:
+        try:
+            self.connection_config.connect_to_couchbase()
+        except Exception as e:
+            logger.error(f"Failed to validate connection {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
+
+    @requires_dependencies(["couchbase"], extras="couchbase")
+    def _get_doc_ids(self) -> List[str]:
+        query = (
+            f"SELECT META(d).id "
+            f"FROM `{self.connection_config.bucket}`."
+            f"`{self.connection_config.scope}`."
+            f"`{self.connection_config.collection}` as d"
+        )
+
+        max_attempts = 5
+        attempts = 0
+        while attempts < max_attempts:
+            try:
+                cluster = self.connection_config.connect_to_couchbase()
+                result = cluster.query(query)
+                document_ids = [row["id"] for row in result]
+                return document_ids
+            except Exception as e:
+                attempts += 1
+                time.sleep(3)
+                if attempts == max_attempts:
+                    raise SourceConnectionError(f"failed to get document ids: {e}")
+
+    def run(self, **kwargs: Any) -> Generator[FileData, None, None]:
+        ids = self._get_doc_ids()
+
+        id_batches = [
+            ids[i * self.index_config.batch_size : (i + 1) * self.index_config.batch_size]
+            for i in range(
+                (len(ids) + self.index_config.batch_size - 1) // self.index_config.batch_size
+            )
+        ]
+        for batch in id_batches:
+            # Make sure the hash is always a positive number to create identified
+            identified = str(hash(tuple(batch)) + sys.maxsize + 1)
+            yield FileData(
+                identifier=identified,
+                connector_type=CONNECTOR_TYPE,
+                metadata=FileDataSourceMetadata(
+                    url=f"{self.connection_config.connection_string}/"
+                    f"{self.connection_config.bucket}",
+                    date_processed=str(time.time()),
+                ),
+                additional_metadata={
+                    "ids": list(batch),
+                    "bucket": self.connection_config.bucket,
+                },
+            )
+
+
+class CouchbaseDownloaderConfig(DownloaderConfig):
+    fields: list[str] = field(default_factory=list)
+
+
+@dataclass
+class CouchbaseDownloader(Downloader):
+    connection_config: CouchbaseConnectionConfig
+    download_config: CouchbaseDownloaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+    def is_async(self) -> bool:
+        return False
+
+    def get_identifier(self, bucket: str, record_id: str) -> str:
+        f = f"{bucket}-{record_id}"
+        if self.download_config.fields:
+            f = "{}-{}".format(
+                f,
+                hashlib.sha256(",".join(self.download_config.fields).encode()).hexdigest()[:8],
+            )
+        return f
+
+    def map_cb_results(self, cb_results: dict) -> str:
+        doc_body = cb_results
+        flattened_dict = flatten_dict(dictionary=doc_body)
+        str_values = [str(value) for value in flattened_dict.values()]
+        concatenated_values = "\n".join(str_values)
+        return concatenated_values
+
+    def generate_download_response(
+        self, result: dict, bucket: str, file_data: FileData
+    ) -> DownloadResponse:
+        record_id = result["id"]
+        filename_id = self.get_identifier(bucket=bucket, record_id=record_id)
+        filename = f"{filename_id}.txt"
+        download_path = self.download_dir / Path(filename)
+        logger.debug(
+            f"Downloading results from bucket {bucket} and id {record_id} to {download_path}"
+        )
+        download_path.parent.mkdir(parents=True, exist_ok=True)
+        try:
+            with open(download_path, "w", encoding="utf8") as f:
+                f.write(self.map_cb_results(cb_results=result))
+        except Exception as e:
+            logger.error(
+                f"failed to download from bucket {bucket} "
+                f"and id {record_id} to {download_path}: {e}",
+                exc_info=True,
+            )
+            raise SourceConnectionNetworkError(f"failed to download file {file_data.identifier}")
+        return DownloadResponse(
+            file_data=FileData(
+                identifier=filename_id,
+                connector_type=CONNECTOR_TYPE,
+                metadata=FileDataSourceMetadata(
+                    version=None,
+                    date_processed=str(time.time()),
+                    record_locator={
+                        "connection_string": self.connection_config.connection_string,
+                        "bucket": bucket,
+                        "scope": self.connection_config.scope,
+                        "collection": self.connection_config.collection,
+                        "document_id": record_id,
+                    },
+                ),
+            ),
+            path=download_path,
+        )
+
+    def run(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        bucket_name: str = file_data.additional_metadata["bucket"]
+        ids: list[str] = file_data.additional_metadata["ids"]
+
+        cluster = self.connection_config.connect_to_couchbase()
+        bucket = cluster.bucket(bucket_name)
+        scope = bucket.scope(self.connection_config.scope)
+        collection = scope.collection(self.connection_config.collection)
+
+        download_resp = self.process_all_doc_ids(ids, collection, bucket_name, file_data)
+        return list(download_resp)
+
+    def process_doc_id(self, doc_id, collection, bucket_name, file_data):
+        result = collection.get(doc_id)
+        return self.generate_download_response(
+            result=result.content_as[dict], bucket=bucket_name, file_data=file_data
+        )
+
+    def process_all_doc_ids(self, ids, collection, bucket_name, file_data):
+        for doc_id in ids:
+            yield self.process_doc_id(doc_id, collection, bucket_name, file_data)
+
+    async def run_async(self, file_data: FileData, **kwargs: Any) -> download_responses:
+        raise NotImplementedError()
+
+
+couchbase_destination_entry = DestinationRegistryEntry(
+    connection_config=CouchbaseConnectionConfig,
+    uploader=CouchbaseUploader,
+    uploader_config=CouchbaseUploaderConfig,
+    upload_stager=CouchbaseUploadStager,
+    upload_stager_config=CouchbaseUploadStagerConfig,
+)
+
+couchbase_source_entry = SourceRegistryEntry(
+    connection_config=CouchbaseConnectionConfig,
+    indexer=CouchbaseIndexer,
+    indexer_config=CouchbaseIndexerConfig,
+    downloader=CouchbaseDownloader,
+    downloader_config=CouchbaseDownloaderConfig,
+)

unstructured_ingest/v2/processes/connectors/databricks_volumes.py

@@ -1,8 +1,9 @@
 import os
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Optional
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from pydantic import Field, Secret
+
 from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces import (
@@ -21,45 +22,99 @@ if TYPE_CHECKING:
 CONNECTOR_TYPE = "databricks_volumes"
 
 
-@dataclass
 class DatabricksVolumesAccessConfig(AccessConfig):
-    account_id: Optional[str] = None
-    username: Optional[str] = None
-    password: Optional[str] = None
-    client_id: Optional[str] = None
-    client_secret: Optional[str] = None
-    token: Optional[str] = None
+    account_id: Optional[str] = Field(
+        default=None,
+        description="The Databricks account ID for the Databricks "
+        "accounts endpoint. Only has effect when Host is "
+        "either https://accounts.cloud.databricks.com/ (AWS), "
+        "https://accounts.azuredatabricks.net/ (Azure), "
+        "or https://accounts.gcp.databricks.com/ (GCP).",
+    )
+    username: Optional[str] = Field(
+        default=None,
+        description="The Databricks username part of basic authentication. "
+        "Only possible when Host is *.cloud.databricks.com (AWS).",
+    )
+    password: Optional[str] = Field(
+        default=None,
+        description="The Databricks password part of basic authentication. "
+        "Only possible when Host is *.cloud.databricks.com (AWS).",
+    )
+    client_id: Optional[str] = Field(default=None)
+    client_secret: Optional[str] = Field(default=None)
+    token: Optional[str] = Field(
+        default=None,
+        description="The Databricks personal access token (PAT) (AWS, Azure, and GCP) or "
+        "Azure Active Directory (Azure AD) token (Azure).",
+    )
     profile: Optional[str] = None
-    azure_workspace_resource_id: Optional[str] = None
-    azure_client_secret: Optional[str] = None
-    azure_client_id: Optional[str] = None
-    azure_tenant_id: Optional[str] = None
-    azure_environment: Optional[str] = None
-    auth_type: Optional[str] = None
+    azure_workspace_resource_id: Optional[str] = Field(
+        default=None,
+        description="The Azure Resource Manager ID for the Azure Databricks workspace, "
+        "which is exchanged for a Databricks host URL.",
+    )
+    azure_client_secret: Optional[str] = Field(
+        default=None, description="The Azure AD service principal’s client secret."
+    )
+    azure_client_id: Optional[str] = Field(
+        default=None, description="The Azure AD service principal’s application ID."
+    )
+    azure_tenant_id: Optional[str] = Field(
+        default=None, description="The Azure AD service principal’s tenant ID."
+    )
+    azure_environment: Optional[str] = Field(
+        default=None,
+        description="The Azure environment type for a " "specific set of API endpoints",
+        examples=["Public", "UsGov", "China", "Germany"],
+    )
+    auth_type: Optional[str] = Field(
+        default=None,
+        description="When multiple auth attributes are available in the "
+        "environment, use the auth type specified by this "
+        "argument. This argument also holds the currently "
+        "selected auth.",
+    )
     cluster_id: Optional[str] = None
     google_credentials: Optional[str] = None
     google_service_account: Optional[str] = None
 
 
-@dataclass
+SecretDatabricksVolumesAccessConfig = Secret[DatabricksVolumesAccessConfig]
+
+
 class DatabricksVolumesConnectionConfig(ConnectionConfig):
-    access_config: DatabricksVolumesAccessConfig = enhanced_field(
-        default_factory=DatabricksVolumesAccessConfig, sensitive=True
+    access_config: SecretDatabricksVolumesAccessConfig = Field(
+        default_factory=lambda: SecretDatabricksVolumesAccessConfig(
+            secret_value=DatabricksVolumesAccessConfig()
+        )
+    )
+    host: Optional[str] = Field(
+        default=None,
+        description="The Databricks host URL for either the "
+        "Databricks workspace endpoint or the "
+        "Databricks accounts endpoint.",
     )
-    host: Optional[str] = None
 
 
-@dataclass
 class DatabricksVolumesUploaderConfig(UploaderConfig):
-    volume: str
-    catalog: str
-    volume_path: Optional[str] = None
-    overwrite: bool = False
-    schema: str = "default"
+    volume: str = Field(description="Name of volume in the Unity Catalog")
+    catalog: str = Field(description="Name of the catalog in the Databricks Unity Catalog service")
+    volume_path: Optional[str] = Field(
+        default=None, description="Optional path within the volume to write to"
+    )
+    overwrite: bool = Field(
+        default=False, description="If true, an existing file will be overwritten."
+    )
+    databricks_schema: str = Field(
+        default="default",
+        alias="schema",
+        description="Schema associated with the volume to write to in the Unity Catalog service",
+    )
 
     @property
     def path(self) -> str:
-        path = f"/Volumes/{self.catalog}/{self.schema}/{self.volume}"
+        path = f"/Volumes/{self.catalog}/{self.databricks_schema}/{self.volume}"
         if self.volume_path:
             path = f"{path}/{self.volume_path}"
         return path
@@ -70,19 +125,19 @@ class DatabricksVolumesUploader(Uploader):
     connector_type: str = CONNECTOR_TYPE
     upload_config: DatabricksVolumesUploaderConfig
     connection_config: DatabricksVolumesConnectionConfig
-    client: Optional["WorkspaceClient"] = field(init=False, default=None)
 
     @requires_dependencies(dependencies=["databricks.sdk"], extras="databricks-volumes")
-    def __post_init__(self) -> "WorkspaceClient":
+    def get_client(self) -> "WorkspaceClient":
         from databricks.sdk import WorkspaceClient
 
-        self.client = WorkspaceClient(
-            host=self.connection_config.host, **self.connection_config.access_config.to_dict()
+        return WorkspaceClient(
+            host=self.connection_config.host,
+            **self.connection_config.access_config.get_secret_value().dict(),
         )
 
     def precheck(self) -> None:
         try:
-            assert self.client.current_user.me().active
+            assert self.get_client().current_user.me().active
         except Exception as e:
             logger.error(f"failed to validate connection: {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
@@ -91,7 +146,7 @@ class DatabricksVolumesUploader(Uploader):
         for content in contents:
             with open(content.path, "rb") as elements_file:
                 output_path = os.path.join(self.upload_config.path, content.path.name)
-                self.client.files.upload(
+                self.get_client().files.upload(
                     file_path=output_path,
                     contents=elements_file,
                     overwrite=self.upload_config.overwrite,

unstructured_ingest/v2/processes/connectors/elasticsearch.py

@@ -5,9 +5,10 @@ import uuid
 from dataclasses import dataclass, field
 from pathlib import Path
 from time import time
-from typing import TYPE_CHECKING, Any, Generator, Optional
+from typing import TYPE_CHECKING, Any, Generator, Optional, Union
+
+from pydantic import BaseModel, Field, Secret, SecretStr
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin, enhanced_field
 from unstructured_ingest.error import (
     DestinationConnectionError,
     SourceConnectionError,
@@ -44,57 +45,74 @@ if TYPE_CHECKING:
 CONNECTOR_TYPE = "elasticsearch"
 
 
-@dataclass
 class ElasticsearchAccessConfig(AccessConfig):
-    password: Optional[str] = None
-    api_key: Optional[str] = enhanced_field(default=None, overload_name="es_api_key")
-    bearer_auth: Optional[str] = None
-    ssl_assert_fingerprint: Optional[str] = None
-
-
-@dataclass
-class ElasticsearchClientInput(EnhancedDataClassJsonMixin):
+    password: Optional[str] = Field(
+        default=None, description="password when using basic auth or connecting to a cloud instance"
+    )
+    es_api_key: Optional[str] = Field(default=None, description="api key used for authentication")
+    bearer_auth: Optional[str] = Field(
+        default=None, description="bearer token used for HTTP bearer authentication"
+    )
+    ssl_assert_fingerprint: Optional[str] = Field(
+        default=None, description="SHA256 fingerprint value"
+    )
+
+
+class ElasticsearchClientInput(BaseModel):
     hosts: Optional[list[str]] = None
     cloud_id: Optional[str] = None
-    ca_certs: Optional[str] = None
-    basic_auth: Optional[tuple[str, str]] = enhanced_field(sensitive=True, default=None)
-    api_key: Optional[str] = enhanced_field(sensitive=True, default=None)
+    ca_certs: Optional[Path] = None
+    basic_auth: Optional[Secret[tuple[str, str]]] = None
+    api_key: Optional[Union[Secret[tuple[str, str]], SecretStr]] = None
 
 
-@dataclass
 class ElasticsearchConnectionConfig(ConnectionConfig):
-    hosts: Optional[list[str]] = None
-    username: Optional[str] = None
-    cloud_id: Optional[str] = None
-    api_key_id: Optional[str] = None
-    ca_certs: Optional[str] = None
-    access_config: ElasticsearchAccessConfig = enhanced_field(sensitive=True)
+    hosts: Optional[list[str]] = Field(
+        default=None,
+        description="list of the Elasticsearch hosts to connect to",
+        examples=["http://localhost:9200"],
+    )
+    username: Optional[str] = Field(default=None, description="username when using basic auth")
+    cloud_id: Optional[str] = Field(default=None, description="id used to connect to Elastic Cloud")
+    api_key_id: Optional[str] = Field(
+        default=None,
+        description="id associated with api key used for authentication: "
+        "https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html",  # noqa: E501
+    )
+    ca_certs: Optional[Path] = None
+    access_config: Secret[ElasticsearchAccessConfig]
 
     def get_client_kwargs(self) -> dict:
         # Update auth related fields to conform to what the SDK expects based on the
         # supported methods:
         # https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/connecting.html
-        client_input = ElasticsearchClientInput()
+        client_input_kwargs: dict[str, Any] = {}
+        access_config = self.access_config.get_secret_value()
         if self.hosts:
-            client_input.hosts = self.hosts
+            client_input_kwargs["hosts"] = self.hosts
         if self.cloud_id:
-            client_input.cloud_id = self.cloud_id
+            client_input_kwargs["cloud_id"] = self.cloud_id
         if self.ca_certs:
-            client_input.ca_certs = self.ca_certs
-        if self.access_config.password and (
-            self.cloud_id or self.ca_certs or self.access_config.ssl_assert_fingerprint
+            client_input_kwargs["ca_certs"] = self.ca_certs
+        if access_config.password and (
+            self.cloud_id or self.ca_certs or access_config.ssl_assert_fingerprint
         ):
-            client_input.basic_auth = ("elastic", self.access_config.password)
-        elif not self.cloud_id and self.username and self.access_config.password:
-            client_input.basic_auth = (self.username, self.access_config.password)
-        elif self.access_config.api_key and self.api_key_id:
-            client_input.api_key = (self.api_key_id, self.access_config.api_key)
-        elif self.access_config.api_key:
-            client_input.api_key = self.access_config.api_key
-        logger.debug(
-            f"Elasticsearch client inputs mapped to: {client_input.to_dict(redact_sensitive=True)}"
+            client_input_kwargs["basic_auth"] = ("elastic", access_config.password)
+        elif not self.cloud_id and self.username and access_config.password:
+            client_input_kwargs["basic_auth"] = (self.username, access_config.password)
+        elif access_config.es_api_key and self.api_key_id:
+            client_input_kwargs["api_key"] = (self.api_key_id, access_config.es_api_key)
+        elif access_config.es_api_key:
+            client_input_kwargs["api_key"] = access_config.es_api_key
+        client_input = ElasticsearchClientInput(**client_input_kwargs)
+        logger.debug(f"Elasticsearch client inputs mapped to: {client_input.dict()}")
+        client_kwargs = client_input.dict()
+        client_kwargs["basic_auth"] = (
+            client_input.basic_auth.get_secret_value() if client_input.basic_auth else None
+        )
+        client_kwargs["api_key"] = (
+            client_input.api_key.get_secret_value() if client_input.api_key else None
         )
-        client_kwargs = client_input.to_dict(redact_sensitive=False)
         client_kwargs = {k: v for k, v in client_kwargs.items() if v is not None}
         return client_kwargs
 
@@ -114,7 +132,6 @@ class ElasticsearchConnectionConfig(ConnectionConfig):
             raise SourceConnectionError(f"failed to validate connection: {e}")
 
 
-@dataclass
 class ElasticsearchIndexerConfig(IndexerConfig):
     index_name: str
     batch_size: int = 100
@@ -186,7 +203,6 @@ class ElasticsearchIndexer(Indexer):
             )
 
 
-@dataclass
 class ElasticsearchDownloaderConfig(DownloaderConfig):
     fields: list[str] = field(default_factory=list)
 
@@ -292,9 +308,10 @@ class ElasticsearchDownloader(Downloader):
         return download_responses
 
 
-@dataclass
 class ElasticsearchUploadStagerConfig(UploadStagerConfig):
-    index_name: str
+    index_name: str = Field(
+        description="Name of the Elasticsearch index to pull data from, or upload data to."
+    )
 
 
 @dataclass
@@ -333,11 +350,19 @@ class ElasticsearchUploadStager(UploadStager):
         return output_path
 
 
-@dataclass
 class ElasticsearchUploaderConfig(UploaderConfig):
-    index_name: str
-    batch_size_bytes: int = 15_000_000
-    num_threads: int = 4
+    index_name: str = Field(
+        description="Name of the Elasticsearch index to pull data from, or upload data to."
+    )
+    batch_size_bytes: int = Field(
+        default=15_000_000,
+        description="Size limit (in bytes) for each batch of items to be uploaded. Check"
+        " https://www.elastic.co/guide/en/elasticsearch/guide/current/bulk.html"
+        "#_how_big_is_too_big for more information.",
+    )
+    num_threads: int = Field(
+        default=4, description="Number of threads to be used while uploading content"
+    )
 
 
 @dataclass