unstructured-ingest 0.0.3__py3-none-any.whl → 0.0.4__py3-none-any.whl

unstructured_ingest/v2/pipeline/steps/embed.py

@@ -5,13 +5,11 @@ from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Optional, TypedDict
 
-from unstructured.staging.base import elements_to_dicts
-
 from unstructured_ingest.v2.interfaces import FileData
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
-from unstructured_ingest.v2.pipeline.utils import sterilize_dict
 from unstructured_ingest.v2.processes.embedder import Embedder
+from unstructured_ingest.v2.utils import serialize_base_model_json
 
 STEP_ID = "embed"
 
@@ -30,11 +28,7 @@ class EmbedStep(PipelineStep):
         return f"{self.identifier} ({self.process.config.embedding_provider})"
 
     def __post_init__(self):
-        config = (
-            sterilize_dict(self.process.config.to_dict(redact_sensitive=True))
-            if self.process.config
-            else None
-        )
+        config = self.process.config.json() if self.process.config else None
         logger.info(f"Created {self.identifier} with configs: {config}")
 
     def should_embed(self, filepath: Path, file_data: FileData) -> bool:
@@ -71,13 +65,13 @@ class EmbedStep(PipelineStep):
 
         self._save_output(
             output_filepath=str(output_filepath),
-            embedded_content=elements_to_dicts(embed_content_raw),
+            embedded_content=embed_content_raw,
         )
         return EmbedStepResponse(file_data_path=file_data_path, path=str(output_filepath))
 
     def get_hash(self, extras: Optional[list[str]]) -> str:
-        hashable_string = json.dumps(
-            self.process.config.to_dict(), sort_keys=True, ensure_ascii=True
+        hashable_string = serialize_base_model_json(
+            model=self.process.config, sort_keys=True, ensure_ascii=True
         )
         if extras:
             hashable_string += "".join(extras)

unstructured_ingest/v2/pipeline/steps/filter.py

@@ -5,7 +5,6 @@ from typing import Callable, Optional
 from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
-from unstructured_ingest.v2.pipeline.utils import sterilize_dict
 from unstructured_ingest.v2.processes.filter import Filterer
 
 STEP_ID = "filter"
@@ -17,11 +16,7 @@ class FilterStep(PipelineStep):
     identifier: str = STEP_ID
 
     def __post_init__(self):
-        config = (
-            sterilize_dict(self.process.config.to_dict(redact_sensitive=True))
-            if self.process.config
-            else None
-        )
+        config = self.process.config.json() if self.process.config else None
         logger.info(f"Created {self.identifier} with configs: {config}")
 
     async def _run_async(self, fn: Callable, file_data_path: str, **kwargs) -> Optional[dict]:

unstructured_ingest/v2/pipeline/steps/index.py

@@ -6,7 +6,7 @@ from typing import Generator, Optional, TypeVar
 from unstructured_ingest.v2.interfaces.indexer import Indexer
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
-from unstructured_ingest.v2.pipeline.utils import sterilize_dict
+from unstructured_ingest.v2.utils import serialize_base_model_json
 
 IndexerT = TypeVar("IndexerT", bound=Indexer)
 
@@ -22,15 +22,9 @@ class IndexStep(PipelineStep):
         return f"{self.identifier} ({self.process.__class__.__name__})"
 
     def __post_init__(self):
-        config = (
-            sterilize_dict(self.process.index_config.to_dict(redact_sensitive=True))
-            if self.process.index_config
-            else None
-        )
+        config = self.process.index_config.json() if self.process.index_config else None
         connection_config = (
-            sterilize_dict(self.process.connection_config.to_dict(redact_sensitive=True))
-            if self.process.connection_config
-            else None
+            self.process.connection_config.json() if self.process.connection_config else None
         )
         logger.info(
             f"Created {self.identifier} with configs: {config}, "
@@ -55,7 +49,17 @@ class IndexStep(PipelineStep):
                 continue
 
     def get_hash(self, extras: Optional[list[str]]) -> str:
-        hashable_string = json.dumps(self.process.index_config.to_dict())
+        index_config_dict = json.loads(
+            serialize_base_model_json(model=self.process.index_config, sort_keys=True)
+        )
+        connection_config_dict = json.loads(
+            serialize_base_model_json(model=self.process.connection_config, sort_keys=True)
+        )
+        hashable_dict = {
+            "index_config": index_config_dict,
+            "connection_config": connection_config_dict,
+        }
+        hashable_string = json.dumps(hashable_dict, sort_keys=True)
         if extras:
             hashable_string += "".join(extras)
         return hashlib.sha256(hashable_string.encode()).hexdigest()[:12]

unstructured_ingest/v2/pipeline/steps/partition.py

@@ -8,8 +8,8 @@ from typing import Callable, Optional, TypedDict
 from unstructured_ingest.v2.interfaces import FileData
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
-from unstructured_ingest.v2.pipeline.utils import sterilize_dict
 from unstructured_ingest.v2.processes.partitioner import Partitioner
+from unstructured_ingest.v2.utils import serialize_base_model_json
 
 STEP_ID = "partition"
 
@@ -28,7 +28,7 @@ class PartitionStep(PipelineStep):
         return f"{self.identifier} ({self.process.config.strategy})"
 
     def __post_init__(self):
-        config = sterilize_dict(self.process.config.to_dict(redact_sensitive=True))
+        config = self.process.config.json()
         logger.info(f"Created {self.identifier} with configs: {config}")
 
     def should_partition(self, filepath: Path, file_data: FileData) -> bool:
@@ -56,7 +56,7 @@ class PartitionStep(PipelineStep):
         if not self.should_partition(filepath=output_filepath, file_data=file_data):
             logger.debug(f"Skipping partitioning, output already exists: {output_filepath}")
             return PartitionStepResponse(file_data_path=file_data_path, path=str(output_filepath))
-        fn_kwargs = {"filename": path, "metadata": file_data.metadata}
+        fn_kwargs = {"filename": path, "metadata": file_data.metadata.to_dict()}
         if not asyncio.iscoroutinefunction(fn):
             partitioned_content = fn(**fn_kwargs)
         elif semaphore := self.context.semaphore:
@@ -70,8 +70,8 @@ class PartitionStep(PipelineStep):
         return PartitionStepResponse(file_data_path=file_data_path, path=str(output_filepath))
 
     def get_hash(self, extras: Optional[list[str]]) -> str:
-        hashable_string = json.dumps(
-            self.process.config.to_dict(), sort_keys=True, ensure_ascii=True
+        hashable_string = serialize_base_model_json(
+            model=self.process.config, sort_keys=True, ensure_ascii=True
         )
         if extras:
             hashable_string += "".join(extras)

unstructured_ingest/v2/pipeline/steps/stage.py

@@ -1,6 +1,5 @@
 import asyncio
 import hashlib
-import json
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Callable, Optional, TypedDict
@@ -9,7 +8,7 @@ from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.interfaces.upload_stager import UploadStager
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
-from unstructured_ingest.v2.pipeline.utils import sterilize_dict
+from unstructured_ingest.v2.utils import serialize_base_model_json
 
 STEP_ID = "upload_stage"
 
@@ -29,9 +28,7 @@ class UploadStageStep(PipelineStep):
 
     def __post_init__(self):
         config = (
-            sterilize_dict(self.process.upload_stager_config.to_dict(redact_sensitive=True))
-            if self.process.upload_stager_config
-            else None
+            self.process.upload_stager_config.json() if self.process.upload_stager_config else None
         )
         self.cache_dir.mkdir(parents=True, exist_ok=True)
         logger.info(f"Created {self.identifier} with configs: {config}")
@@ -56,8 +53,8 @@ class UploadStageStep(PipelineStep):
         return UploadStageStepResponse(file_data_path=file_data_path, path=str(staged_output_path))
 
     def get_hash(self, extras: Optional[list[str]]) -> str:
-        hashable_string = json.dumps(
-            self.process.upload_stager_config.to_dict(), sort_keys=True, ensure_ascii=True
+        hashable_string = serialize_base_model_json(
+            model=self.process.upload_stager_config, sort_keys=True, ensure_ascii=True
         )
         if extras:
             hashable_string += "".join(extras)

unstructured_ingest/v2/pipeline/steps/uncompress.py

@@ -5,7 +5,6 @@ from typing import Callable, TypedDict
 from unstructured_ingest.v2.interfaces.file_data import FileData
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep
-from unstructured_ingest.v2.pipeline.utils import sterilize_dict
 from unstructured_ingest.v2.processes.uncompress import Uncompressor
 
 STEP_ID = "uncompress"
@@ -21,11 +20,7 @@ class UncompressStep(PipelineStep):
     identifier: str = STEP_ID
 
     def __post_init__(self):
-        config = (
-            sterilize_dict(self.process.config.to_dict(redact_sensitive=True))
-            if self.process.config
-            else None
-        )
+        config = self.process.config.json() if self.process.config else None
         logger.info(f"Created {self.identifier} with configs: {config}")
 
     def _run(self, path: str, file_data_path: str) -> list[UncompressStepResponse]:

unstructured_ingest/v2/pipeline/steps/upload.py

@@ -7,7 +7,6 @@ from unstructured_ingest.v2.interfaces import FileData
 from unstructured_ingest.v2.interfaces.uploader import UploadContent, Uploader
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.pipeline.interfaces import PipelineStep, iterable_input, timed
-from unstructured_ingest.v2.pipeline.utils import sterilize_dict
 
 STEP_ID = "upload"
 
@@ -26,15 +25,9 @@ class UploadStep(PipelineStep):
         return f"{self.identifier} ({self.process.__class__.__name__})"
 
     def __post_init__(self):
-        config = (
-            sterilize_dict(self.process.upload_config.to_dict(redact_sensitive=True))
-            if self.process.upload_config
-            else None
-        )
+        config = self.process.upload_config.json() if self.process.upload_config else None
         connection_config = (
-            sterilize_dict(self.process.connection_config.to_dict(redact_sensitive=True))
-            if self.process.connection_config
-            else None
+            self.process.connection_config.json() if self.process.connection_config else None
         )
         logger.info(
             f"Created {self.identifier} with configs: {config}, "

unstructured_ingest/v2/processes/__init__.py

@@ -0,0 +1,18 @@
+from .chunker import Chunker, ChunkerConfig
+from .embedder import Embedder, EmbedderConfig
+from .filter import Filterer, FiltererConfig
+from .partitioner import Partitioner, PartitionerConfig
+from .uncompress import UncompressConfig, Uncompressor
+
+__all__ = [
+    "Chunker",
+    "ChunkerConfig",
+    "Embedder",
+    "EmbedderConfig",
+    "Filterer",
+    "FiltererConfig",
+    "Partitioner",
+    "PartitionerConfig",
+    "Uncompressor",
+    "UncompressConfig",
+]

unstructured_ingest/v2/processes/chunker.py

@@ -3,29 +3,70 @@ from dataclasses import dataclass, fields
 from pathlib import Path
 from typing import Any, Optional
 
-from unstructured.chunking import dispatch
-from unstructured.documents.elements import Element, assign_and_map_hash_ids
-from unstructured.staging.base import dict_to_elements, elements_from_json
+from pydantic import BaseModel, Field, SecretStr
 
-from unstructured_ingest.enhanced_dataclass import EnhancedDataClassJsonMixin, enhanced_field
+from unstructured_ingest.utils.chunking import assign_and_map_hash_ids
+from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.interfaces.process import BaseProcess
 from unstructured_ingest.v2.logger import logger
 
+CHUNK_MAX_CHARS_DEFAULT: int = 500
+CHUNK_MULTI_PAGE_DEFAULT: bool = True
 
-@dataclass
-class ChunkerConfig(EnhancedDataClassJsonMixin):
-    chunking_strategy: Optional[str] = None
-    chunking_endpoint: Optional[str] = "https://api.unstructured.io/general/v0/general"
-    chunk_by_api: bool = False
-    chunk_api_key: Optional[str] = enhanced_field(default=None, sensitive=True)
 
-    chunk_combine_text_under_n_chars: Optional[int] = None
-    chunk_include_orig_elements: Optional[bool] = None
-    chunk_max_characters: Optional[int] = None
-    chunk_multipage_sections: Optional[bool] = None
-    chunk_new_after_n_chars: Optional[int] = None
-    chunk_overlap: Optional[int] = None
-    chunk_overlap_all: Optional[bool] = None
+class ChunkerConfig(BaseModel):
+    chunking_strategy: Optional[str] = Field(
+        default=None, description="The rule-set to use to form chunks. Omit to disable chunking."
+    )
+    chunking_endpoint: Optional[str] = Field(
+        default="https://api.unstructured.io/general/v0/general",
+        description="If chunking via api, use the following host.",
+    )
+    chunk_by_api: bool = Field(default=False, description="Flag to use api for chunking")
+    chunk_api_key: Optional[SecretStr] = Field(
+        default=None, description="API Key for chunking endpoint."
+    )
+
+    chunk_combine_text_under_n_chars: Optional[int] = Field(
+        default=None,
+        description="Combine consecutive chunks when the first does not exceed this length and"
+        " the second will fit without exceeding the hard-maximum length. Only"
+        " operative for 'by_title' chunking-strategy.",
+    )
+    chunk_include_orig_elements: Optional[bool] = Field(
+        default=None,
+        description="When chunking, add the original elements consolidated to form each chunk to"
+        " `.metadata.orig_elements` on that chunk.",
+    )
+    chunk_max_characters: int = Field(
+        default=CHUNK_MAX_CHARS_DEFAULT,
+        description="Hard maximum chunk length. No chunk will exceed this length. An oversized"
+        " element will be divided by text-splitting to fit this window.",
+    )
+    chunk_multipage_sections: bool = Field(
+        default=CHUNK_MULTI_PAGE_DEFAULT,
+        description="Ignore page boundaries when chunking such that elements from two different"
+        " pages can appear in the same chunk. Only operative for 'by_title'"
+        " chunking-strategy.",
+    )
+    chunk_new_after_n_chars: Optional[int] = Field(
+        default=None,
+        description="Soft-maximum chunk length. Another element will not be added to a chunk of"
+        " this length even when it would fit without exceeding the hard-maximum"
+        " length.",
+    )
+    chunk_overlap: Optional[int] = Field(
+        default=None,
+        description="Prefix chunk text with last overlap=N characters of prior chunk. Only"
+        " applies to oversized chunks divided by text-splitting. To apply overlap to"
+        " non-oversized chunks use the --overlap-all option.",
+    )
+    chunk_overlap_all: Optional[bool] = Field(
+        default=None,
+        description="Apply overlap to chunks formed from whole elements as well as those formed"
+        " by text-splitting oversized elements. Overlap length is take from --overlap"
+        " option value.",
+    )
 
     def to_chunking_kwargs(self) -> dict[str, Any]:
         return {
@@ -47,10 +88,14 @@ class Chunker(BaseProcess, ABC):
     def is_async(self) -> bool:
         return self.config.chunk_by_api
 
-    def run(self, elements_filepath: Path, **kwargs: Any) -> list[Element]:
+    @requires_dependencies(dependencies=["unstructured"])
+    def run(self, elements_filepath: Path, **kwargs: Any) -> list[dict]:
+        from unstructured.chunking import dispatch
+        from unstructured.staging.base import elements_from_json
+
         elements = elements_from_json(filename=str(elements_filepath))
         if not elements:
-            return elements
+            return [e.to_dict() for e in elements]
         local_chunking_strategies = ("basic", "by_title")
         if self.config.chunking_strategy not in local_chunking_strategies:
             logger.warning(
@@ -58,17 +103,19 @@ class Chunker(BaseProcess, ABC):
                     self.config.chunking_strategy, ", ".join(local_chunking_strategies)
                 )
             )
-            return elements
+            return [e.to_dict() for e in elements]
         chunked_elements = dispatch.chunk(elements=elements, **self.config.to_chunking_kwargs())
-        assign_and_map_hash_ids(chunked_elements)
-        return chunked_elements
+        chunked_elements_dicts = [e.to_dict() for e in chunked_elements]
+        chunked_elements_dicts = assign_and_map_hash_ids(elements=chunked_elements_dicts)
+        return chunked_elements_dicts
 
-    async def run_async(self, elements_filepath: Path, **kwargs: Any) -> list[Element]:
+    @requires_dependencies(dependencies=["unstructured_client"], extras="remote")
+    async def run_async(self, elements_filepath: Path, **kwargs: Any) -> list[dict]:
         from unstructured_client import UnstructuredClient
         from unstructured_client.models.shared import Files, PartitionParameters
 
         client = UnstructuredClient(
-            api_key_auth=self.config.chunk_api_key,
+            api_key_auth=self.config.chunk_api_key.get_secret_value(),
             server_url=self.config.chunking_endpoint,
         )
         partition_request = self.config.to_chunking_kwargs()
@@ -89,9 +136,8 @@ class Chunker(BaseProcess, ABC):
                 file_name=str(elements_filepath.resolve()),
             )
             filtered_partition_request["files"] = files
-        partition_params = PartitionParameters(**filtered_partition_request)
+            partition_params = PartitionParameters(**filtered_partition_request)
         resp = client.general.partition(partition_params)
-        elements_raw = resp.elements or []
-        elements = dict_to_elements(elements_raw)
-        assign_and_map_hash_ids(elements)
+        elements = resp.elements or []
+        elements = assign_and_map_hash_ids(elements=elements)
         return elements

unstructured_ingest/v2/processes/connector_registry.py

@@ -1,3 +1,4 @@
+from abc import ABC
 from dataclasses import dataclass
 from typing import Optional, Type, TypeVar
 
@@ -25,7 +26,12 @@ UploaderT = TypeVar("UploaderT", bound=Uploader)
 
 
 @dataclass
-class SourceRegistryEntry:
+class RegistryEntry(ABC):
+    pass
+
+
+@dataclass
+class SourceRegistryEntry(RegistryEntry):
     indexer: Type[IndexerT]
     downloader: Type[DownloaderT]
 
@@ -44,7 +50,7 @@ def add_source_entry(source_type: str, entry: SourceRegistryEntry):
 
 
 @dataclass
-class DestinationRegistryEntry:
+class DestinationRegistryEntry(RegistryEntry):
     uploader: Type[UploaderT]
     upload_stager: Optional[Type[UploadStagerT]] = None
 

unstructured_ingest/v2/processes/connectors/__init__.py

@@ -6,10 +6,14 @@ from unstructured_ingest.v2.processes.connector_registry import (
     add_source_entry,
 )
 
-from .astra import CONNECTOR_TYPE as ASTRA_CONNECTOR_TYPE
-from .astra import astra_destination_entry
+from .astradb import CONNECTOR_TYPE as ASTRA_DB_CONNECTOR_TYPE
+from .astradb import astra_db_destination_entry
+from .azure_cognitive_search import CONNECTOR_TYPE as AZURE_COGNTIVE_SEARCH_CONNECTOR_TYPE
+from .azure_cognitive_search import azure_cognitive_search_destination_entry
 from .chroma import CONNECTOR_TYPE as CHROMA_CONNECTOR_TYPE
 from .chroma import chroma_destination_entry
+from .couchbase import CONNECTOR_TYPE as COUCHBASE_CONNECTOR_TYPE
+from .couchbase import couchbase_destination_entry
 from .databricks_volumes import CONNECTOR_TYPE as DATABRICKS_VOLUMES_CONNECTOR_TYPE
 from .databricks_volumes import databricks_volumes_destination_entry
 from .elasticsearch import CONNECTOR_TYPE as ELASTICSEARCH_CONNECTOR_TYPE
@@ -39,10 +43,12 @@ from .sql import sql_destination_entry
 from .weaviate import CONNECTOR_TYPE as WEAVIATE_CONNECTOR_TYPE
 from .weaviate import weaviate_destination_entry
 
-add_destination_entry(destination_type=ASTRA_CONNECTOR_TYPE, entry=astra_destination_entry)
+add_destination_entry(destination_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_destination_entry)
 
 add_destination_entry(destination_type=CHROMA_CONNECTOR_TYPE, entry=chroma_destination_entry)
 
+add_destination_entry(destination_type=COUCHBASE_CONNECTOR_TYPE, entry=couchbase_destination_entry)
+
 add_source_entry(source_type=ELASTICSEARCH_CONNECTOR_TYPE, entry=elasticsearch_source_entry)
 add_destination_entry(
     destination_type=ELASTICSEARCH_CONNECTOR_TYPE, entry=elasticsearch_destination_entry
@@ -77,3 +83,7 @@ add_destination_entry(
     destination_type=SINGLESTORE_CONNECTOR_TYPE, entry=singlestore_destination_entry
 )
 add_destination_entry(destination_type=MILVUS_CONNECTOR_TYPE, entry=milvus_destination_entry)
+add_destination_entry(
+    destination_type=AZURE_COGNTIVE_SEARCH_CONNECTOR_TYPE,
+    entry=azure_cognitive_search_destination_entry,
+)

unstructured_ingest/v2/processes/connectors/{astra.py → astradb.py}

@@ -3,10 +3,10 @@ from dataclasses import dataclass, field
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional
 
-from unstructured import __name__ as integration_name
-from unstructured.__version__ import __version__ as integration_version
+from pydantic import Field, Secret
 
-from unstructured_ingest.enhanced_dataclass import enhanced_field
+from unstructured_ingest import __name__ as integration_name
+from unstructured_ingest.__version__ import __version__ as integration_version
 from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.data_prep import batch_generator
 from unstructured_ingest.utils.dep_check import requires_dependencies
@@ -28,30 +28,30 @@ from unstructured_ingest.v2.processes.connector_registry import (
 if TYPE_CHECKING:
     from astrapy.db import AstraDBCollection
 
-CONNECTOR_TYPE = "astra"
+CONNECTOR_TYPE = "astradb"
 
 
 @dataclass
-class AstraAccessConfig(AccessConfig):
-    token: str
-    api_endpoint: str
+class AstraDBAccessConfig(AccessConfig):
+    token: str = Field(description="Astra DB Token with access to the database.")
+    api_endpoint: str = Field(description="The API endpoint for the Astra DB.")
 
 
 @dataclass
-class AstraConnectionConfig(ConnectionConfig):
+class AstraDBConnectionConfig(ConnectionConfig):
     connection_type: str = CONNECTOR_TYPE
-    access_config: AstraAccessConfig = enhanced_field(sensitive=True)
+    access_config: Secret[AstraDBAccessConfig]
 
 
 @dataclass
-class AstraUploadStagerConfig(UploadStagerConfig):
+class AstraDBUploadStagerConfig(UploadStagerConfig):
     pass
 
 
 @dataclass
-class AstraUploadStager(UploadStager):
-    upload_stager_config: AstraUploadStagerConfig = field(
-        default_factory=lambda: AstraUploadStagerConfig()
+class AstraDBUploadStager(UploadStager):
+    upload_stager_config: AstraDBUploadStagerConfig = field(
+        default_factory=lambda: AstraDBUploadStagerConfig()
     )
 
     def conform_dict(self, element_dict: dict) -> dict:
@@ -80,19 +80,28 @@ class AstraUploadStager(UploadStager):
         return output_path
 
 
-@dataclass
-class AstraUploaderConfig(UploaderConfig):
-    collection_name: str
-    embedding_dimension: int
-    namespace: Optional[str] = None
-    requested_indexing_policy: Optional[dict[str, Any]] = None
-    batch_size: int = 20
+class AstraDBUploaderConfig(UploaderConfig):
+    collection_name: str = Field(
+        description="The name of the Astra DB collection. "
+        "Note that the collection name must only include letters, "
+        "numbers, and underscores."
+    )
+    embedding_dimension: int = Field(
+        default=384, description="The dimensionality of the embeddings"
+    )
+    namespace: Optional[str] = Field(default=None, description="The Astra DB connection namespace.")
+    requested_indexing_policy: Optional[dict[str, Any]] = Field(
+        default=None,
+        description="The indexing policy to use for the collection.",
+        examples=['{"deny": ["metadata"]}'],
+    )
+    batch_size: int = Field(default=20, description="Number of records per batch")
 
 
 @dataclass
-class AstraUploader(Uploader):
-    connection_config: AstraConnectionConfig
-    upload_config: AstraUploaderConfig
+class AstraDBUploader(Uploader):
+    connection_config: AstraDBConnectionConfig
+    upload_config: AstraDBUploaderConfig
     connector_type: str = CONNECTOR_TYPE
 
     def precheck(self) -> None:
@@ -102,7 +111,7 @@ class AstraUploader(Uploader):
             logger.error(f"Failed to validate connection {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")
 
-    @requires_dependencies(["astrapy"], extras="astra")
+    @requires_dependencies(["astrapy"], extras="astradb")
     def get_collection(self) -> "AstraDBCollection":
         from astrapy.db import AstraDB
 
@@ -111,14 +120,15 @@ class AstraUploader(Uploader):
         embedding_dimension = self.upload_config.embedding_dimension
         requested_indexing_policy = self.upload_config.requested_indexing_policy
 
-        # If the user has requested an indexing policy, pass it to the AstraDB
+        # If the user has requested an indexing policy, pass it to the Astra DB
         options = {"indexing": requested_indexing_policy} if requested_indexing_policy else None
 
         # Build the Astra DB object.
         # caller_name/version for AstraDB tracking
+        access_configs = self.connection_config.access_config.get_secret_value()
         astra_db = AstraDB(
-            api_endpoint=self.connection_config.access_config.api_endpoint,
-            token=self.connection_config.access_config.token,
+            api_endpoint=access_configs.api_endpoint,
+            token=access_configs.token,
             namespace=self.upload_config.namespace,
             caller_name=integration_name,
             caller_version=integration_version,
@@ -144,17 +154,17 @@ class AstraUploader(Uploader):
             f"collection {self.upload_config.collection_name}"
         )
 
-        astra_batch_size = self.upload_config.batch_size
+        astra_db_batch_size = self.upload_config.batch_size
         collection = self.get_collection()
 
-        for chunk in batch_generator(elements_dict, astra_batch_size):
+        for chunk in batch_generator(elements_dict, astra_db_batch_size):
             collection.insert_many(chunk)
 
 
-astra_destination_entry = DestinationRegistryEntry(
-    connection_config=AstraConnectionConfig,
-    upload_stager_config=AstraUploadStagerConfig,
-    upload_stager=AstraUploadStager,
-    uploader_config=AstraUploaderConfig,
-    uploader=AstraUploader,
+astra_db_destination_entry = DestinationRegistryEntry(
+    connection_config=AstraDBConnectionConfig,
+    upload_stager_config=AstraDBUploadStagerConfig,
+    upload_stager=AstraDBUploadStager,
+    uploader_config=AstraDBUploaderConfig,
+    uploader=AstraDBUploader,
 )