unstructured-ingest 1.2.32__py3-none-any.whl

unstructured_ingest/processes/connectors/kafka/kafka.py

@@ -0,0 +1,275 @@
+import json
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from time import time
+from typing import TYPE_CHECKING, Any, ContextManager, Generator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.data_types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
+from unstructured_ingest.error import (
+    DestinationConnectionError,
+    SourceConnectionError,
+    SourceConnectionNetworkError,
+    UnstructuredIngestError,
+    ValueError,
+)
+from unstructured_ingest.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Downloader,
+    DownloaderConfig,
+    DownloadResponse,
+    Indexer,
+    IndexerConfig,
+    Uploader,
+    UploaderConfig,
+)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.utils.data_prep import batch_generator
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from confluent_kafka import Consumer, Producer
+
+
+class KafkaAccessConfig(AccessConfig, ABC):
+    pass
+
+
+class KafkaConnectionConfig(ConnectionConfig, ABC):
+    access_config: Secret[KafkaAccessConfig]
+    bootstrap_server: str
+    port: int
+    group_id: str = Field(
+        description="A consumer group is a way to allow a pool of consumers "
+        "to divide the consumption of data over topics and partitions.",
+        default="default_group_id",
+    )
+
+    @abstractmethod
+    def get_consumer_configuration(self) -> dict:
+        pass
+
+    @abstractmethod
+    def get_producer_configuration(self) -> dict:
+        pass
+
+    @contextmanager
+    @requires_dependencies(["confluent_kafka"], extras="kafka")
+    def get_consumer(self) -> ContextManager["Consumer"]:
+        from confluent_kafka import Consumer
+
+        consumer = Consumer(self.get_consumer_configuration())
+        try:
+            logger.debug("kafka consumer connected")
+            yield consumer
+        finally:
+            consumer.close()
+
+    @requires_dependencies(["confluent_kafka"], extras="kafka")
+    def get_producer(self) -> "Producer":
+        from confluent_kafka import Producer
+
+        producer = Producer(self.get_producer_configuration())
+        return producer
+
+
+class KafkaIndexerConfig(IndexerConfig):
+    topic: str = Field(description="which topic to consume from")
+    num_messages_to_consume: Optional[int] = 100
+    timeout: Optional[float] = Field(default=3.0, description="polling timeout", ge=3.0)
+
+    def update_consumer(self, consumer: "Consumer") -> None:
+        consumer.subscribe([self.topic])
+
+
+@dataclass
+class KafkaIndexer(Indexer, ABC):
+    connection_config: KafkaConnectionConfig
+    index_config: KafkaIndexerConfig
+
+    @contextmanager
+    def get_consumer(self) -> ContextManager["Consumer"]:
+        with self.connection_config.get_consumer() as consumer:
+            self.index_config.update_consumer(consumer=consumer)
+            yield consumer
+
+    @requires_dependencies(["confluent_kafka"], extras="kafka")
+    def generate_messages(self) -> Generator[Any, None, None]:
+        from confluent_kafka import KafkaError
+
+        messages_consumed = 0
+        max_empty_polls = 10
+        empty_polls = 0
+        num_messages_to_consume = self.index_config.num_messages_to_consume
+        with self.get_consumer() as consumer:
+            while messages_consumed < num_messages_to_consume and empty_polls < max_empty_polls:
+                msg = consumer.poll(timeout=self.index_config.timeout)
+                if msg is None:
+                    logger.debug("No Kafka messages found")
+                    empty_polls += 1
+                    continue
+                if msg.error():
+                    if msg.error().code() == KafkaError._PARTITION_EOF:
+                        logger.info(
+                            "Reached end of partition for topic %s [%d] at offset %d"
+                            % (msg.topic(), msg.partition(), msg.offset())
+                        )
+                        break
+                    else:
+                        raise UnstructuredIngestError(msg.error())
+                try:
+                    empty_polls = 0
+                    messages_consumed += 1
+                    yield msg
+                finally:
+                    consumer.commit(asynchronous=False)
+
+    def generate_file_data(self, msg) -> FileData:
+        msg_content = msg.value().decode("utf8")
+        identifier = f"{msg.topic()}_{msg.partition()}_{msg.offset()}"
+        additional_metadata = {
+            "topic": msg.topic(),
+            "partition": msg.partition(),
+            "offset": msg.offset(),
+            "content": msg_content,
+        }
+        filename = f"{identifier}.txt"
+        return FileData(
+            identifier=identifier,
+            connector_type=self.connector_type,
+            source_identifiers=SourceIdentifiers(
+                filename=filename,
+                fullpath=filename,
+            ),
+            metadata=FileDataSourceMetadata(
+                date_processed=str(time()),
+            ),
+            additional_metadata=additional_metadata,
+            display_name=filename,
+        )
+
+    def run(self) -> Generator[FileData, None, None]:
+        for message in self.generate_messages():
+            yield self.generate_file_data(message)
+
+    async def run_async(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+        raise NotImplementedError()
+
+    def precheck(self):
+        try:
+            with self.get_consumer() as consumer:
+                # timeout needs at least 3 secs, more info:
+                # https://forum.confluent.io/t/kafkacat-connect-failure-to-confcloud-ssl/2513
+                cluster_meta = consumer.list_topics(timeout=5)
+                current_topics = [
+                    topic for topic in cluster_meta.topics if topic != "__consumer_offsets"
+                ]
+                if self.index_config.topic not in current_topics:
+                    raise SourceConnectionError(
+                        "expected topic '{}' not detected in cluster: '{}'".format(
+                            self.index_config.topic, ", ".join(current_topics)
+                        )
+                    )
+                logger.info(f"successfully checked available topics: {current_topics}")
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise SourceConnectionError(f"failed to validate connection: {e}")
+
+
+class KafkaDownloaderConfig(DownloaderConfig):
+    pass
+
+
+@dataclass
+class KafkaDownloader(Downloader, ABC):
+    connection_config: KafkaConnectionConfig
+    download_config: KafkaDownloaderConfig = field(default_factory=KafkaDownloaderConfig)
+    version: Optional[str] = None
+    source_url: Optional[str] = None
+
+    def run(self, file_data: FileData, **kwargs: Any) -> DownloadResponse:
+        source_identifiers = file_data.source_identifiers
+        if source_identifiers is None:
+            raise ValueError("FileData is missing source_identifiers")
+
+        # Build the download path using source_identifiers
+        download_path = Path(self.download_dir) / source_identifiers.relative_path
+        download_path.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            content = file_data.additional_metadata["content"]
+            with open(download_path, "w") as file:
+                file.write(content)
+        except Exception as e:
+            logger.error(f"Failed to download file {file_data.identifier}: {e}")
+            raise SourceConnectionNetworkError(f"failed to download file {file_data.identifier}")
+
+        return self.generate_download_response(file_data=file_data, download_path=download_path)
+
+
+class KafkaUploaderConfig(UploaderConfig):
+    batch_size: int = Field(default=100, description="Batch size")
+    topic: str = Field(description="which topic to write to")
+    timeout: Optional[float] = Field(
+        default=10.0, description="Timeout in seconds to flush batch of messages"
+    )
+
+
+@dataclass
+class KafkaUploader(Uploader, ABC):
+    connection_config: KafkaConnectionConfig
+    upload_config: KafkaUploaderConfig
+
+    def precheck(self):
+        try:
+            with self.connection_config.get_consumer() as consumer:
+                cluster_meta = consumer.list_topics(timeout=self.upload_config.timeout)
+                current_topics = [
+                    topic for topic in cluster_meta.topics if topic != "__consumer_offsets"
+                ]
+                logger.info(f"successfully checked available topics: {current_topics}")
+                if self.upload_config.topic not in current_topics:
+                    raise DestinationConnectionError(
+                        "expected topic '{}' not detected in cluster: '{}'".format(
+                            self.upload_config.topic, ", ".join(current_topics)
+                        )
+                    )
+
+        except Exception as e:
+            logger.error(f"failed to validate connection: {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
+
+    def produce_batch(self, elements: list[dict]) -> None:
+        producer = self.connection_config.get_producer()
+        failed_producer = False
+
+        def acked(err, msg):
+            nonlocal failed_producer
+            if err is not None:
+                failed_producer = True
+                logger.error("Failed to deliver kafka message: %s: %s" % (str(msg), str(err)))
+
+        for element in elements:
+            producer.produce(
+                topic=self.upload_config.topic,
+                value=json.dumps(element),
+                callback=acked,
+            )
+
+        while producer_len := len(producer):
+            logger.debug(f"another iteration of kafka producer flush. Queue length: {producer_len}")
+            producer.flush(timeout=self.upload_config.timeout)
+        if failed_producer:
+            raise UnstructuredIngestError("failed to produce all kafka messages in batch")
+
+    def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        for element_batch in batch_generator(data, batch_size=self.upload_config.batch_size):
+            self.produce_batch(elements=element_batch)

unstructured_ingest/processes/connectors/kafka/local.py

@@ -0,0 +1,103 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.processes.connector_registry import (
+    DestinationRegistryEntry,
+    SourceRegistryEntry,
+)
+from unstructured_ingest.processes.connectors.kafka.kafka import (
+    KafkaAccessConfig,
+    KafkaConnectionConfig,
+    KafkaDownloader,
+    KafkaDownloaderConfig,
+    KafkaIndexer,
+    KafkaIndexerConfig,
+    KafkaUploader,
+    KafkaUploaderConfig,
+)
+
+if TYPE_CHECKING:
+    pass
+
+CONNECTOR_TYPE = "kafka-local"
+
+
+class LocalKafkaAccessConfig(KafkaAccessConfig):
+    pass
+
+
+class LocalKafkaConnectionConfig(KafkaConnectionConfig):
+    access_config: Secret[LocalKafkaAccessConfig] = Field(
+        default=LocalKafkaAccessConfig(), validate_default=True
+    )
+
+    def get_consumer_configuration(self) -> dict:
+        bootstrap = self.bootstrap_server
+        port = self.port
+
+        conf = {
+            "bootstrap.servers": f"{bootstrap}:{port}",
+            "group.id": self.group_id,
+            "enable.auto.commit": "false",
+            "auto.offset.reset": "earliest",
+        }
+        return conf
+
+    def get_producer_configuration(self) -> dict:
+        bootstrap = self.bootstrap_server
+        port = self.port
+
+        conf = {
+            "bootstrap.servers": f"{bootstrap}:{port}",
+        }
+        return conf
+
+
+class LocalKafkaIndexerConfig(KafkaIndexerConfig):
+    pass
+
+
+@dataclass
+class LocalKafkaIndexer(KafkaIndexer):
+    connection_config: LocalKafkaConnectionConfig
+    index_config: LocalKafkaIndexerConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+class LocalKafkaDownloaderConfig(KafkaDownloaderConfig):
+    pass
+
+
+@dataclass
+class LocalKafkaDownloader(KafkaDownloader):
+    connection_config: LocalKafkaConnectionConfig
+    download_config: LocalKafkaDownloaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+class LocalKafkaUploaderConfig(KafkaUploaderConfig):
+    pass
+
+
+@dataclass
+class LocalKafkaUploader(KafkaUploader):
+    connection_config: LocalKafkaConnectionConfig
+    upload_config: LocalKafkaUploaderConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+kafka_local_source_entry = SourceRegistryEntry(
+    connection_config=LocalKafkaConnectionConfig,
+    indexer=LocalKafkaIndexer,
+    indexer_config=LocalKafkaIndexerConfig,
+    downloader=LocalKafkaDownloader,
+    downloader_config=LocalKafkaDownloaderConfig,
+)
+
+kafka_local_destination_entry = DestinationRegistryEntry(
+    connection_config=LocalKafkaConnectionConfig,
+    uploader=LocalKafkaUploader,
+    uploader_config=LocalKafkaUploaderConfig,
+)

unstructured_ingest/processes/connectors/kdbai.py

@@ -0,0 +1,156 @@
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, Generator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.data_types.file_data import FileData
+from unstructured_ingest.error import DestinationConnectionError
+from unstructured_ingest.interfaces import (
+    AccessConfig,
+    ConnectionConfig,
+    Uploader,
+    UploaderConfig,
+    UploadStager,
+    UploadStagerConfig,
+)
+from unstructured_ingest.logger import logger
+from unstructured_ingest.processes.connector_registry import (
+    DestinationRegistryEntry,
+)
+from unstructured_ingest.utils.data_prep import (
+    flatten_dict,
+    get_data_df,
+    get_enhanced_element_id,
+    split_dataframe,
+)
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from kdbai_client import Database, Session, Table
+    from pandas import DataFrame
+
+CONNECTOR_TYPE = "kdbai"
+
+
+class KdbaiAccessConfig(AccessConfig):
+    api_key: Optional[str] = Field(
+        default=None,
+        description="A string for the api-key, can be left empty "
+        "when connecting to local KDBAI instance.",
+    )
+
+
+class KdbaiConnectionConfig(ConnectionConfig):
+    access_config: Secret[KdbaiAccessConfig] = Field(
+        default=KdbaiAccessConfig(), validate_default=True
+    )
+    endpoint: str = Field(
+        default="http://localhost:8082", description="Endpoint url where KDBAI is hosted."
+    )
+
+    @requires_dependencies(["kdbai_client"], extras="kdbai")
+    @contextmanager
+    def get_client(self) -> Generator["Session", None, None]:
+        from kdbai_client import Session
+
+        session = None
+        try:
+            session = Session(
+                api_key=self.access_config.get_secret_value().api_key, endpoint=self.endpoint
+            )
+            yield session
+        finally:
+            if session:
+                session.close()
+
+
+class KdbaiUploadStagerConfig(UploadStagerConfig):
+    pass
+
+
+@dataclass
+class KdbaiUploadStager(UploadStager):
+    upload_stager_config: KdbaiUploadStagerConfig = field(default_factory=KdbaiUploadStagerConfig)
+
+    def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
+        data = element_dict.copy()
+        return {
+            "id": get_enhanced_element_id(element_dict=data, file_data=file_data),
+            "element_id": data.get("element_id"),
+            "document": data.pop("text", None),
+            "embeddings": data.get("embeddings"),
+            "metadata": flatten_dict(
+                dictionary=data.get("metadata"),
+                flatten_lists=True,
+                remove_none=True,
+            ),
+        }
+
+
+class KdbaiUploaderConfig(UploaderConfig):
+    database_name: str = Field(
+        default="default", description="The name of the KDBAI database to write into."
+    )
+    table_name: str = Field(description="The name of the KDBAI table to write into.")
+    batch_size: int = Field(default=100, description="Number of records per batch")
+
+
+@dataclass
+class KdbaiUploader(Uploader):
+    connection_config: KdbaiConnectionConfig
+    upload_config: KdbaiUploaderConfig
+    connector_type: str = field(default=CONNECTOR_TYPE, init=False)
+
+    def precheck(self) -> None:
+        try:
+            self.get_database()
+        except Exception as e:
+            logger.error(f"Failed to validate connection {e}", exc_info=True)
+            raise DestinationConnectionError(f"failed to validate connection: {e}")
+
+    @contextmanager
+    def get_database(self) -> Generator["Database", None, None]:
+        with self.connection_config.get_client() as client:
+            db = client.database(self.upload_config.database_name)
+            yield db
+
+    @contextmanager
+    def get_table(self) -> Generator["Table", None, None]:
+        with self.get_database() as db:
+            table = db.table(self.upload_config.table_name)
+            yield table
+
+    def upsert_batch(self, batch: "DataFrame"):
+        with self.get_table() as table:
+            table.insert(batch)
+
+    def process_dataframe(self, df: "DataFrame"):
+        logger.debug(
+            f"uploading {len(df)} entries to {self.connection_config.endpoint} "
+            f"db {self.upload_config.database_name} in table {self.upload_config.table_name}"
+        )
+        for batch_df in split_dataframe(df=df, chunk_size=self.upload_config.batch_size):
+            self.upsert_batch(batch=batch_df)
+
+    @requires_dependencies(["pandas"], extras="kdbai")
+    def run_data(self, data: list[dict], file_data: FileData, **kwargs: Any) -> None:
+        import pandas as pd
+
+        df = pd.DataFrame(data=data)
+        self.process_dataframe(df=df)
+
+    @requires_dependencies(["pandas"], extras="kdbai")
+    def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        data = get_data_df(path=path)
+        self.process_dataframe(df=data)
+
+
+kdbai_destination_entry = DestinationRegistryEntry(
+    connection_config=KdbaiConnectionConfig,
+    uploader=KdbaiUploader,
+    uploader_config=KdbaiUploaderConfig,
+    upload_stager=KdbaiUploadStager,
+    upload_stager_config=KdbaiUploadStagerConfig,
+)

unstructured_ingest/processes/connectors/lancedb/__init__.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from unstructured_ingest.processes.connector_registry import add_destination_entry
+
+from .aws import CONNECTOR_TYPE as LANCEDB_S3_CONNECTOR_TYPE
+from .aws import lancedb_aws_destination_entry
+from .azure import CONNECTOR_TYPE as LANCEDB_AZURE_CONNECTOR_TYPE
+from .azure import lancedb_azure_destination_entry
+from .cloud import CONNECTOR_TYPE as LANCEDB_CLOUD_CONNECTOR_TYPE
+from .cloud import lancedb_cloud_destination_entry
+from .gcp import CONNECTOR_TYPE as LANCEDB_GCS_CONNECTOR_TYPE
+from .gcp import lancedb_gcp_destination_entry
+from .local import CONNECTOR_TYPE as LANCEDB_LOCAL_CONNECTOR_TYPE
+from .local import lancedb_local_destination_entry
+
+add_destination_entry(
+    destination_type=LANCEDB_S3_CONNECTOR_TYPE, entry=lancedb_aws_destination_entry
+)
+add_destination_entry(
+    destination_type=LANCEDB_AZURE_CONNECTOR_TYPE, entry=lancedb_azure_destination_entry
+)
+add_destination_entry(
+    destination_type=LANCEDB_GCS_CONNECTOR_TYPE, entry=lancedb_gcp_destination_entry
+)
+add_destination_entry(
+    destination_type=LANCEDB_LOCAL_CONNECTOR_TYPE, entry=lancedb_local_destination_entry
+)
+add_destination_entry(
+    destination_type=LANCEDB_CLOUD_CONNECTOR_TYPE, entry=lancedb_cloud_destination_entry
+)

unstructured_ingest/processes/connectors/lancedb/aws.py

@@ -0,0 +1,43 @@
+from dataclasses import dataclass
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.interfaces.connector import AccessConfig
+from unstructured_ingest.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.processes.connectors.lancedb.lancedb import (
+    LanceDBRemoteConnectionConfig,
+    LanceDBUploader,
+    LanceDBUploaderConfig,
+    LanceDBUploadStager,
+    LanceDBUploadStagerConfig,
+)
+
+CONNECTOR_TYPE = "lancedb_aws"
+
+
+class LanceDBAwsAccessConfig(AccessConfig):
+    aws_access_key_id: str = Field(description="The AWS access key ID to use.")
+    aws_secret_access_key: str = Field(description="The AWS secret access key to use.")
+
+
+class LanceDBAwsConnectionConfig(LanceDBRemoteConnectionConfig):
+    access_config: Secret[LanceDBAwsAccessConfig]
+
+    def get_storage_options(self) -> dict:
+        return {**self.access_config.get_secret_value().model_dump(), "timeout": self.timeout}
+
+
+@dataclass
+class LanceDBAwsUploader(LanceDBUploader):
+    upload_config: LanceDBUploaderConfig
+    connection_config: LanceDBAwsConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+lancedb_aws_destination_entry = DestinationRegistryEntry(
+    connection_config=LanceDBAwsConnectionConfig,
+    uploader=LanceDBAwsUploader,
+    uploader_config=LanceDBUploaderConfig,
+    upload_stager_config=LanceDBUploadStagerConfig,
+    upload_stager=LanceDBUploadStager,
+)

unstructured_ingest/processes/connectors/lancedb/azure.py

@@ -0,0 +1,43 @@
+from dataclasses import dataclass
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.interfaces.connector import AccessConfig
+from unstructured_ingest.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.processes.connectors.lancedb.lancedb import (
+    LanceDBRemoteConnectionConfig,
+    LanceDBUploader,
+    LanceDBUploaderConfig,
+    LanceDBUploadStager,
+    LanceDBUploadStagerConfig,
+)
+
+CONNECTOR_TYPE = "lancedb_azure"
+
+
+class LanceDBAzureAccessConfig(AccessConfig):
+    azure_storage_account_name: str = Field(description="The name of the azure storage account.")
+    azure_storage_account_key: str = Field(description="The serialized azure service account key.")
+
+
+class LanceDBAzureConnectionConfig(LanceDBRemoteConnectionConfig):
+    access_config: Secret[LanceDBAzureAccessConfig]
+
+    def get_storage_options(self) -> dict:
+        return {**self.access_config.get_secret_value().model_dump(), "timeout": self.timeout}
+
+
+@dataclass
+class LanceDBAzureUploader(LanceDBUploader):
+    upload_config: LanceDBUploaderConfig
+    connection_config: LanceDBAzureConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+lancedb_azure_destination_entry = DestinationRegistryEntry(
+    connection_config=LanceDBAzureConnectionConfig,
+    uploader=LanceDBAzureUploader,
+    uploader_config=LanceDBUploaderConfig,
+    upload_stager_config=LanceDBUploadStagerConfig,
+    upload_stager=LanceDBUploadStager,
+)

unstructured_ingest/processes/connectors/lancedb/cloud.py

@@ -0,0 +1,42 @@
+from dataclasses import dataclass
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.interfaces.connector import AccessConfig
+from unstructured_ingest.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.processes.connectors.lancedb.lancedb import (
+    LanceDBRemoteConnectionConfig,
+    LanceDBUploader,
+    LanceDBUploaderConfig,
+    LanceDBUploadStager,
+    LanceDBUploadStagerConfig,
+)
+
+CONNECTOR_TYPE = "lancedb_cloud"
+
+
+class LanceDBCloudAccessConfig(AccessConfig):
+    api_key: str = Field(description="Api key associated with LanceDb cloud")
+
+
+class LanceDBCloudConnectionConfig(LanceDBRemoteConnectionConfig):
+    access_config: Secret[LanceDBCloudAccessConfig]
+
+    def get_storage_options(self) -> dict:
+        return {**self.access_config.get_secret_value().model_dump(), "timeout": self.timeout}
+
+
+@dataclass
+class LanceDBCloudUploader(LanceDBUploader):
+    upload_config: LanceDBUploaderConfig
+    connection_config: LanceDBCloudConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+lancedb_cloud_destination_entry = DestinationRegistryEntry(
+    connection_config=LanceDBCloudConnectionConfig,
+    uploader=LanceDBCloudUploader,
+    uploader_config=LanceDBUploaderConfig,
+    upload_stager_config=LanceDBUploadStagerConfig,
+    upload_stager=LanceDBUploadStager,
+)