unstructured-ingest 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

unstructured_ingest/v2/processes/connectors/milvus.py

@@ -1,7 +1,8 @@
 import json
+from contextlib import contextmanager
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional, Union
+from typing import TYPE_CHECKING, Any, Generator, Optional, Union
 
 import pandas as pd
 from dateutil import parser
@@ -10,6 +11,7 @@ from pydantic import Field, Secret
 from unstructured_ingest.error import WriteError
 from unstructured_ingest.utils.data_prep import flatten_dict
 from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -90,24 +92,27 @@ class MilvusUploadStager(UploadStager):
             pass
         return parser.parse(date_string).timestamp()
 
-    def conform_dict(self, data: dict) -> None:
-        if self.upload_stager_config.flatten_metadata and (metadata := data.pop("metadata", None)):
-            data.update(flatten_dict(metadata, keys_to_omit=["data_source_record_locator"]))
+    def conform_dict(self, data: dict, file_data: FileData) -> dict:
+        working_data = data.copy()
+        if self.upload_stager_config.flatten_metadata and (
+            metadata := working_data.pop("metadata", None)
+        ):
+            working_data.update(flatten_dict(metadata, keys_to_omit=["data_source_record_locator"]))
 
         # TODO: milvus sdk doesn't seem to support defaults via the schema yet,
         #  remove once that gets updated
         defaults = {"is_continuation": False}
         for default in defaults:
-            if default not in data:
-                data[default] = defaults[default]
+            if default not in working_data:
+                working_data[default] = defaults[default]
 
         if self.upload_stager_config.fields_to_include:
-            data_keys = set(data.keys())
+            data_keys = set(working_data.keys())
             for data_key in data_keys:
                 if data_key not in self.upload_stager_config.fields_to_include:
-                    data.pop(data_key)
+                    working_data.pop(data_key)
             for field_include_key in self.upload_stager_config.fields_to_include:
-                if field_include_key not in data:
+                if field_include_key not in working_data:
                     raise KeyError(f"Field '{field_include_key}' is missing in data!")
 
         datetime_columns = [
@@ -120,11 +125,15 @@ class MilvusUploadStager(UploadStager):
         json_dumps_fields = ["languages", "data_source_permissions_data"]
 
         for datetime_column in datetime_columns:
-            if datetime_column in data:
-                data[datetime_column] = self.parse_date_string(data[datetime_column])
+            if datetime_column in working_data:
+                working_data[datetime_column] = self.parse_date_string(
+                    working_data[datetime_column]
+                )
         for json_dumps_field in json_dumps_fields:
-            if json_dumps_field in data:
-                data[json_dumps_field] = json.dumps(data[json_dumps_field])
+            if json_dumps_field in working_data:
+                working_data[json_dumps_field] = json.dumps(working_data[json_dumps_field])
+        working_data[RECORD_ID_LABEL] = file_data.identifier
+        return working_data
 
     def run(
         self,
@@ -136,18 +145,27 @@ class MilvusUploadStager(UploadStager):
     ) -> Path:
         with open(elements_filepath) as elements_file:
             elements_contents: list[dict[str, Any]] = json.load(elements_file)
-        for element in elements_contents:
-            self.conform_dict(data=element)
-
-        output_path = Path(output_dir) / Path(f"{output_filename}.json")
+        new_content = [
+            self.conform_dict(data=element, file_data=file_data) for element in elements_contents
+        ]
+        output_filename_path = Path(output_filename)
+        if output_filename_path.suffix == ".json":
+            output_path = Path(output_dir) / output_filename_path
+        else:
+            output_path = Path(output_dir) / output_filename_path.with_suffix(".json")
         output_path.parent.mkdir(parents=True, exist_ok=True)
         with output_path.open("w") as output_file:
-            json.dump(elements_contents, output_file, indent=2)
+            json.dump(new_content, output_file, indent=2)
         return output_path
 
 
 class MilvusUploaderConfig(UploaderConfig):
+    db_name: Optional[str] = Field(default=None, description="Milvus database name")
     collection_name: str = Field(description="Milvus collections to write to")
+    record_id_key: str = Field(
+        default=RECORD_ID_LABEL,
+        description="searchable key to find entries for the same record on previous runs",
+    )
 
 
 @dataclass
@@ -156,6 +174,16 @@ class MilvusUploader(Uploader):
     upload_config: MilvusUploaderConfig
     connector_type: str = CONNECTOR_TYPE
 
+    @contextmanager
+    def get_client(self) -> Generator["MilvusClient", None, None]:
+        client = self.connection_config.get_client()
+        if db_name := self.upload_config.db_name:
+            client.using_database(db_name=db_name)
+        try:
+            yield client
+        finally:
+            client.close()
+
     def upload(self, content: UploadContent) -> None:
         file_extension = content.path.suffix
         if file_extension == ".json":
@@ -165,23 +193,39 @@ class MilvusUploader(Uploader):
         else:
             raise ValueError(f"Unsupported file extension: {file_extension}")
 
+    def delete_by_record_id(self, file_data: FileData) -> None:
+        logger.info(
+            f"deleting any content with metadata {RECORD_ID_LABEL}={file_data.identifier} "
+            f"from milvus collection {self.upload_config.collection_name}"
+        )
+        with self.get_client() as client:
+            delete_filter = f'{self.upload_config.record_id_key} == "{file_data.identifier}"'
+            resp = client.delete(
+                collection_name=self.upload_config.collection_name, filter=delete_filter
+            )
+            logger.info(
+                "deleted {} records from milvus collection {}".format(
+                    resp["delete_count"], self.upload_config.collection_name
+                )
+            )
+
     @requires_dependencies(["pymilvus"], extras="milvus")
     def insert_results(self, data: Union[dict, list[dict]]):
         from pymilvus import MilvusException
 
-        logger.debug(
+        logger.info(
             f"uploading {len(data)} entries to {self.connection_config.db_name} "
             f"db in collection {self.upload_config.collection_name}"
         )
-        client = self.connection_config.get_client()
+        with self.get_client() as client:
 
-        try:
-            res = client.insert(collection_name=self.upload_config.collection_name, data=data)
-        except MilvusException as milvus_exception:
-            raise WriteError("failed to upload records to milvus") from milvus_exception
-        if "err_count" in res and isinstance(res["err_count"], int) and res["err_count"] > 0:
-            err_count = res["err_count"]
-            raise WriteError(f"failed to upload {err_count} docs")
+            try:
+                res = client.insert(collection_name=self.upload_config.collection_name, data=data)
+            except MilvusException as milvus_exception:
+                raise WriteError("failed to upload records to milvus") from milvus_exception
+            if "err_count" in res and isinstance(res["err_count"], int) and res["err_count"] > 0:
+                err_count = res["err_count"]
+                raise WriteError(f"failed to upload {err_count} docs")
 
     def upload_csv(self, content: UploadContent) -> None:
         df = pd.read_csv(content.path)
@@ -194,6 +238,7 @@ class MilvusUploader(Uploader):
         self.insert_results(data=data)
 
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        self.delete_by_record_id(file_data=file_data)
         self.upload(content=UploadContent(path=path, file_data=file_data))
 
 

unstructured_ingest/v2/processes/connectors/pinecone.py

@@ -30,6 +30,7 @@ if TYPE_CHECKING:
 CONNECTOR_TYPE = "pinecone"
 MAX_PAYLOAD_SIZE = 2 * 1024 * 1024  # 2MB
 MAX_POOL_THREADS = 100
+MAX_METADATA_BYTES = 40960  # 40KB https://docs.pinecone.io/reference/quotas-and-limits#hard-limits
 
 
 class PineconeAccessConfig(AccessConfig):
@@ -103,6 +104,10 @@ class PineconeUploaderConfig(UploaderConfig):
         default=None,
         description="The namespace to write to. If not specified, the default namespace is used",
     )
+    record_id_key: str = Field(
+        default=RECORD_ID_LABEL,
+        description="searchable key to find entries for the same record on previous runs",
+    )
 
 
 @dataclass
@@ -133,6 +138,13 @@ class PineconeUploadStager(UploadStager):
             remove_none=True,
         )
         metadata[RECORD_ID_LABEL] = file_data.identifier
+        metadata_size_bytes = len(json.dumps(metadata).encode())
+        if metadata_size_bytes > MAX_METADATA_BYTES:
+            logger.info(
+                f"Metadata size is {metadata_size_bytes} bytes, which exceeds the limit of"
+                f" {MAX_METADATA_BYTES} bytes per vector. Dropping the metadata."
+            )
+            metadata = {}
 
         return {
             "id": str(uuid.uuid4()),
@@ -183,23 +195,28 @@ class PineconeUploader(Uploader):
 
     def pod_delete_by_record_id(self, file_data: FileData) -> None:
         logger.debug(
-            f"deleting any content with metadata {RECORD_ID_LABEL}={file_data.identifier} "
+            f"deleting any content with metadata "
+            f"{self.upload_config.record_id_key}={file_data.identifier} "
             f"from pinecone pod index"
         )
         index = self.connection_config.get_index(pool_threads=MAX_POOL_THREADS)
-        delete_kwargs = {"filter": {RECORD_ID_LABEL: {"$eq": file_data.identifier}}}
+        delete_kwargs = {
+            "filter": {self.upload_config.record_id_key: {"$eq": file_data.identifier}}
+        }
         if namespace := self.upload_config.namespace:
             delete_kwargs["namespace"] = namespace
 
         resp = index.delete(**delete_kwargs)
         logger.debug(
-            f"deleted any content with metadata {RECORD_ID_LABEL}={file_data.identifier} "
+            f"deleted any content with metadata "
+            f"{self.upload_config.record_id_key}={file_data.identifier} "
             f"from pinecone index: {resp}"
         )
 
     def serverless_delete_by_record_id(self, file_data: FileData) -> None:
         logger.debug(
-            f"deleting any content with metadata {RECORD_ID_LABEL}={file_data.identifier} "
+            f"deleting any content with metadata "
+            f"{self.upload_config.record_id_key}={file_data.identifier} "
             f"from pinecone serverless index"
         )
         index = self.connection_config.get_index(pool_threads=MAX_POOL_THREADS)
@@ -209,7 +226,7 @@ class PineconeUploader(Uploader):
             return
         dimension = index_stats["dimension"]
         query_params = {
-            "filter": {RECORD_ID_LABEL: {"$eq": file_data.identifier}},
+            "filter": {self.upload_config.record_id_key: {"$eq": file_data.identifier}},
             "vector": [0] * dimension,
             "top_k": total_vectors,
         }
@@ -226,7 +243,8 @@ class PineconeUploader(Uploader):
                 delete_params["namespace"] = namespace
             index.delete(**delete_params)
         logger.debug(
-            f"deleted any content with metadata {RECORD_ID_LABEL}={file_data.identifier} "
+            f"deleted any content with metadata "
+            f"{self.upload_config.record_id_key}={file_data.identifier} "
             f"from pinecone index"
         )
 
@@ -269,7 +287,6 @@ class PineconeUploader(Uploader):
             f"writing a total of {len(elements_dict)} elements via"
             f" document batches to destination"
             f" index named {self.connection_config.index_name}"
-            f" with batch size {self.upload_config.batch_size}"
         )
         # Determine if serverless or pod based index
         pinecone_client = self.connection_config.get_client()

unstructured_ingest/v2/processes/connectors/sql/sql.py

@@ -16,6 +16,8 @@ from dateutil import parser
 from pydantic import Field, Secret
 
 from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
+from unstructured_ingest.utils.data_prep import split_dataframe
+from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
     ConnectionConfig,
@@ -236,35 +238,25 @@ class SQLUploadStagerConfig(UploadStagerConfig):
 class SQLUploadStager(UploadStager):
     upload_stager_config: SQLUploadStagerConfig = field(default_factory=SQLUploadStagerConfig)
 
-    def run(
-        self,
-        elements_filepath: Path,
-        file_data: FileData,
-        output_dir: Path,
-        output_filename: str,
-        **kwargs: Any,
-    ) -> Path:
-        with open(elements_filepath) as elements_file:
-            elements_contents: list[dict] = json.load(elements_file)
-        output_path = Path(output_dir) / Path(f"{output_filename}.json")
-        output_path.parent.mkdir(parents=True, exist_ok=True)
-
+    @staticmethod
+    def conform_dict(data: dict, file_data: FileData) -> pd.DataFrame:
+        working_data = data.copy()
         output = []
-        for data in elements_contents:
-            metadata: dict[str, Any] = data.pop("metadata", {})
+        for element in working_data:
+            metadata: dict[str, Any] = element.pop("metadata", {})
             data_source = metadata.pop("data_source", {})
             coordinates = metadata.pop("coordinates", {})
 
-            data.update(metadata)
-            data.update(data_source)
-            data.update(coordinates)
+            element.update(metadata)
+            element.update(data_source)
+            element.update(coordinates)
 
-            data["id"] = str(uuid.uuid4())
+            element["id"] = str(uuid.uuid4())
 
             # remove extraneous, not supported columns
-            data = {k: v for k, v in data.items() if k in _COLUMNS}
-
-            output.append(data)
+            element = {k: v for k, v in element.items() if k in _COLUMNS}
+            element[RECORD_ID_LABEL] = file_data.identifier
+            output.append(element)
 
         df = pd.DataFrame.from_dict(output)
         for column in filter(lambda x: x in df.columns, _DATE_COLUMNS):
@@ -281,6 +273,26 @@ class SQLUploadStager(UploadStager):
             ("version", "page_number", "regex_metadata"),
         ):
             df[column] = df[column].apply(str)
+        return df
+
+    def run(
+        self,
+        elements_filepath: Path,
+        file_data: FileData,
+        output_dir: Path,
+        output_filename: str,
+        **kwargs: Any,
+    ) -> Path:
+        with open(elements_filepath) as elements_file:
+            elements_contents: list[dict] = json.load(elements_file)
+
+        df = self.conform_dict(data=elements_contents, file_data=file_data)
+        if Path(output_filename).suffix != ".json":
+            output_filename = f"{output_filename}.json"
+        else:
+            output_filename = f"{Path(output_filename).stem}.json"
+        output_path = Path(output_dir) / Path(f"{output_filename}")
+        output_path.parent.mkdir(parents=True, exist_ok=True)
 
         with output_path.open("w") as output_file:
             df.to_json(output_file, orient="records", lines=True)
@@ -290,6 +302,10 @@ class SQLUploadStager(UploadStager):
 class SQLUploaderConfig(UploaderConfig):
     batch_size: int = Field(default=50, description="Number of records per batch")
     table_name: str = Field(default="elements", description="which table to upload contents to")
+    record_id_key: str = Field(
+        default=RECORD_ID_LABEL,
+        description="searchable key to find entries for the same record on previous runs",
+    )
 
 
 @dataclass
@@ -323,18 +339,45 @@ class SQLUploader(Uploader):
             output.append(tuple(parsed))
         return output
 
+    def _fit_to_schema(self, df: pd.DataFrame, columns: list[str]) -> pd.DataFrame:
+        columns = set(df.columns)
+        schema_fields = set(columns)
+        columns_to_drop = columns - schema_fields
+        missing_columns = schema_fields - columns
+
+        if columns_to_drop:
+            logger.warning(
+                "Following columns will be dropped to match the table's schema: "
+                f"{', '.join(columns_to_drop)}"
+            )
+        if missing_columns:
+            logger.info(
+                "Following null filled columns will be added to match the table's schema:"
+                f" {', '.join(missing_columns)} "
+            )
+
+        df = df.drop(columns=columns_to_drop)
+
+        for column in missing_columns:
+            df[column] = pd.Series()
+
     def upload_contents(self, path: Path) -> None:
         df = pd.read_json(path, orient="records", lines=True)
         df.replace({np.nan: None}, inplace=True)
+        self._fit_to_schema(df=df, columns=self.get_table_columns())
 
         columns = list(df.columns)
         stmt = f"INSERT INTO {self.upload_config.table_name} ({','.join(columns)}) VALUES({','.join([self.values_delimiter for x in columns])})"  # noqa E501
-
-        for rows in pd.read_json(
-            path, orient="records", lines=True, chunksize=self.upload_config.batch_size
-        ):
+        logger.info(
+            f"writing a total of {len(df)} elements via"
+            f" document batches to destination"
+            f" table named {self.upload_config.table_name}"
+            f" with batch size {self.upload_config.batch_size}"
+        )
+        for rows in split_dataframe(df=df, chunk_size=self.upload_config.batch_size):
             with self.connection_config.get_cursor() as cursor:
                 values = self.prepare_data(columns, tuple(rows.itertuples(index=False, name=None)))
+                # For debugging purposes:
                 # for val in values:
                 #     try:
                 #         cursor.execute(stmt, val)
@@ -343,5 +386,33 @@ class SQLUploader(Uploader):
                 #         print(f"failed to write {len(columns)}, {len(val)}: {stmt} -> {val}")
                 cursor.executemany(stmt, values)
 
+    def get_table_columns(self) -> list[str]:
+        with self.connection_config.get_cursor() as cursor:
+            cursor.execute(f"SELECT * from {self.upload_config.table_name}")
+            return [desc[0] for desc in cursor.description]
+
+    def can_delete(self) -> bool:
+        return self.upload_config.record_id_key in self.get_table_columns()
+
+    def delete_by_record_id(self, file_data: FileData) -> None:
+        logger.debug(
+            f"deleting any content with data "
+            f"{self.upload_config.record_id_key}={file_data.identifier} "
+            f"from table {self.upload_config.table_name}"
+        )
+        stmt = f"DELETE FROM {self.upload_config.table_name} WHERE {self.upload_config.record_id_key} = {self.values_delimiter}"  # noqa: E501
+        with self.connection_config.get_cursor() as cursor:
+            cursor.execute(stmt, [file_data.identifier])
+            rowcount = cursor.rowcount
+            logger.info(f"deleted {rowcount} rows from table {self.upload_config.table_name}")
+
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
+        if self.can_delete():
+            self.delete_by_record_id(file_data=file_data)
+        else:
+            logger.warning(
+                f"table doesn't contain expected "
+                f"record id column "
+                f"{self.upload_config.record_id_key}, skipping delete"
+            )
         self.upload_contents(path=path)

unstructured_ingest/v2/processes/connectors/weaviate/__init__.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from unstructured_ingest.v2.processes.connector_registry import (
+    add_destination_entry,
+)
+
+from .cloud import CONNECTOR_TYPE as CLOUD_WEAVIATE_CONNECTOR_TYPE
+from .cloud import weaviate_cloud_destination_entry
+from .embedded import CONNECTOR_TYPE as EMBEDDED_WEAVIATE_CONNECTOR_TYPE
+from .embedded import weaviate_embedded_destination_entry
+from .local import CONNECTOR_TYPE as LOCAL_WEAVIATE_CONNECTOR_TYPE
+from .local import weaviate_local_destination_entry
+
+add_destination_entry(
+    destination_type=LOCAL_WEAVIATE_CONNECTOR_TYPE, entry=weaviate_local_destination_entry
+)
+add_destination_entry(
+    destination_type=CLOUD_WEAVIATE_CONNECTOR_TYPE, entry=weaviate_cloud_destination_entry
+)
+add_destination_entry(
+    destination_type=EMBEDDED_WEAVIATE_CONNECTOR_TYPE, entry=weaviate_embedded_destination_entry
+)

unstructured_ingest/v2/processes/connectors/weaviate/cloud.py

@@ -0,0 +1,164 @@
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Generator, Optional
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connectors.weaviate.weaviate import (
+    WeaviateAccessConfig,
+    WeaviateConnectionConfig,
+    WeaviateUploader,
+    WeaviateUploaderConfig,
+    WeaviateUploadStager,
+    WeaviateUploadStagerConfig,
+)
+
+if TYPE_CHECKING:
+    from weaviate.auth import AuthCredentials
+    from weaviate.client import WeaviateClient
+
+CONNECTOR_TYPE = "weaviate-cloud"
+
+
+class CloudWeaviateAccessConfig(WeaviateAccessConfig):
+    access_token: Optional[str] = Field(
+        default=None, description="Used to create the bearer token."
+    )
+    api_key: Optional[str] = None
+    client_secret: Optional[str] = None
+    password: Optional[str] = None
+
+
+class CloudWeaviateConnectionConfig(WeaviateConnectionConfig):
+    cluster_url: str = Field(
+        description="The WCD cluster URL or hostname to connect to. "
+        "Usually in the form: rAnD0mD1g1t5.something.weaviate.cloud"
+    )
+    username: Optional[str] = None
+    anonymous: bool = Field(default=False, description="if set, all auth values will be ignored")
+    refresh_token: Optional[str] = Field(
+        default=None,
+        description="Will tie this value to the bearer token. If not provided, "
+        "the authentication will expire once the lifetime of the access token is up.",
+    )
+    access_config: Secret[CloudWeaviateAccessConfig]
+
+    def model_post_init(self, __context: Any) -> None:
+        if self.anonymous:
+            return
+        access_config = self.access_config.get_secret_value()
+        auths = {
+            "api_key": access_config.api_key is not None,
+            "bearer_token": access_config.access_token is not None,
+            "client_secret": access_config.client_secret is not None,
+            "client_password": access_config.password is not None and self.username is not None,
+        }
+        if len(auths) == 0:
+            raise ValueError("No auth values provided and anonymous is False")
+        if len(auths) > 1:
+            existing_auths = [auth_method for auth_method, flag in auths.items() if flag]
+            raise ValueError(
+                "Multiple auth values provided, only one approach can be used: {}".format(
+                    ", ".join(existing_auths)
+                )
+            )
+
+    @requires_dependencies(["weaviate"], extras="weaviate")
+    def get_api_key_auth(self) -> Optional["AuthCredentials"]:
+        from weaviate.classes.init import Auth
+
+        if api_key := self.access_config.get_secret_value().api_key:
+            return Auth.api_key(api_key=api_key)
+        return None
+
+    @requires_dependencies(["weaviate"], extras="weaviate")
+    def get_bearer_token_auth(self) -> Optional["AuthCredentials"]:
+        from weaviate.classes.init import Auth
+
+        if access_token := self.access_config.get_secret_value().access_token:
+            return Auth.bearer_token(access_token=access_token, refresh_token=self.refresh_token)
+        return None
+
+    @requires_dependencies(["weaviate"], extras="weaviate")
+    def get_client_secret_auth(self) -> Optional["AuthCredentials"]:
+        from weaviate.classes.init import Auth
+
+        if client_secret := self.access_config.get_secret_value().client_secret:
+            return Auth.client_credentials(client_secret=client_secret)
+        return None
+
+    @requires_dependencies(["weaviate"], extras="weaviate")
+    def get_client_password_auth(self) -> Optional["AuthCredentials"]:
+        from weaviate.classes.init import Auth
+
+        if (username := self.username) and (
+            password := self.access_config.get_secret_value().password
+        ):
+            return Auth.client_password(username=username, password=password)
+        return None
+
+    @requires_dependencies(["weaviate"], extras="weaviate")
+    def get_auth(self) -> "AuthCredentials":
+        auths = [
+            self.get_api_key_auth(),
+            self.get_client_secret_auth(),
+            self.get_bearer_token_auth(),
+            self.get_client_password_auth(),
+        ]
+        auths = [auth for auth in auths if auth]
+        if len(auths) == 0:
+            raise ValueError("No auth values provided and anonymous is False")
+        if len(auths) > 1:
+            raise ValueError("Multiple auth values provided, only one approach can be used")
+        return auths[0]
+
+    @contextmanager
+    @requires_dependencies(["weaviate"], extras="weaviate")
+    def get_client(self) -> Generator["WeaviateClient", None, None]:
+        from weaviate import connect_to_weaviate_cloud
+        from weaviate.classes.init import AdditionalConfig
+
+        auth_credentials = None if self.anonymous else self.get_auth()
+        with connect_to_weaviate_cloud(
+            cluster_url=self.cluster_url,
+            auth_credentials=auth_credentials,
+            additional_config=AdditionalConfig(timeout=self.get_timeout()),
+        ) as weaviate_client:
+            yield weaviate_client
+
+
+class CloudWeaviateUploadStagerConfig(WeaviateUploadStagerConfig):
+    pass
+
+
+@dataclass
+class CloudWeaviateUploadStager(WeaviateUploadStager):
+    upload_stager_config: CloudWeaviateUploadStagerConfig = field(
+        default_factory=lambda: WeaviateUploadStagerConfig()
+    )
+
+
+class CloudWeaviateUploaderConfig(WeaviateUploaderConfig):
+    pass
+
+
+@dataclass
+class CloudWeaviateUploader(WeaviateUploader):
+    connection_config: CloudWeaviateConnectionConfig = field(
+        default_factory=lambda: CloudWeaviateConnectionConfig()
+    )
+    upload_config: CloudWeaviateUploaderConfig = field(
+        default_factory=lambda: CloudWeaviateUploaderConfig()
+    )
+    connector_type: str = CONNECTOR_TYPE
+
+
+weaviate_cloud_destination_entry = DestinationRegistryEntry(
+    connection_config=CloudWeaviateConnectionConfig,
+    uploader=CloudWeaviateUploader,
+    uploader_config=CloudWeaviateUploaderConfig,
+    upload_stager=CloudWeaviateUploadStager,
+    upload_stager_config=CloudWeaviateUploadStagerConfig,
+)