unstructured-ingest 0.3.2__tar.gz → 0.3.4__tar.gz

{unstructured_ingest-0.3.2 → unstructured_ingest-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.3.2
+Version: 0.3.4
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,13 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: dataclasses_json
-Requires-Dist: click
 Requires-Dist: pydantic>=2.7
+Requires-Dist: click
+Requires-Dist: dataclasses_json
 Requires-Dist: opentelemetry-sdk
-Requires-Dist: tqdm
 Requires-Dist: python-dateutil
 Requires-Dist: pandas
+Requires-Dist: tqdm
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
@@ -71,8 +71,8 @@ Requires-Dist: adlfs; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
 Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
@@ -81,13 +81,13 @@ Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: atlassian-python-api; extra == "confluence"
 Requires-Dist: requests; extra == "confluence"
+Requires-Dist: atlassian-python-api; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: boto3; extra == "delta-table"
+Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord-py; extra == "discord"
 Provides-Extra: dropbox
@@ -97,8 +97,8 @@ Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
 Requires-Dist: fsspec; extra == "gcs"
-Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
+Requires-Dist: bs4; extra == "gcs"
 Provides-Extra: github
 Requires-Dist: pygithub>1.58.0; extra == "github"
 Requires-Dist: requests; extra == "github"
@@ -107,22 +107,24 @@ Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: hubspot-api-client; extra == "hubspot"
+Requires-Dist: urllib3; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
 Requires-Dist: confluent-kafka; extra == "kafka"
 Provides-Extra: kdbai
 Requires-Dist: kdbai-client>=1.4.0; extra == "kdbai"
+Provides-Extra: lancedb
+Requires-Dist: lancedb; extra == "lancedb"
 Provides-Extra: milvus
 Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: notion
+Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
-Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: msal; extra == "onedrive"

{unstructured_ingest-0.3.2 → unstructured_ingest-0.3.4}/setup.py

@@ -1,7 +1,7 @@
 """
 setup.py
 
-unstructured - pre-processing tools for unstructured data
+unstructured-ingest - pre-processing tools for unstructured data
 
 Copyright 2022 Unstructured Technologies, Inc.
 
@@ -104,6 +104,7 @@ connectors_reqs = {
     "jira": load_requirements("requirements/connectors/jira.in"),
     "kafka": load_requirements("requirements/connectors/kafka.in"),
     "kdbai": load_requirements("requirements/connectors/kdbai.in"),
+    "lancedb": load_requirements("requirements/connectors/lancedb.in"),
     "milvus": load_requirements("requirements/connectors/milvus.in"),
     "mongodb": load_requirements("requirements/connectors/mongodb.in"),
     "notion": load_requirements("requirements/connectors/notion.in"),

{unstructured_ingest-0.3.2 → unstructured_ingest-0.3.4}/test/integration/connectors/test_lancedb.py

@@ -14,9 +14,9 @@ from upath import UPath
 from test.integration.connectors.utils.constants import DESTINATION_TAG
 from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.lancedb.aws import (
-    LanceDBS3AccessConfig,
-    LanceDBS3ConnectionConfig,
-    LanceDBS3Uploader,
+    LanceDBAwsAccessConfig,
+    LanceDBAwsConnectionConfig,
+    LanceDBAwsUploader,
 )
 from unstructured_ingest.v2.processes.connectors.lancedb.azure import (
     LanceDBAzureAccessConfig,
@@ -156,7 +156,7 @@ def _get_uri(target: Literal["local", "s3", "gcs", "az"], local_base_path: Path)
 
 def _get_uploader(
     uri: str,
-) -> Union[LanceDBAzureUploader, LanceDBAzureUploader, LanceDBS3Uploader, LanceDBGSPUploader]:
+) -> Union[LanceDBAzureUploader, LanceDBAzureUploader, LanceDBAwsUploader, LanceDBGSPUploader]:
     target = uri.split("://", maxsplit=1)[0] if uri.startswith(("s3", "az", "gs")) else "local"
     if target == "az":
         azure_connection_string = os.getenv("AZURE_DEST_CONNECTION_STR")
@@ -170,10 +170,10 @@ def _get_uploader(
         )
 
     elif target == "s3":
-        return LanceDBS3Uploader(
+        return LanceDBAwsUploader(
             upload_config=LanceDBUploaderConfig(table_name=TABLE_NAME),
-            connection_config=LanceDBS3ConnectionConfig(
-                access_config=LanceDBS3AccessConfig(
+            connection_config=LanceDBAwsConnectionConfig(
+                access_config=LanceDBAwsAccessConfig(
                     aws_access_key_id=os.getenv("S3_INGEST_TEST_ACCESS_KEY"),
                     aws_secret_access_key=os.getenv("S3_INGEST_TEST_SECRET_KEY"),
                 ),

{unstructured_ingest-0.3.2 → unstructured_ingest-0.3.4}/test/integration/connectors/test_milvus.py

@@ -15,6 +15,7 @@ from pymilvus.milvus_client import IndexParams
 from test.integration.connectors.utils.constants import DESTINATION_TAG, env_setup_path
 from test.integration.connectors.utils.docker import healthcheck_wait
 from test.integration.connectors.utils.docker_compose import docker_compose_context
+from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.milvus import (
     CONNECTOR_TYPE,
@@ -24,9 +25,10 @@ from unstructured_ingest.v2.processes.connectors.milvus import (
     MilvusUploadStager,
 )
 
-DB_URI = "http://localhost:19530"
 DB_NAME = "test_database"
-COLLECTION_NAME = "test_collection"
+EXISTENT_COLLECTION_NAME = "test_collection"
+NONEXISTENT_COLLECTION_NAME = "nonexistent_collection"
+DB_URI = "http://localhost:19530"
 
 
 def get_schema() -> CollectionSchema:
@@ -55,7 +57,9 @@ def get_index_params() -> IndexParams:
     return index_params
 
 
-@pytest.fixture
+# NOTE: Precheck tests are read-only so they don't interfere with destination test,
+# using scope="module" we can limit number of times the docker-compose has to be run
+@pytest.fixture(scope="module")
 def collection():
     docker_client = docker.from_env()
     with docker_compose_context(docker_compose_path=env_setup_path / "milvus"):
@@ -73,10 +77,10 @@ def collection():
             schema = get_schema()
             index_params = get_index_params()
             collection_resp = milvus_client.create_collection(
-                collection_name=COLLECTION_NAME, schema=schema, index_params=index_params
+                collection_name=EXISTENT_COLLECTION_NAME, schema=schema, index_params=index_params
             )
-            print(f"Created collection {COLLECTION_NAME}: {collection_resp}")
-            yield COLLECTION_NAME
+            print(f"Created collection {EXISTENT_COLLECTION_NAME}: {collection_resp}")
+            yield EXISTENT_COLLECTION_NAME
         finally:
             milvus_client.close()
 
@@ -139,3 +143,27 @@ async def test_milvus_destination(
     uploader.run(path=staged_filepath, file_data=file_data)
     with uploader.get_client() as client:
         validate_count(client=client, expected_count=expected_count)
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+def test_precheck_succeeds(collection: str):
+    uploader = MilvusUploader(
+        connection_config=MilvusConnectionConfig(uri=DB_URI),
+        upload_config=MilvusUploaderConfig(db_name=DB_NAME, collection_name=collection),
+    )
+    uploader.precheck()
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+def test_precheck_fails_on_nonexistent_collection(collection: str):
+    uploader = MilvusUploader(
+        connection_config=MilvusConnectionConfig(uri=DB_URI),
+        upload_config=MilvusUploaderConfig(
+            db_name=DB_NAME, collection_name=NONEXISTENT_COLLECTION_NAME
+        ),
+    )
+    with pytest.raises(
+        DestinationConnectionError,
+        match=f"Collection '{NONEXISTENT_COLLECTION_NAME}' does not exist",
+    ):
+        uploader.precheck()

unstructured_ingest-0.3.4/test/integration/connectors/weaviate/test_cloud.py

@@ -0,0 +1,34 @@
+import pytest
+from pydantic import ValidationError
+
+from unstructured_ingest.v2.processes.connectors.weaviate.cloud import (
+    CloudWeaviateAccessConfig,
+    CloudWeaviateConnectionConfig,
+)
+
+
+def test_weaviate_failing_connection_config():
+    with pytest.raises(ValidationError):
+        CloudWeaviateConnectionConfig(
+            access_config=CloudWeaviateAccessConfig(api_key="my key", password="password"),
+            username="username",
+            cluster_url="clusterurl",
+        )
+
+
+def test_weaviate_connection_config_happy_path():
+    CloudWeaviateConnectionConfig(
+        access_config=CloudWeaviateAccessConfig(
+            api_key="my key",
+        ),
+        cluster_url="clusterurl",
+    )
+
+
+def test_weaviate_connection_config_anonymous():
+    CloudWeaviateConnectionConfig(
+        access_config=CloudWeaviateAccessConfig(api_key="my key", password="password"),
+        username="username",
+        anonymous=True,
+        cluster_url="clusterurl",
+    )

unstructured_ingest-0.3.4/test/integration/embedders/test_azure_openai.py

@@ -0,0 +1,59 @@
+import json
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+from test.integration.embedders.utils import validate_embedding_output, validate_raw_embedder
+from test.integration.utils import requires_env
+from unstructured_ingest.embed.azure_openai import (
+    AzureOpenAIEmbeddingConfig,
+    AzureOpenAIEmbeddingEncoder,
+)
+from unstructured_ingest.v2.processes.embedder import Embedder, EmbedderConfig
+
+API_KEY = "AZURE_OPENAI_API_KEY"
+ENDPOINT = "AZURE_OPENAI_ENDPOINT"
+
+
+@dataclass(frozen=True)
+class AzureData:
+    api_key: str
+    endpoint: str
+
+
+def get_azure_data() -> AzureData:
+    api_key = os.getenv(API_KEY, None)
+    assert api_key
+    endpoint = os.getenv(ENDPOINT, None)
+    assert endpoint
+    return AzureData(api_key, endpoint)
+
+
+@requires_env(API_KEY, ENDPOINT)
+def test_azure_openai_embedder(embedder_file: Path):
+    azure_data = get_azure_data()
+    embedder_config = EmbedderConfig(
+        embedding_provider="azure-openai",
+        embedding_api_key=azure_data.api_key,
+        embedding_azure_endpoint=azure_data.endpoint,
+    )
+    embedder = Embedder(config=embedder_config)
+    results = embedder.run(elements_filepath=embedder_file)
+    assert results
+    with embedder_file.open("r") as f:
+        original_elements = json.load(f)
+    validate_embedding_output(original_elements=original_elements, output_elements=results)
+
+
+@requires_env(API_KEY, ENDPOINT)
+def test_raw_azure_openai_embedder(embedder_file: Path):
+    azure_data = get_azure_data()
+    embedder = AzureOpenAIEmbeddingEncoder(
+        config=AzureOpenAIEmbeddingConfig(
+            api_key=azure_data.api_key,
+            azure_endpoint=azure_data.endpoint,
+        )
+    )
+    validate_raw_embedder(
+        embedder=embedder, embedder_file=embedder_file, expected_dimensions=(1536,)
+    )

{unstructured_ingest-0.3.2 → unstructured_ingest-0.3.4}/test/unit/test_utils.py

@@ -8,7 +8,11 @@ import pytz
 
 from unstructured_ingest.cli.utils import extract_config
 from unstructured_ingest.interfaces import BaseConfig
-from unstructured_ingest.utils.string_and_date_utils import ensure_isoformat_datetime, json_to_dict
+from unstructured_ingest.utils.string_and_date_utils import (
+    ensure_isoformat_datetime,
+    json_to_dict,
+    truncate_string_bytes,
+)
 
 
 @dataclass
@@ -162,3 +166,19 @@ def test_ensure_isoformat_datetime_fails_on_string():
 def test_ensure_isoformat_datetime_fails_on_int():
     with pytest.raises(TypeError):
         ensure_isoformat_datetime(1111)
+
+
+def test_truncate_string_bytes_return_truncated_string():
+    test_string = "abcdef안녕하세요ghijklmn방갑습니opqrstu 더 길어지면 안되는 문자열vwxyz"
+    max_bytes = 11
+    result = truncate_string_bytes(test_string, max_bytes)
+    assert result == "abcdef안"
+    assert len(result.encode("utf-8")) <= max_bytes
+
+
+def test_truncate_string_bytes_return_untouched_string():
+    test_string = "abcdef"
+    max_bytes = 11
+    result = truncate_string_bytes(test_string, max_bytes)
+    assert result == "abcdef"
+    assert len(result.encode("utf-8")) <= max_bytes

unstructured_ingest-0.3.4/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.3.4"  # pragma: no cover

unstructured_ingest-0.3.4/unstructured_ingest/embed/azure_openai.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from pydantic import Field
+
+from unstructured_ingest.embed.openai import OpenAIEmbeddingConfig, OpenAIEmbeddingEncoder
+from unstructured_ingest.utils.dep_check import requires_dependencies
+
+if TYPE_CHECKING:
+    from openai import AzureOpenAI
+
+
+class AzureOpenAIEmbeddingConfig(OpenAIEmbeddingConfig):
+    api_version: str = Field(description="Azure API version", default="2024-06-01")
+    azure_endpoint: str
+    embedder_model_name: str = Field(default="text-embedding-ada-002", alias="model_name")
+
+    @requires_dependencies(["openai"], extras="openai")
+    def get_client(self) -> "AzureOpenAI":
+        from openai import AzureOpenAI
+
+        return AzureOpenAI(
+            api_key=self.api_key.get_secret_value(),
+            api_version=self.api_version,
+            azure_endpoint=self.azure_endpoint,
+        )
+
+
+@dataclass
+class AzureOpenAIEmbeddingEncoder(OpenAIEmbeddingEncoder):
+    config: AzureOpenAIEmbeddingConfig

{unstructured_ingest-0.3.2 → unstructured_ingest-0.3.4}/unstructured_ingest/utils/string_and_date_utils.py

@@ -37,3 +37,13 @@ def ensure_isoformat_datetime(timestamp: t.Union[datetime, str]) -> str:
             raise ValueError(f"String '{timestamp}' could not be parsed as a datetime.") from e
     else:
         raise TypeError(f"Expected input type datetime or str, but got {type(timestamp)}.")
+
+
+def truncate_string_bytes(string: str, max_bytes: int, encoding: str = "utf-8") -> str:
+    """
+    Truncates a string to a specified maximum number of bytes.
+    """
+    encoded_string = str(string).encode(encoding)
+    if len(encoded_string) <= max_bytes:
+        return string
+    return encoded_string[:max_bytes].decode(encoding, errors="ignore")

{unstructured_ingest-0.3.2 → unstructured_ingest-0.3.4}/unstructured_ingest/v2/processes/connectors/astradb.py

@@ -19,6 +19,7 @@ from unstructured_ingest.error import (
 )
 from unstructured_ingest.utils.data_prep import batch_generator
 from unstructured_ingest.utils.dep_check import requires_dependencies
+from unstructured_ingest.utils.string_and_date_utils import truncate_string_bytes
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces import (
     AccessConfig,
@@ -50,6 +51,8 @@ if TYPE_CHECKING:
 
 CONNECTOR_TYPE = "astradb"
 
+MAX_CONTENT_PARAM_BYTE_SIZE = 8000
+
 
 class AstraDBAccessConfig(AccessConfig):
     token: str = Field(description="Astra DB Token with access to the database.")
@@ -301,7 +304,20 @@ class AstraDBUploadStager(UploadStager):
         default_factory=lambda: AstraDBUploadStagerConfig()
     )
 
+    def truncate_dict_elements(self, element_dict: dict) -> None:
+        text = element_dict.pop("text", None)
+        if text is not None:
+            element_dict["text"] = truncate_string_bytes(text, MAX_CONTENT_PARAM_BYTE_SIZE)
+        metadata = element_dict.get("metadata")
+        if metadata is not None and isinstance(metadata, dict):
+            text_as_html = element_dict["metadata"].pop("text_as_html", None)
+            if text_as_html is not None:
+                element_dict["metadata"]["text_as_html"] = truncate_string_bytes(
+                    text_as_html, MAX_CONTENT_PARAM_BYTE_SIZE
+                )
+
     def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
+        self.truncate_dict_elements(element_dict)
         return {
             "$vector": element_dict.pop("embeddings", None),
             "content": element_dict.pop("text", None),

{unstructured_ingest-0.3.2 → unstructured_ingest-0.3.4}/unstructured_ingest/v2/processes/connectors/couchbase.py

@@ -219,6 +219,9 @@ class CouchbaseIndexer(Indexer):
 
 
 class CouchbaseDownloaderConfig(DownloaderConfig):
+    collection_id: str = Field(
+        default="id", description="The unique key of the id field in the collection"
+    )
     fields: list[str] = field(default_factory=list)
 
 
@@ -250,7 +253,7 @@ class CouchbaseDownloader(Downloader):
     def generate_download_response(
         self, result: dict, bucket: str, file_data: FileData
     ) -> DownloadResponse:
-        record_id = result["id"]
+        record_id = result[self.download_config.collection_id]
         filename_id = self.get_identifier(bucket=bucket, record_id=record_id)
         filename = f"{filename_id}.txt"
         download_path = self.download_dir / Path(filename)

unstructured_ingest-0.3.4/unstructured_ingest/v2/processes/connectors/lancedb/__init__.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from unstructured_ingest.v2.processes.connector_registry import add_destination_entry
+
+from .aws import CONNECTOR_TYPE as LANCEDB_S3_CONNECTOR_TYPE
+from .aws import lancedb_aws_destination_entry
+from .azure import CONNECTOR_TYPE as LANCEDB_AZURE_CONNECTOR_TYPE
+from .azure import lancedb_azure_destination_entry
+from .cloud import CONNECTOR_TYPE as LANCEDB_CLOUD_CONNECTOR_TYPE
+from .cloud import lancedb_cloud_destination_entry
+from .gcp import CONNECTOR_TYPE as LANCEDB_GCS_CONNECTOR_TYPE
+from .gcp import lancedb_gcp_destination_entry
+from .local import CONNECTOR_TYPE as LANCEDB_LOCAL_CONNECTOR_TYPE
+from .local import lancedb_local_destination_entry
+
+add_destination_entry(
+    destination_type=LANCEDB_S3_CONNECTOR_TYPE, entry=lancedb_aws_destination_entry
+)
+add_destination_entry(
+    destination_type=LANCEDB_AZURE_CONNECTOR_TYPE, entry=lancedb_azure_destination_entry
+)
+add_destination_entry(
+    destination_type=LANCEDB_GCS_CONNECTOR_TYPE, entry=lancedb_gcp_destination_entry
+)
+add_destination_entry(
+    destination_type=LANCEDB_LOCAL_CONNECTOR_TYPE, entry=lancedb_local_destination_entry
+)
+add_destination_entry(
+    destination_type=LANCEDB_CLOUD_CONNECTOR_TYPE, entry=lancedb_cloud_destination_entry
+)

{unstructured_ingest-0.3.2 → unstructured_ingest-0.3.4}/unstructured_ingest/v2/processes/connectors/lancedb/aws.py

@@ -15,28 +15,28 @@ from unstructured_ingest.v2.processes.connectors.lancedb.lancedb import (
 CONNECTOR_TYPE = "lancedb_aws"
 
 
-class LanceDBS3AccessConfig(AccessConfig):
+class LanceDBAwsAccessConfig(AccessConfig):
     aws_access_key_id: str = Field(description="The AWS access key ID to use.")
     aws_secret_access_key: str = Field(description="The AWS secret access key to use.")
 
 
-class LanceDBS3ConnectionConfig(LanceDBRemoteConnectionConfig):
-    access_config: Secret[LanceDBS3AccessConfig]
+class LanceDBAwsConnectionConfig(LanceDBRemoteConnectionConfig):
+    access_config: Secret[LanceDBAwsAccessConfig]
 
     def get_storage_options(self) -> dict:
         return {**self.access_config.get_secret_value().model_dump(), "timeout": self.timeout}
 
 
 @dataclass
-class LanceDBS3Uploader(LanceDBUploader):
+class LanceDBAwsUploader(LanceDBUploader):
     upload_config: LanceDBUploaderConfig
-    connection_config: LanceDBS3ConnectionConfig
+    connection_config: LanceDBAwsConnectionConfig
     connector_type: str = CONNECTOR_TYPE
 
 
 lancedb_aws_destination_entry = DestinationRegistryEntry(
-    connection_config=LanceDBS3ConnectionConfig,
-    uploader=LanceDBS3Uploader,
+    connection_config=LanceDBAwsConnectionConfig,
+    uploader=LanceDBAwsUploader,
     uploader_config=LanceDBUploaderConfig,
     upload_stager_config=LanceDBUploadStagerConfig,
     upload_stager=LanceDBUploadStager,

unstructured_ingest-0.3.4/unstructured_ingest/v2/processes/connectors/lancedb/cloud.py

@@ -0,0 +1,42 @@
+from dataclasses import dataclass
+
+from pydantic import Field, Secret
+
+from unstructured_ingest.v2.interfaces.connector import AccessConfig
+from unstructured_ingest.v2.processes.connector_registry import DestinationRegistryEntry
+from unstructured_ingest.v2.processes.connectors.lancedb.lancedb import (
+    LanceDBRemoteConnectionConfig,
+    LanceDBUploader,
+    LanceDBUploaderConfig,
+    LanceDBUploadStager,
+    LanceDBUploadStagerConfig,
+)
+
+CONNECTOR_TYPE = "lancedb_cloud"
+
+
+class LanceDBCloudAccessConfig(AccessConfig):
+    api_key: str = Field(description="Api key associated with LanceDb cloud")
+
+
+class LanceDBCloudConnectionConfig(LanceDBRemoteConnectionConfig):
+    access_config: Secret[LanceDBCloudAccessConfig]
+
+    def get_storage_options(self) -> dict:
+        return {**self.access_config.get_secret_value().model_dump(), "timeout": self.timeout}
+
+
+@dataclass
+class LanceDBCloudUploader(LanceDBUploader):
+    upload_config: LanceDBUploaderConfig
+    connection_config: LanceDBCloudConnectionConfig
+    connector_type: str = CONNECTOR_TYPE
+
+
+lancedb_cloud_destination_entry = DestinationRegistryEntry(
+    connection_config=LanceDBCloudConnectionConfig,
+    uploader=LanceDBCloudUploader,
+    uploader_config=LanceDBUploaderConfig,
+    upload_stager_config=LanceDBUploadStagerConfig,
+    upload_stager=LanceDBUploadStager,
+)

{unstructured_ingest-0.3.2 → unstructured_ingest-0.3.4}/unstructured_ingest/v2/processes/connectors/milvus.py

@@ -8,7 +8,7 @@ import pandas as pd
 from dateutil import parser
 from pydantic import Field, Secret
 
-from unstructured_ingest.error import WriteError
+from unstructured_ingest.error import DestinationConnectionError, WriteError
 from unstructured_ingest.utils.data_prep import flatten_dict
 from unstructured_ingest.utils.dep_check import requires_dependencies
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
@@ -66,7 +66,6 @@ class MilvusConnectionConfig(ConnectionConfig):
 
 
 class MilvusUploadStagerConfig(UploadStagerConfig):
-
     fields_to_include: Optional[list[str]] = None
     """If set - list of fields to include in the output.
     Unspecified fields are removed from the elements.
@@ -174,6 +173,14 @@ class MilvusUploader(Uploader):
     upload_config: MilvusUploaderConfig
     connector_type: str = CONNECTOR_TYPE
 
+    @DestinationConnectionError.wrap
+    def precheck(self):
+        with self.get_client() as client:
+            if not client.has_collection(self.upload_config.collection_name):
+                raise DestinationConnectionError(
+                    f"Collection '{self.upload_config.collection_name}' does not exist"
+                )
+
     @contextmanager
     def get_client(self) -> Generator["MilvusClient", None, None]:
         client = self.connection_config.get_client()
@@ -218,7 +225,6 @@ class MilvusUploader(Uploader):
             f"db in collection {self.upload_config.collection_name}"
         )
         with self.get_client() as client:
-
             try:
                 res = client.insert(collection_name=self.upload_config.collection_name, data=data)
             except MilvusException as milvus_exception:

{unstructured_ingest-0.3.2 → unstructured_ingest-0.3.4}/unstructured_ingest/v2/processes/connectors/weaviate/cloud.py

@@ -55,10 +55,11 @@ class CloudWeaviateConnectionConfig(WeaviateConnectionConfig):
             "client_secret": access_config.client_secret is not None,
             "client_password": access_config.password is not None and self.username is not None,
         }
-        if len(auths) == 0:
+        existing_auths = [auth_method for auth_method, flag in auths.items() if flag]
+
+        if len(existing_auths) == 0:
             raise ValueError("No auth values provided and anonymous is False")
-        if len(auths) > 1:
-            existing_auths = [auth_method for auth_method, flag in auths.items() if flag]
+        if len(existing_auths) > 1:
             raise ValueError(
                 "Multiple auth values provided, only one approach can be used: {}".format(
                     ", ".join(existing_auths)