unstructured-ingest 0.3.0__py3-none-any.whl → 0.3.1__py3-none-any.whl

test/integration/connectors/test_pinecone.py

@@ -1,7 +1,9 @@
 import json
 import os
+import re
 import time
 from pathlib import Path
+from typing import Generator
 from uuid import uuid4
 
 import pytest
@@ -12,6 +14,7 @@ from test.integration.connectors.utils.constants import (
     DESTINATION_TAG,
 )
 from test.integration.utils import requires_env
+from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connectors.pinecone import (
@@ -24,6 +27,12 @@ from unstructured_ingest.v2.processes.connectors.pinecone import (
     PineconeUploadStagerConfig,
 )
 
+METADATA_BYTES_LIMIT = (
+    40960  # 40KB https://docs.pinecone.io/reference/quotas-and-limits#hard-limits
+)
+VECTOR_DIMENSION = 384
+SPEC = {"serverless": {"cloud": "aws", "region": "us-east-1"}}
+ALLOWED_METADATA_FIELD = "text"
 API_KEY = "PINECONE_API_KEY"
 
 
@@ -62,7 +71,7 @@ def wait_for_ready(client: Pinecone, index_name: str, timeout=60, interval=1) ->
 
 
 @pytest.fixture
-def pinecone_index() -> str:
+def pinecone_index() -> Generator[str, None, None]:
     pinecone = Pinecone(api_key=get_api_key())
     random_id = str(uuid4()).split("-")[0]
     index_name = f"ingest-test-{random_id}"
@@ -159,3 +168,46 @@ async def test_pinecone_destination(pinecone_index: str, upload_file: Path, temp
     validate_pinecone_index(
         index_name=pinecone_index, expected_num_of_vectors=expected_num_of_vectors
     )
+
+
+@requires_env(API_KEY)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+def test_large_metadata(pinecone_index: str, tmp_path: Path, upload_file: Path):
+    stager = PineconeUploadStager()
+    uploader = PineconeUploader(
+        connection_config=PineconeConnectionConfig(
+            access_config=PineconeAccessConfig(api_key=get_api_key()),
+            index_name=pinecone_index,
+        ),
+        upload_config=PineconeUploaderConfig(),
+    )
+    large_metadata_upload_file = tmp_path / "mock-upload-file.pdf.json"
+    large_metadata = {ALLOWED_METADATA_FIELD: "0" * 2 * METADATA_BYTES_LIMIT}
+
+    with open(upload_file) as file:
+        elements = json.load(file)
+
+    with open(large_metadata_upload_file, "w") as file:
+        mock_element = elements[0]
+        mock_element["metadata"] = large_metadata
+        json.dump([mock_element], file)
+
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(
+            fullpath=large_metadata_upload_file.name, filename=large_metadata_upload_file.name
+        ),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock-file-data",
+    )
+    staged_file = stager.run(
+        file_data, large_metadata_upload_file, tmp_path, large_metadata_upload_file.name
+    )
+    try:
+        uploader.run(staged_file, file_data)
+    except DestinationConnectionError as e:
+        error_line = r"Metadata size is \d+ bytes, which exceeds the limit of \d+ bytes per vector"
+        if re.search(re.compile(error_line), str(e)) is None:
+            raise e
+        raise pytest.fail("Upload request failed due to metadata exceeding limits.")
+
+    validate_pinecone_index(pinecone_index, 1, interval=5)

test/integration/connectors/utils/docker.py

@@ -1,9 +1,43 @@
 import time
 from contextlib import contextmanager
-from typing import Optional
+from typing import Optional, Union
 
 import docker
 from docker.models.containers import Container
+from pydantic import BaseModel, Field, field_serializer
+
+
+class HealthCheck(BaseModel):
+    test: Union[str, list[str]]
+    interval: int = Field(
+        gt=0, default=30, description="The time to wait between checks in seconds."
+    )
+    timeout: int = Field(
+        gt=0, default=30, description="The time to wait before considering the check to have hung."
+    )
+    retries: int = Field(
+        gt=0,
+        default=3,
+        description="The number of consecutive failures needed "
+        "to consider a container as unhealthy.",
+    )
+    start_period: int = Field(
+        gt=0,
+        default=0,
+        description="Start period for the container to initialize before starting health-retries countdown in seconds.",  # noqa: E501
+    )
+
+    @field_serializer("interval")
+    def serialize_interval(self, interval: int) -> int:
+        return int(interval * 10e8)
+
+    @field_serializer("timeout")
+    def serialize_timeout(self, timeout: int) -> int:
+        return int(timeout * 10e8)
+
+    @field_serializer("start_period")
+    def serialize_start_period(self, start_period: int) -> int:
+        return int(start_period * 10e8)
 
 
 def get_container(
@@ -12,7 +46,7 @@ def get_container(
     ports: dict,
     environment: Optional[dict] = None,
     volumes: Optional[dict] = None,
-    healthcheck: Optional[dict] = None,
+    healthcheck: Optional[HealthCheck] = None,
 ) -> Container:
     run_kwargs = {
         "image": image,
@@ -24,25 +58,49 @@ def get_container(
     if volumes:
         run_kwargs["volumes"] = volumes
     if healthcheck:
-        run_kwargs["healthcheck"] = healthcheck
+        run_kwargs["healthcheck"] = healthcheck.model_dump()
     container: Container = docker_client.containers.run(**run_kwargs)
     return container
 
 
-def has_healthcheck(container: Container) -> bool:
-    return container.attrs.get("Config", {}).get("Healthcheck", None) is not None
+def get_healthcheck(container: Container) -> Optional[HealthCheck]:
+    healthcheck_config = container.attrs.get("Config", {}).get("Healthcheck", None)
+    if not healthcheck_config:
+        return None
+    healthcheck_data = {
+        "test": healthcheck_config["Test"],
+    }
+    if interval := healthcheck_config.get("Interval"):
+        healthcheck_data["interval"] = interval / 10e8
+    if start_period := healthcheck_config.get("StartPeriod"):
+        healthcheck_data["start_period"] = start_period / 10e8
+    if retries := healthcheck_config.get("Retries"):
+        healthcheck_data["retries"] = retries
+    return HealthCheck.model_validate(healthcheck_data)
 
 
-def healthcheck_wait(container: Container, timeout: int = 10) -> None:
+def healthcheck_wait(
+    container: Container, retries: int = 30, interval: int = 1, start_period: Optional[int] = None
+) -> None:
+    if start_period:
+        time.sleep(start_period)
     health = container.health
-    start = time.time()
-    while health != "healthy" and time.time() - start < timeout:
-        time.sleep(1)
+    tries = 0
+    while health != "healthy" and tries < retries:
+        tries += 1
+        logs = container.attrs.get("State", {}).get("Health", {}).get("Log")
+        latest_log = logs[-1] if logs else None
+        print(
+            f"attempt {tries} - waiting for docker container "
+            f"to be healthy: {health} latest log: {latest_log}"
+        )
+        time.sleep(interval)
         container.reload()
         health = container.health
     if health != "healthy":
-        health_dict = container.attrs.get("State", {}).get("Health", {})
-        raise TimeoutError(f"Docker container never came up healthy: {health_dict}")
+        logs = container.attrs.get("State", {}).get("Health", {}).get("Log")
+        latest_log = logs[-1] if logs else None
+        raise TimeoutError(f"Docker container never came up healthy: {latest_log}")
 
 
 @contextmanager
@@ -51,11 +109,13 @@ def container_context(
     ports: dict,
     environment: Optional[dict] = None,
     volumes: Optional[dict] = None,
-    healthcheck: Optional[dict] = None,
-    healthcheck_timeout: int = 10,
+    healthcheck: Optional[HealthCheck] = None,
+    healthcheck_retries: int = 30,
     docker_client: Optional[docker.DockerClient] = None,
 ):
     docker_client = docker_client or docker.from_env()
+    print(f"pulling image {image}")
+    docker_client.images.pull(image)
     container: Optional[Container] = None
     try:
         container = get_container(
@@ -66,8 +126,14 @@ def container_context(
             volumes=volumes,
             healthcheck=healthcheck,
         )
-        if has_healthcheck(container):
-            healthcheck_wait(container=container, timeout=healthcheck_timeout)
+        if healthcheck_data := get_healthcheck(container):
+            # Mirror whatever healthcheck config set on container
+            healthcheck_wait(
+                container=container,
+                retries=healthcheck_retries,
+                start_period=healthcheck_data.start_period,
+                interval=healthcheck_data.interval,
+            )
         yield container
     except AssertionError as e:
         if container:

test/integration/connectors/utils/validation.py

@@ -240,6 +240,10 @@ def update_fixtures(
     # Rewrite the current file data
     if save_filedata:
         file_data_output_path = output_dir / "file_data"
+        print(
+            f"Writing {len(all_file_data)} file data to "
+            f"saved fixture location {file_data_output_path}"
+        )
         file_data_output_path.mkdir(parents=True, exist_ok=True)
         for file_data in all_file_data:
             file_data_path = file_data_output_path / f"{file_data.identifier}.json"
@@ -256,6 +260,10 @@ def update_fixtures(
     # If applicable, save raw downloads
     if save_downloads:
         raw_download_output_path = output_dir / "downloads"
+        print(
+            f"Writing {len(download_files)} downloaded files to "
+            f"saved fixture location {raw_download_output_path}"
+        )
         shutil.copytree(download_dir, raw_download_output_path)
 
 
@@ -328,6 +336,7 @@ async def source_connector_validation(
             postdownload_file_data = replace(resp["file_data"])
             all_postdownload_file_data.append(postdownload_file_data)
     if not overwrite_fixtures:
+        print("Running validation")
         run_all_validations(
             configs=configs,
             predownload_file_data=all_predownload_file_data,
@@ -336,6 +345,7 @@ async def source_connector_validation(
             test_output_dir=test_output_dir,
         )
     else:
+        print("Running fixtures update")
         update_fixtures(
             output_dir=test_output_dir,
             download_dir=download_dir,

test/integration/connectors/weaviate/conftest.py

@@ -0,0 +1,15 @@
+import json
+from pathlib import Path
+
+import pytest
+
+
+@pytest.fixture
+def collections_schema_config() -> dict:
+    int_test_dir = Path(__file__).parent
+    assets_dir = int_test_dir / "assets"
+    config_file = assets_dir / "elements.json"
+    assert config_file.exists()
+    assert config_file.is_file()
+    with config_file.open() as config_data:
+        return json.load(config_data)

test/integration/connectors/weaviate/test_local.py

@@ -0,0 +1,131 @@
+import json
+import time
+from pathlib import Path
+
+import pytest
+import requests
+import weaviate
+from weaviate.client import WeaviateClient
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.docker import container_context
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.weaviate.local import (
+    CONNECTOR_TYPE,
+    LocalWeaviateConnectionConfig,
+    LocalWeaviateUploader,
+    LocalWeaviateUploaderConfig,
+    LocalWeaviateUploadStager,
+)
+
+COLLECTION_NAME = "elements"
+
+
+def wait_for_container(timeout: int = 10, interval: int = 1) -> None:
+    start_time = time.time()
+    while time.time() - start_time < timeout:
+        try:
+            requests.get("http://localhost:8080/v1/.well-known/read")
+            return
+        except Exception as e:
+            print(f"Failed to validate container healthy, sleeping for {interval} seconds: {e}")
+            time.sleep(interval)
+    raise TimeoutError("Docker container never came up healthy")
+
+
+@pytest.fixture
+def collection(collections_schema_config: dict) -> str:
+    with container_context(
+        image="semitechnologies/weaviate:1.27.3",
+        ports={8080: 8080, 50051: 50051},
+    ):
+        wait_for_container()
+        with weaviate.connect_to_local() as weaviate_client:
+            weaviate_client.collections.create_from_dict(config=collections_schema_config)
+        yield COLLECTION_NAME
+
+
+def get_count(client: WeaviateClient) -> int:
+    collection = client.collections.get(COLLECTION_NAME)
+    resp = collection.aggregate.over_all(total_count=True)
+    return resp.total_count
+
+
+def validate_count(expected_count: int, retries: int = 10, interval: int = 1) -> None:
+    with weaviate.connect_to_local() as weaviate_client:
+        current_count = get_count(client=weaviate_client)
+        retry_count = 0
+        while current_count != expected_count and retry_count < retries:
+            retry_count += 1
+            time.sleep(interval)
+            current_count = get_count(client=weaviate_client)
+        assert current_count == expected_count, (
+            f"Expected count ({expected_count}) doesn't match how "
+            f"much came back from collection: {current_count}"
+        )
+
+
+def run_uploader_and_validate(
+    uploader: LocalWeaviateUploader, path: Path, file_data: FileData, expected_count: int
+):
+    uploader.precheck()
+    uploader.run(path=path, file_data=file_data)
+    validate_count(expected_count=expected_count)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+def test_weaviate_local_destination(upload_file: Path, collection: str, tmp_path: Path):
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock file data",
+    )
+    stager = LocalWeaviateUploadStager()
+
+    staged_filepath = stager.run(
+        elements_filepath=upload_file,
+        file_data=file_data,
+        output_dir=tmp_path,
+        output_filename=upload_file.name,
+    )
+    dynamic_uploader = LocalWeaviateUploader(
+        upload_config=LocalWeaviateUploaderConfig(
+            collection=COLLECTION_NAME,
+        ),
+        connection_config=LocalWeaviateConnectionConfig(),
+    )
+    fixed_size_uploader = LocalWeaviateUploader(
+        upload_config=LocalWeaviateUploaderConfig(
+            collection=COLLECTION_NAME, batch_size=10, dynamic_batch=False
+        ),
+        connection_config=LocalWeaviateConnectionConfig(),
+    )
+    rate_limited_uploader = LocalWeaviateUploader(
+        upload_config=LocalWeaviateUploaderConfig(
+            collection=COLLECTION_NAME, requests_per_minute=50, dynamic_batch=False
+        ),
+        connection_config=LocalWeaviateConnectionConfig(),
+    )
+    with staged_filepath.open() as f:
+        staged_elements = json.load(f)
+    expected_count = len(staged_elements)
+
+    run_uploader_and_validate(
+        uploader=dynamic_uploader,
+        path=staged_filepath,
+        file_data=file_data,
+        expected_count=expected_count,
+    )
+    run_uploader_and_validate(
+        uploader=fixed_size_uploader,
+        path=staged_filepath,
+        file_data=file_data,
+        expected_count=expected_count,
+    )
+    run_uploader_and_validate(
+        uploader=rate_limited_uploader,
+        path=staged_filepath,
+        file_data=file_data,
+        expected_count=expected_count,
+    )

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.3.0"  # pragma: no cover
+__version__ = "0.3.1"  # pragma: no cover

unstructured_ingest/pipeline/reformat/embedding.py

@@ -61,4 +61,4 @@ class Embedder(ReformatNode):
             return None
 
     def get_path(self) -> Path:
-        return (Path(self.pipeline_context.work_dir) / "embedded").resolve()
+        return (Path(self.pipeline_context.work_dir) / "embedded.py").resolve()

unstructured_ingest/utils/data_prep.py

@@ -1,7 +1,9 @@
 import itertools
 import json
 from datetime import datetime
-from typing import Any, Iterable, Optional, Sequence, TypeVar, cast
+from typing import Any, Generator, Iterable, Optional, Sequence, TypeVar, cast
+
+import pandas as pd
 
 DATE_FORMATS = ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d+%H:%M:%S", "%Y-%m-%dT%H:%M:%S%z")
 
@@ -9,6 +11,12 @@ T = TypeVar("T")
 IterableT = Iterable[T]
 
 
+def split_dataframe(df: pd.DataFrame, chunk_size: int = 100) -> Generator[pd.DataFrame, None, None]:
+    num_chunks = len(df) // chunk_size + 1
+    for i in range(num_chunks):
+        yield df[i * chunk_size : (i + 1) * chunk_size]
+
+
 def batch_generator(iterable: IterableT, batch_size: int = 100) -> IterableT:
     """A helper function to break an iterable into batches of size batch_size."""
     it = iter(iterable)

unstructured_ingest/v2/processes/connectors/__init__.py

@@ -1,10 +1,13 @@
 from __future__ import annotations
 
 import unstructured_ingest.v2.processes.connectors.databricks  # noqa: F401
+import unstructured_ingest.v2.processes.connectors.elasticsearch  # noqa: F401
 import unstructured_ingest.v2.processes.connectors.fsspec  # noqa: F401
 import unstructured_ingest.v2.processes.connectors.kafka  # noqa: F401
+import unstructured_ingest.v2.processes.connectors.lancedb  # noqa: F401
 import unstructured_ingest.v2.processes.connectors.qdrant  # noqa: F401
 import unstructured_ingest.v2.processes.connectors.sql  # noqa: F401
+import unstructured_ingest.v2.processes.connectors.weaviate  # noqa: F401
 from unstructured_ingest.v2.processes.connector_registry import (
     add_destination_entry,
     add_source_entry,
@@ -24,8 +27,6 @@ from .couchbase import CONNECTOR_TYPE as COUCHBASE_CONNECTOR_TYPE
 from .couchbase import couchbase_destination_entry, couchbase_source_entry
 from .delta_table import CONNECTOR_TYPE as DELTA_TABLE_CONNECTOR_TYPE
 from .delta_table import delta_table_destination_entry
-from .elasticsearch import CONNECTOR_TYPE as ELASTICSEARCH_CONNECTOR_TYPE
-from .elasticsearch import elasticsearch_destination_entry, elasticsearch_source_entry
 from .gitlab import CONNECTOR_TYPE as GITLAB_CONNECTOR_TYPE
 from .gitlab import gitlab_source_entry
 from .google_drive import CONNECTOR_TYPE as GOOGLE_DRIVE_CONNECTOR_TYPE
@@ -40,8 +41,6 @@ from .mongodb import CONNECTOR_TYPE as MONGODB_CONNECTOR_TYPE
 from .mongodb import mongodb_destination_entry, mongodb_source_entry
 from .onedrive import CONNECTOR_TYPE as ONEDRIVE_CONNECTOR_TYPE
 from .onedrive import onedrive_destination_entry, onedrive_source_entry
-from .opensearch import CONNECTOR_TYPE as OPENSEARCH_CONNECTOR_TYPE
-from .opensearch import opensearch_destination_entry, opensearch_source_entry
 from .outlook import CONNECTOR_TYPE as OUTLOOK_CONNECTOR_TYPE
 from .outlook import outlook_source_entry
 from .pinecone import CONNECTOR_TYPE as PINECONE_CONNECTOR_TYPE
@@ -52,8 +51,6 @@ from .sharepoint import CONNECTOR_TYPE as SHAREPOINT_CONNECTOR_TYPE
 from .sharepoint import sharepoint_source_entry
 from .slack import CONNECTOR_TYPE as SLACK_CONNECTOR_TYPE
 from .slack import slack_source_entry
-from .weaviate import CONNECTOR_TYPE as WEAVIATE_CONNECTOR_TYPE
-from .weaviate import weaviate_destination_entry
 
 add_source_entry(source_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_source_entry)
 add_destination_entry(destination_type=ASTRA_DB_CONNECTOR_TYPE, entry=astra_db_destination_entry)
@@ -67,10 +64,6 @@ add_destination_entry(
     destination_type=DELTA_TABLE_CONNECTOR_TYPE, entry=delta_table_destination_entry
 )
 
-add_source_entry(source_type=ELASTICSEARCH_CONNECTOR_TYPE, entry=elasticsearch_source_entry)
-add_destination_entry(
-    destination_type=ELASTICSEARCH_CONNECTOR_TYPE, entry=elasticsearch_destination_entry
-)
 
 add_source_entry(source_type=GOOGLE_DRIVE_CONNECTOR_TYPE, entry=google_drive_source_entry)
 
@@ -80,15 +73,9 @@ add_destination_entry(destination_type=LOCAL_CONNECTOR_TYPE, entry=local_destina
 add_source_entry(source_type=ONEDRIVE_CONNECTOR_TYPE, entry=onedrive_source_entry)
 add_destination_entry(destination_type=ONEDRIVE_CONNECTOR_TYPE, entry=onedrive_destination_entry)
 
-add_source_entry(source_type=OPENSEARCH_CONNECTOR_TYPE, entry=opensearch_source_entry)
-add_destination_entry(
-    destination_type=OPENSEARCH_CONNECTOR_TYPE, entry=opensearch_destination_entry
-)
 
 add_source_entry(source_type=SALESFORCE_CONNECTOR_TYPE, entry=salesforce_source_entry)
 
-add_destination_entry(destination_type=WEAVIATE_CONNECTOR_TYPE, entry=weaviate_destination_entry)
-
 add_destination_entry(destination_type=MONGODB_CONNECTOR_TYPE, entry=mongodb_destination_entry)
 add_source_entry(source_type=MONGODB_CONNECTOR_TYPE, entry=mongodb_source_entry)
 

unstructured_ingest/v2/processes/connectors/astradb.py

@@ -170,7 +170,7 @@ class AstraDBIndexer(Indexer):
 
     def precheck(self) -> None:
         try:
-            self.get_collection()
+            self.get_collection().options()
         except Exception as e:
             logger.error(f"Failed to validate connection {e}", exc_info=True)
             raise SourceConnectionError(f"failed to validate connection: {e}")
@@ -345,7 +345,7 @@ class AstraDBUploader(Uploader):
                 connection_config=self.connection_config,
                 collection_name=self.upload_config.collection_name,
                 keyspace=self.upload_config.keyspace,
-            )
+            ).options()
         except Exception as e:
             logger.error(f"Failed to validate connection {e}", exc_info=True)
             raise DestinationConnectionError(f"failed to validate connection: {e}")

unstructured_ingest/v2/processes/connectors/azure_ai_search.py

@@ -155,6 +155,10 @@ class AzureAISearchUploadStager(UploadStager):
             self.conform_dict(data=element, file_data=file_data) for element in elements_contents
         ]
 
+        if Path(output_filename).suffix != ".json":
+            output_filename = f"{output_filename}.json"
+        else:
+            output_filename = f"{Path(output_filename).stem}.json"
         output_path = Path(output_dir) / Path(f"{output_filename}.json")
         output_path.parent.mkdir(parents=True, exist_ok=True)
         with open(output_path, "w") as output_file:

unstructured_ingest/v2/processes/connectors/delta_table.py

@@ -1,7 +1,8 @@
 import json
 import os
+import traceback
 from dataclasses import dataclass, field
-from multiprocessing import Process
+from multiprocessing import Process, Queue
 from pathlib import Path
 from typing import Any, Optional
 from urllib.parse import urlparse
@@ -27,6 +28,15 @@ from unstructured_ingest.v2.processes.connector_registry import DestinationRegis
 CONNECTOR_TYPE = "delta_table"
 
 
+def write_deltalake_with_error_handling(queue, **kwargs):
+    from deltalake.writer import write_deltalake
+
+    try:
+        write_deltalake(**kwargs)
+    except Exception:
+        queue.put(traceback.format_exc())
+
+
 class DeltaTableAccessConfig(AccessConfig):
     aws_access_key_id: Optional[str] = Field(default=None, description="AWS Access Key Id")
     aws_secret_access_key: Optional[str] = Field(default=None, description="AWS Secret Access Key")
@@ -157,7 +167,6 @@ class DeltaTableUploader(Uploader):
 
     @requires_dependencies(["deltalake"], extras="delta-table")
     def run(self, path: Path, file_data: FileData, **kwargs: Any) -> None:
-        from deltalake.writer import write_deltalake
 
         df = self.read_dataframe(path)
         updated_upload_path = os.path.join(
@@ -176,17 +185,24 @@ class DeltaTableUploader(Uploader):
             "mode": "overwrite",
             "storage_options": storage_options,
         }
+        queue = Queue()
         # NOTE: deltalake writer on Linux sometimes can finish but still trigger a SIGABRT and cause
         # ingest to fail, even though all tasks are completed normally. Putting the writer into a
         # process mitigates this issue by ensuring python interpreter waits properly for deltalake's
         # rust backend to finish
         writer = Process(
-            target=write_deltalake,
-            kwargs=writer_kwargs,
+            target=write_deltalake_with_error_handling,
+            kwargs={"queue": queue, **writer_kwargs},
         )
         writer.start()
         writer.join()
 
+        # Check if the queue has any exception message
+        if not queue.empty():
+            error_message = queue.get()
+            logger.error(f"Exception occurred in write_deltalake: {error_message}")
+            raise RuntimeError(f"Error in write_deltalake: {error_message}")
+
 
 delta_table_destination_entry = DestinationRegistryEntry(
     connection_config=DeltaTableConnectionConfig,

unstructured_ingest/v2/processes/connectors/elasticsearch/__init__.py

@@ -0,0 +1,19 @@
+from unstructured_ingest.v2.processes.connector_registry import (
+    add_destination_entry,
+    add_source_entry,
+)
+
+from .elasticsearch import CONNECTOR_TYPE as ELASTICSEARCH_CONNECTOR_TYPE
+from .elasticsearch import elasticsearch_destination_entry, elasticsearch_source_entry
+from .opensearch import CONNECTOR_TYPE as OPENSEARCH_CONNECTOR_TYPE
+from .opensearch import opensearch_destination_entry, opensearch_source_entry
+
+add_source_entry(source_type=ELASTICSEARCH_CONNECTOR_TYPE, entry=elasticsearch_source_entry)
+add_destination_entry(
+    destination_type=ELASTICSEARCH_CONNECTOR_TYPE, entry=elasticsearch_destination_entry
+)
+
+add_source_entry(source_type=OPENSEARCH_CONNECTOR_TYPE, entry=opensearch_source_entry)
+add_destination_entry(
+    destination_type=OPENSEARCH_CONNECTOR_TYPE, entry=opensearch_destination_entry
+)