unstructured-ingest 0.3.13__py3-none-any.whl

test/integration/connectors/test_chroma.py

@@ -0,0 +1,120 @@
+import json
+from pathlib import Path
+
+import chromadb
+import pytest
+from _pytest.fixtures import TopRequest
+
+from test.integration.connectors.utils.constants import (
+    DESTINATION_TAG,
+)
+from test.integration.connectors.utils.docker import HealthCheck, container_context
+from test.integration.connectors.utils.validation.destination import (
+    StagerValidationConfigs,
+    stager_validation,
+)
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.chroma import (
+    CONNECTOR_TYPE,
+    ChromaConnectionConfig,
+    ChromaUploader,
+    ChromaUploaderConfig,
+    ChromaUploadStager,
+    ChromaUploadStagerConfig,
+)
+
+
+@pytest.fixture
+def chroma_instance():
+    with container_context(
+        image="chromadb/chroma:latest",
+        ports={8000: 8000},
+        name="chroma_int_test",
+        healthcheck=HealthCheck(
+            interval=5,
+            timeout=10,
+            retries=3,
+            test="timeout 10s bash -c ':> /dev/tcp/127.0.0.1/8000' || exit 1",
+        ),
+    ) as ctx:
+        yield ctx
+
+
+def validate_collection(collection_name: str, num_embeddings: int):
+    print(f"Checking contents of Chroma collection: {collection_name}")
+
+    chroma_client = chromadb.HttpClient(
+        host="localhost",
+        port="8000",
+        tenant="default_tenant",
+        database="default_database",
+    )
+
+    collection = chroma_client.get_or_create_collection(name=collection_name)
+
+    number_of_embeddings = collection.count()
+    expected_embeddings = num_embeddings
+    print(
+        f"# of embeddings in collection vs expected: {number_of_embeddings}/{expected_embeddings}"
+    )
+
+    assert number_of_embeddings == expected_embeddings, (
+        f"Number of rows in generated table ({number_of_embeddings}) "
+        f"doesn't match expected value: {expected_embeddings}"
+    )
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+def test_chroma_destination(
+    upload_file: Path,
+    chroma_instance,
+    tmp_path: Path,
+):
+    collection_name = "test_collection"
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock file data",
+    )
+    stager = ChromaUploadStager(upload_stager_config=ChromaUploadStagerConfig())
+
+    uploader = ChromaUploader(
+        connection_config=ChromaConnectionConfig(
+            host="localhost",
+            port=8000,
+            tenant="default_tenant",
+            database="default_database",
+        ),
+        upload_config=ChromaUploaderConfig(collection_name=collection_name),
+    )
+    staged_filepath = stager.run(
+        elements_filepath=upload_file,
+        file_data=file_data,
+        output_dir=tmp_path,
+        output_filename=upload_file.name,
+    )
+    uploader.precheck()
+    uploader.run(path=staged_filepath, file_data=file_data)
+
+    # Run validation
+    with staged_filepath.open() as f:
+        staged_elements = json.load(f)
+    expected_count = len(staged_elements)
+    validate_collection(collection_name=collection_name, num_embeddings=expected_count)
+
+
+@pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "stager")
+def test_chroma_stager(
+    request: TopRequest,
+    upload_file_str: str,
+    tmp_path: Path,
+):
+    upload_file: Path = request.getfixturevalue(upload_file_str)
+    stager = ChromaUploadStager()
+    stager_validation(
+        configs=StagerValidationConfigs(test_id=CONNECTOR_TYPE, expected_count=22),
+        input_file=upload_file,
+        stager=stager,
+        tmp_dir=tmp_path,
+    )

test/integration/connectors/test_confluence.py

@@ -0,0 +1,113 @@
+import os
+
+import pytest
+
+from test.integration.connectors.utils.constants import (
+    SOURCE_TAG,
+)
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.processes.connectors.confluence import (
+    CONNECTOR_TYPE,
+    ConfluenceAccessConfig,
+    ConfluenceConnectionConfig,
+    ConfluenceDownloader,
+    ConfluenceDownloaderConfig,
+    ConfluenceIndexer,
+    ConfluenceIndexerConfig,
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@requires_env("CONFLUENCE_USER_EMAIL", "CONFLUENCE_API_TOKEN")
+async def test_confluence_source(temp_dir):
+    # Retrieve environment variables
+    confluence_url = "https://unstructured-ingest-test.atlassian.net"
+    user_email = os.environ["CONFLUENCE_USER_EMAIL"]
+    api_token = os.environ["CONFLUENCE_API_TOKEN"]
+    spaces = ["testteamsp", "MFS"]
+
+    # Create connection and indexer configurations
+    access_config = ConfluenceAccessConfig(api_token=api_token)
+    connection_config = ConfluenceConnectionConfig(
+        url=confluence_url,
+        user_email=user_email,
+        access_config=access_config,
+    )
+    index_config = ConfluenceIndexerConfig(
+        max_num_of_spaces=500,
+        max_num_of_docs_from_each_space=100,
+        spaces=spaces,
+    )
+
+    download_config = ConfluenceDownloaderConfig(download_dir=temp_dir)
+
+    # Instantiate indexer and downloader
+    indexer = ConfluenceIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = ConfluenceDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="confluence",
+            expected_num_files=11,
+            validate_downloaded_files=True,
+        ),
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@requires_env("CONFLUENCE_USER_EMAIL", "CONFLUENCE_API_TOKEN")
+async def test_confluence_source_large(temp_dir):
+    # Retrieve environment variables
+    confluence_url = "https://unstructured-ingest-test.atlassian.net"
+    user_email = os.environ["CONFLUENCE_USER_EMAIL"]
+    api_token = os.environ["CONFLUENCE_API_TOKEN"]
+    spaces = ["testteamsp1"]
+
+    # Create connection and indexer configurations
+    access_config = ConfluenceAccessConfig(api_token=api_token)
+    connection_config = ConfluenceConnectionConfig(
+        url=confluence_url,
+        user_email=user_email,
+        access_config=access_config,
+    )
+    index_config = ConfluenceIndexerConfig(
+        max_num_of_spaces=10,
+        max_num_of_docs_from_each_space=250,
+        spaces=spaces,
+    )
+
+    download_config = ConfluenceDownloaderConfig(download_dir=temp_dir)
+
+    # Instantiate indexer and downloader
+    indexer = ConfluenceIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = ConfluenceDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="confluence_large", expected_num_files=250, validate_file_data=False
+        ),
+    )

test/integration/connectors/test_delta_table.py

@@ -0,0 +1,185 @@
+import multiprocessing
+import os
+from pathlib import Path
+
+import pytest
+from deltalake import DeltaTable
+from fsspec import get_filesystem_class
+
+from test.integration.connectors.utils.constants import (
+    DESTINATION_TAG,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.delta_table import (
+    CONNECTOR_TYPE,
+    DeltaTableAccessConfig,
+    DeltaTableConnectionConfig,
+    DeltaTableUploader,
+    DeltaTableUploaderConfig,
+    DeltaTableUploadStager,
+    DeltaTableUploadStagerConfig,
+)
+
+multiprocessing.set_start_method("spawn")
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+async def test_delta_table_destination_local(upload_file: Path, temp_dir: Path):
+    destination_path = str(temp_dir)
+    connection_config = DeltaTableConnectionConfig(
+        access_config=DeltaTableAccessConfig(),
+        table_uri=destination_path,
+    )
+    stager_config = DeltaTableUploadStagerConfig()
+    stager = DeltaTableUploadStager(upload_stager_config=stager_config)
+    new_upload_file = stager.run(
+        elements_filepath=upload_file,
+        output_dir=temp_dir,
+        output_filename=upload_file.name,
+    )
+
+    upload_config = DeltaTableUploaderConfig()
+    uploader = DeltaTableUploader(connection_config=connection_config, upload_config=upload_config)
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(
+            fullpath=upload_file.name, filename=new_upload_file.name
+        ),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock file data",
+    )
+
+    if uploader.is_async():
+        await uploader.run_async(path=new_upload_file, file_data=file_data)
+    else:
+        uploader.run(path=new_upload_file, file_data=file_data)
+    delta_table_path = os.path.join(destination_path, upload_file.name)
+    delta_table = DeltaTable(table_uri=delta_table_path)
+    df = delta_table.to_pandas()
+
+    EXPECTED_COLUMNS = 10
+    EXPECTED_ROWS = 22
+    assert (
+        len(df) == EXPECTED_ROWS
+    ), f"Number of rows in table vs expected: {len(df)}/{EXPECTED_ROWS}"
+    assert (
+        len(df.columns) == EXPECTED_COLUMNS
+    ), f"Number of columns in table vs expected: {len(df.columns)}/{EXPECTED_COLUMNS}"
+
+
+def get_aws_credentials() -> dict:
+    access_key = os.getenv("S3_INGEST_TEST_ACCESS_KEY", None)
+    assert access_key
+    secret_key = os.getenv("S3_INGEST_TEST_SECRET_KEY", None)
+    assert secret_key
+    return {
+        "AWS_ACCESS_KEY_ID": access_key,
+        "AWS_SECRET_ACCESS_KEY": secret_key,
+        "AWS_REGION": "us-east-2",
+    }
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@requires_env("S3_INGEST_TEST_ACCESS_KEY", "S3_INGEST_TEST_SECRET_KEY")
+async def test_delta_table_destination_s3(upload_file: Path, temp_dir: Path):
+    aws_credentials = get_aws_credentials()
+    s3_bucket = "s3://utic-platform-test-destination"
+    destination_path = f"{s3_bucket}/destination/test"
+    connection_config = DeltaTableConnectionConfig(
+        access_config=DeltaTableAccessConfig(
+            aws_access_key_id=aws_credentials["AWS_ACCESS_KEY_ID"],
+            aws_secret_access_key=aws_credentials["AWS_SECRET_ACCESS_KEY"],
+        ),
+        aws_region=aws_credentials["AWS_REGION"],
+        table_uri=destination_path,
+    )
+    stager_config = DeltaTableUploadStagerConfig()
+    stager = DeltaTableUploadStager(upload_stager_config=stager_config)
+    new_upload_file = stager.run(
+        elements_filepath=upload_file,
+        output_dir=temp_dir,
+        output_filename=upload_file.name,
+    )
+
+    upload_config = DeltaTableUploaderConfig()
+    uploader = DeltaTableUploader(connection_config=connection_config, upload_config=upload_config)
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(
+            fullpath=upload_file.name, filename=new_upload_file.name
+        ),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock file data",
+    )
+
+    try:
+        uploader.precheck()
+        if uploader.is_async():
+            await uploader.run_async(path=new_upload_file, file_data=file_data)
+        else:
+            uploader.run(path=new_upload_file, file_data=file_data)
+        delta_table_path = os.path.join(destination_path, upload_file.name)
+        delta_table = DeltaTable(table_uri=delta_table_path, storage_options=aws_credentials)
+        df = delta_table.to_pandas()
+
+        EXPECTED_COLUMNS = 10
+        EXPECTED_ROWS = 22
+        assert (
+            len(df) == EXPECTED_ROWS
+        ), f"Number of rows in table vs expected: {len(df)}/{EXPECTED_ROWS}"
+        assert (
+            len(df.columns) == EXPECTED_COLUMNS
+        ), f"Number of columns in table vs expected: {len(df.columns)}/{EXPECTED_COLUMNS}"
+    finally:
+        s3fs = get_filesystem_class("s3")(
+            key=aws_credentials["AWS_ACCESS_KEY_ID"],
+            secret=aws_credentials["AWS_SECRET_ACCESS_KEY"],
+        )
+        s3fs.rm(path=destination_path, recursive=True)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@requires_env("S3_INGEST_TEST_ACCESS_KEY", "S3_INGEST_TEST_SECRET_KEY")
+async def test_delta_table_destination_s3_bad_creds(upload_file: Path, temp_dir: Path):
+    aws_credentials = {
+        "AWS_ACCESS_KEY_ID": "bad key",
+        "AWS_SECRET_ACCESS_KEY": "bad secret",
+        "AWS_REGION": "us-east-2",
+    }
+    s3_bucket = "s3://utic-platform-test-destination"
+    destination_path = f"{s3_bucket}/destination/test"
+    connection_config = DeltaTableConnectionConfig(
+        access_config=DeltaTableAccessConfig(
+            aws_access_key_id=aws_credentials["AWS_ACCESS_KEY_ID"],
+            aws_secret_access_key=aws_credentials["AWS_SECRET_ACCESS_KEY"],
+        ),
+        aws_region=aws_credentials["AWS_REGION"],
+        table_uri=destination_path,
+    )
+    stager_config = DeltaTableUploadStagerConfig()
+    stager = DeltaTableUploadStager(upload_stager_config=stager_config)
+    new_upload_file = stager.run(
+        elements_filepath=upload_file,
+        output_dir=temp_dir,
+        output_filename=upload_file.name,
+    )
+
+    upload_config = DeltaTableUploaderConfig()
+    uploader = DeltaTableUploader(connection_config=connection_config, upload_config=upload_config)
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(
+            fullpath=upload_file.name, filename=new_upload_file.name
+        ),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock file data",
+    )
+
+    with pytest.raises(Exception) as excinfo:
+        if uploader.is_async():
+            await uploader.run_async(path=new_upload_file, file_data=file_data)
+        else:
+            uploader.run(path=new_upload_file, file_data=file_data)
+
+    assert "403 Forbidden" in str(excinfo.value), f"Exception message did not match: {str(excinfo)}"

test/integration/connectors/test_lancedb.py

@@ -0,0 +1,247 @@
+import os
+from pathlib import Path
+from typing import Literal, Union
+from uuid import uuid4
+
+import lancedb
+import pandas as pd
+import pyarrow as pa
+import pytest
+import pytest_asyncio
+from lancedb import AsyncConnection
+from upath import UPath
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG
+from unstructured_ingest.v2.constants import RECORD_ID_LABEL
+from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.lancedb.aws import (
+    LanceDBAwsAccessConfig,
+    LanceDBAwsConnectionConfig,
+    LanceDBAwsUploader,
+)
+from unstructured_ingest.v2.processes.connectors.lancedb.azure import (
+    LanceDBAzureAccessConfig,
+    LanceDBAzureConnectionConfig,
+    LanceDBAzureUploader,
+)
+from unstructured_ingest.v2.processes.connectors.lancedb.gcp import (
+    LanceDBGCSAccessConfig,
+    LanceDBGCSConnectionConfig,
+    LanceDBGSPUploader,
+)
+from unstructured_ingest.v2.processes.connectors.lancedb.lancedb import (
+    CONNECTOR_TYPE,
+    LanceDBUploaderConfig,
+    LanceDBUploadStager,
+)
+from unstructured_ingest.v2.processes.connectors.lancedb.local import (
+    LanceDBLocalAccessConfig,
+    LanceDBLocalConnectionConfig,
+    LanceDBLocalUploader,
+)
+
+DATABASE_NAME = "database"
+TABLE_NAME = "elements"
+DIMENSION = 384
+NUMBER_EXPECTED_ROWS = 22
+S3_BUCKET = "s3://utic-ingest-test-fixtures/"
+GS_BUCKET = "gs://utic-test-ingest-fixtures-output/"
+AZURE_BUCKET = "az://utic-ingest-test-fixtures-output/"
+REQUIRED_ENV_VARS = {
+    "s3": ("S3_INGEST_TEST_ACCESS_KEY", "S3_INGEST_TEST_SECRET_KEY"),
+    "gcs": ("GCP_INGEST_SERVICE_KEY",),
+    "az": ("AZURE_DEST_CONNECTION_STR",),
+    "local": (),
+}
+
+SCHEMA = pa.schema(
+    [
+        pa.field(RECORD_ID_LABEL, pa.string()),
+        pa.field("vector", pa.list_(pa.float16(), DIMENSION)),
+        pa.field("text", pa.string(), nullable=True),
+        pa.field("type", pa.string(), nullable=True),
+        pa.field("element_id", pa.string(), nullable=True),
+        pa.field("metadata-text_as_html", pa.string(), nullable=True),
+        pa.field("metadata-filetype", pa.string(), nullable=True),
+        pa.field("metadata-filename", pa.string(), nullable=True),
+        pa.field("metadata-languages", pa.list_(pa.string()), nullable=True),
+        pa.field("metadata-is_continuation", pa.bool_(), nullable=True),
+        pa.field("metadata-page_number", pa.int32(), nullable=True),
+    ]
+)
+NUMBER_EXPECTED_COLUMNS = len(SCHEMA.names)
+
+
+@pytest_asyncio.fixture
+async def connection_with_uri(request, tmp_path: Path):
+    target = request.param
+    uri = _get_uri(target, local_base_path=tmp_path)
+
+    unset_variables = [env for env in REQUIRED_ENV_VARS[target] if env not in os.environ]
+    if unset_variables:
+        pytest.skip(
+            reason="Following required environment variables were not set: "
+            + f"{', '.join(unset_variables)}"
+        )
+
+    storage_options = {
+        "aws_access_key_id": os.getenv("S3_INGEST_TEST_ACCESS_KEY"),
+        "aws_secret_access_key": os.getenv("S3_INGEST_TEST_SECRET_KEY"),
+        "google_service_account_key": os.getenv("GCP_INGEST_SERVICE_KEY"),
+    }
+    azure_connection_string = os.getenv("AZURE_DEST_CONNECTION_STR")
+    if azure_connection_string:
+        storage_options.update(_parse_azure_connection_string(azure_connection_string))
+
+    storage_options = {key: value for key, value in storage_options.items() if value is not None}
+    connection = await lancedb.connect_async(
+        uri=uri,
+        storage_options=storage_options,
+    )
+    await connection.create_table(name=TABLE_NAME, schema=SCHEMA)
+
+    yield connection, uri
+
+    await connection.drop_database()
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.parametrize("connection_with_uri", ["local", "s3", "gcs", "az"], indirect=True)
+async def test_lancedb_destination(
+    upload_file: Path,
+    connection_with_uri: tuple[AsyncConnection, str],
+    tmp_path: Path,
+) -> None:
+    connection, uri = connection_with_uri
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock-file-data",
+    )
+    stager = LanceDBUploadStager()
+    uploader = _get_uploader(uri)
+    staged_file_path = stager.run(
+        elements_filepath=upload_file,
+        file_data=file_data,
+        output_dir=tmp_path,
+        output_filename=upload_file.name,
+    )
+
+    await uploader.run_async(path=staged_file_path, file_data=file_data)
+
+    # Test upload to empty table
+    with await connection.open_table(TABLE_NAME) as table:
+        table_df: pd.DataFrame = await table.to_pandas()
+
+    assert len(table_df) == NUMBER_EXPECTED_ROWS
+    assert len(table_df.columns) == NUMBER_EXPECTED_COLUMNS
+
+    assert table_df[RECORD_ID_LABEL][0] == file_data.identifier
+    assert table_df["element_id"][0] == "2470d8dc42215b3d68413b55bf00fed2"
+    assert table_df["type"][0] == "CompositeElement"
+    assert table_df["metadata-filename"][0] == "DA-1p-with-duplicate-pages.pdf.json"
+    assert table_df["metadata-text_as_html"][0] is None
+
+    # Test upload of the second file, rows should be appended
+    file_data.identifier = "mock-file-data-2"
+    staged_second_file_path = stager.run(
+        elements_filepath=upload_file,
+        file_data=file_data,
+        output_dir=tmp_path,
+        output_filename=f"{upload_file.stem}-2{upload_file.suffix}",
+    )
+    await uploader.run_async(path=staged_second_file_path, file_data=file_data)
+    with await connection.open_table(TABLE_NAME) as table:
+        appended_table_df: pd.DataFrame = await table.to_pandas()
+    assert len(appended_table_df) == 2 * NUMBER_EXPECTED_ROWS
+
+    # Test re-upload of the first file, rows should be overwritten, not appended
+    await uploader.run_async(path=staged_file_path, file_data=file_data)
+    with await connection.open_table(TABLE_NAME) as table:
+        overwritten_table_df: pd.DataFrame = await table.to_pandas()
+    assert len(overwritten_table_df) == 2 * NUMBER_EXPECTED_ROWS
+
+
+class TestPrecheck:
+    @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+    @pytest.mark.parametrize("connection_with_uri", ["local", "s3", "gcs", "az"], indirect=True)
+    def test_succeeds(
+        self,
+        upload_file: Path,
+        connection_with_uri: tuple[AsyncConnection, str],
+        tmp_path: Path,
+    ) -> None:
+        _, uri = connection_with_uri
+        uploader = _get_uploader(uri)
+        uploader.precheck()
+
+
+def _get_uri(target: Literal["local", "s3", "gcs", "az"], local_base_path: Path) -> str:
+    if target == "local":
+        return str(local_base_path / DATABASE_NAME)
+    if target == "s3":
+        base_uri = UPath(S3_BUCKET)
+    elif target == "gcs":
+        base_uri = UPath(GS_BUCKET)
+    elif target == "az":
+        base_uri = UPath(AZURE_BUCKET)
+
+    return str(base_uri / "destination" / "lancedb" / str(uuid4()) / DATABASE_NAME)
+
+
+def _get_uploader(
+    uri: str,
+) -> Union[LanceDBAzureUploader, LanceDBAzureUploader, LanceDBAwsUploader, LanceDBGSPUploader]:
+    target = uri.split("://", maxsplit=1)[0] if uri.startswith(("s3", "az", "gs")) else "local"
+    upload_config = LanceDBUploaderConfig(table_name=TABLE_NAME)
+    if target == "az":
+        azure_connection_string = os.getenv("AZURE_DEST_CONNECTION_STR")
+        access_config_kwargs = _parse_azure_connection_string(azure_connection_string)
+        return LanceDBAzureUploader(
+            upload_config=upload_config,
+            connection_config=LanceDBAzureConnectionConfig(
+                access_config=LanceDBAzureAccessConfig(**access_config_kwargs),
+                uri=uri,
+            ),
+        )
+
+    elif target == "s3":
+        return LanceDBAwsUploader(
+            upload_config=upload_config,
+            connection_config=LanceDBAwsConnectionConfig(
+                access_config=LanceDBAwsAccessConfig(
+                    aws_access_key_id=os.getenv("S3_INGEST_TEST_ACCESS_KEY"),
+                    aws_secret_access_key=os.getenv("S3_INGEST_TEST_SECRET_KEY"),
+                ),
+                uri=uri,
+            ),
+        )
+    elif target == "gs":
+        return LanceDBGSPUploader(
+            upload_config=upload_config,
+            connection_config=LanceDBGCSConnectionConfig(
+                access_config=LanceDBGCSAccessConfig(
+                    google_service_account_key=os.getenv("GCP_INGEST_SERVICE_KEY")
+                ),
+                uri=uri,
+            ),
+        )
+    else:
+        return LanceDBLocalUploader(
+            upload_config=upload_config,
+            connection_config=LanceDBLocalConnectionConfig(
+                access_config=LanceDBLocalAccessConfig(),
+                uri=uri,
+            ),
+        )
+
+
+def _parse_azure_connection_string(
+    connection_str: str,
+) -> dict[Literal["azure_storage_account_name", "azure_storage_account_key"], str]:
+    parameters = dict(keyvalue.split("=", maxsplit=1) for keyvalue in connection_str.split(";"))
+    return {
+        "azure_storage_account_name": parameters.get("AccountName"),
+        "azure_storage_account_key": parameters.get("AccountKey"),
+    }