unstructured-ingest 0.3.13__py3-none-any.whl → 0.3.15__py3-none-any.whl

test/integration/connectors/databricks/test_volumes_native.py

@@ -10,7 +10,11 @@ import pytest
 from databricks.sdk import WorkspaceClient
 from databricks.sdk.errors.platform import NotFound
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG
+from test.integration.connectors.utils.constants import (
+    BLOB_STORAGE_TAG,
+    DESTINATION_TAG,
+    SOURCE_TAG,
+)
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
@@ -83,7 +87,7 @@ def get_pat_env_data() -> PATEnvData:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
 @requires_env(
     "DATABRICKS_HOST", "DATABRICKS_CLIENT_ID", "DATABRICKS_CLIENT_SECRET", "DATABRICKS_CATALOG"
 )
@@ -115,7 +119,7 @@ async def test_volumes_native_source(tmp_path: Path):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
 @requires_env("DATABRICKS_HOST", "DATABRICKS_PAT", "DATABRICKS_CATALOG")
 async def test_volumes_native_source_pat(tmp_path: Path):
     env_data = get_pat_env_data()
@@ -144,7 +148,7 @@ async def test_volumes_native_source_pat(tmp_path: Path):
         )
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
 @requires_env("DATABRICKS_HOST", "DATABRICKS_PAT", "DATABRICKS_CATALOG")
 def test_volumes_native_source_pat_invalid_catalog():
     env_data = get_pat_env_data()
@@ -162,7 +166,7 @@ def test_volumes_native_source_pat_invalid_catalog():
             _ = list(indexer.run())
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
 @requires_env("DATABRICKS_HOST")
 def test_volumes_native_source_pat_invalid_pat():
     host = os.environ["DATABRICKS_HOST"]
@@ -231,7 +235,7 @@ def validate_upload(client: WorkspaceClient, catalog: str, volume: str, volume_p
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, BLOB_STORAGE_TAG)
 @requires_env(
     "DATABRICKS_HOST", "DATABRICKS_CLIENT_ID", "DATABRICKS_CLIENT_SECRET", "DATABRICKS_CATALOG"
 )

test/integration/connectors/discord/test_discord.py

@@ -6,7 +6,7 @@ from typing import Optional
 
 import pytest
 
-from test.integration.connectors.utils.constants import SOURCE_TAG
+from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
@@ -38,7 +38,7 @@ def get_env_data() -> EnvData:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
 @requires_env("DISCORD_TOKEN", "DISCORD_CHANNELS")
 async def test_discord_source():
     env = get_env_data()
@@ -66,7 +66,7 @@ async def test_discord_source():
         )
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
 @requires_env("DISCORD_CHANNELS")
 def test_discord_source_precheck_fail_no_token():
     indexer_config = DiscordIndexerConfig(channels=get_env_data().channels)
@@ -77,7 +77,7 @@ def test_discord_source_precheck_fail_no_token():
         indexer.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
 @requires_env("DISCORD_TOKEN")
 def test_discord_source_precheck_fail_no_channels():
     indexer_config = DiscordIndexerConfig(channels=[])

test/integration/connectors/duckdb/test_duckdb.py

@@ -5,7 +5,7 @@ import duckdb
 import pytest
 from _pytest.fixtures import TopRequest
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
     stager_validation,
@@ -46,7 +46,7 @@ def validate_duckdb_destination(db_path: Path, expected_num_elements: int):
             conn.close()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "duckdb")
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "duckdb", SQL_TAG)
 def test_duckdb_destination(upload_file: Path, provisioned_db_file: Path, temp_dir: Path):
     file_data = FileData(
         source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
@@ -73,6 +73,7 @@ def test_duckdb_destination(upload_file: Path, provisioned_db_file: Path, temp_d
     validate_duckdb_destination(db_path=provisioned_db_file, expected_num_elements=len(data))
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "duckdb", SQL_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_duckdb_stager(
     request: TopRequest,

test/integration/connectors/duckdb/test_motherduck.py

@@ -7,7 +7,7 @@ import duckdb
 import pandas as pd
 import pytest
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.duckdb.motherduck import (
@@ -61,7 +61,7 @@ def validate_motherduck_destination(database: str, expected_num_elements: int, m
             conn.close()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 @requires_env("MOTHERDUCK_TOKEN")
 def test_motherduck_destination(
     md_token: str, upload_file: Path, provisioned_db: str, temp_dir: Path

test/integration/connectors/elasticsearch/test_elasticsearch.py

@@ -15,7 +15,7 @@ from _pytest.fixtures import TopRequest
 from elasticsearch import Elasticsearch as ElasticsearchClient
 from elasticsearch.helpers import bulk
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, NOSQL_TAG
 from test.integration.connectors.utils.docker import HealthCheck, container_context
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
@@ -177,7 +177,7 @@ def destination_index(elasticsearch_elements_mapping: dict) -> str:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 async def test_elasticsearch_source(source_index: str, movies_dataframe: pd.DataFrame):
     indexer_config = ElasticsearchIndexerConfig(index_name=source_index)
     with tempfile.TemporaryDirectory() as tempdir:
@@ -207,7 +207,7 @@ async def test_elasticsearch_source(source_index: str, movies_dataframe: pd.Data
         )
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 def test_elasticsearch_source_precheck_fail_no_cluster():
     indexer_config = ElasticsearchIndexerConfig(index_name="index")
 
@@ -221,7 +221,7 @@ def test_elasticsearch_source_precheck_fail_no_cluster():
         indexer.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 def test_elasticsearch_source_precheck_fail_no_index(source_index: str):
     indexer_config = ElasticsearchIndexerConfig(index_name="index")
 
@@ -236,7 +236,7 @@ def test_elasticsearch_source_precheck_fail_no_index(source_index: str):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 async def test_elasticsearch_destination(
     upload_file: Path,
     destination_index: str,
@@ -282,7 +282,7 @@ async def test_elasticsearch_destination(
         validate_count(client=client, expected_count=expected_count, index_name=destination_index)
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 def test_elasticsearch_destination_precheck_fail():
     connection_config = ElasticsearchConnectionConfig(
         access_config=ElasticsearchAccessConfig(password=ES_PASSWORD),
@@ -297,7 +297,7 @@ def test_elasticsearch_destination_precheck_fail():
         uploader.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 def test_elasticsearch_destination_precheck_fail_no_index(destination_index: str):
     connection_config = ElasticsearchConnectionConfig(
         access_config=ElasticsearchAccessConfig(password=ES_PASSWORD),
@@ -312,6 +312,7 @@ def test_elasticsearch_destination_precheck_fail_no_index(destination_index: str
         uploader.precheck()
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_elasticsearch_stager(
     request: TopRequest,

test/integration/connectors/elasticsearch/test_opensearch.py

@@ -10,7 +10,7 @@ import pytest
 from _pytest.fixtures import TopRequest
 from opensearchpy import Document, Keyword, OpenSearch, Text
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, NOSQL_TAG, SOURCE_TAG
 from test.integration.connectors.utils.docker import HealthCheck, container_context
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
@@ -166,7 +166,7 @@ def destination_index(opensearch_elements_mapping: dict) -> str:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 async def test_opensearch_source(source_index: str, movies_dataframe: pd.DataFrame):
     indexer_config = OpenSearchIndexerConfig(index_name=source_index)
     with tempfile.TemporaryDirectory() as tempdir:
@@ -197,7 +197,7 @@ async def test_opensearch_source(source_index: str, movies_dataframe: pd.DataFra
         )
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 def test_opensearch_source_precheck_fail_no_cluster():
     indexer_config = OpenSearchIndexerConfig(index_name="index")
 
@@ -212,7 +212,7 @@ def test_opensearch_source_precheck_fail_no_cluster():
         indexer.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 def test_opensearch_source_precheck_fail_no_index(source_index: str):
     indexer_config = OpenSearchIndexerConfig(index_name="index")
 
@@ -228,7 +228,7 @@ def test_opensearch_source_precheck_fail_no_index(source_index: str):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 async def test_opensearch_destination(
     upload_file: Path,
     destination_index: str,
@@ -275,7 +275,7 @@ async def test_opensearch_destination(
         validate_count(client=client, expected_count=expected_count, index_name=destination_index)
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 def test_opensearch_destination_precheck_fail():
     connection_config = OpenSearchConnectionConfig(
         access_config=OpenSearchAccessConfig(password="admin"),
@@ -291,7 +291,7 @@ def test_opensearch_destination_precheck_fail():
         uploader.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 def test_opensearch_destination_precheck_fail_no_index(destination_index: str):
     connection_config = OpenSearchConnectionConfig(
         access_config=OpenSearchAccessConfig(password="admin"),
@@ -307,6 +307,7 @@ def test_opensearch_destination_precheck_fail_no_index(destination_index: str):
         uploader.precheck()
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_opensearch_stager(
     request: TopRequest,

test/integration/connectors/sql/test_databricks_delta_tables.py

@@ -0,0 +1,142 @@
+import json
+import os
+import time
+from contextlib import contextmanager
+from pathlib import Path
+from uuid import uuid4
+
+import pytest
+from databricks.sql import connect
+from databricks.sql.client import Connection as DeltaTableConnection
+from databricks.sql.client import Cursor as DeltaTableCursor
+from pydantic import BaseModel, SecretStr
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG, env_setup_path
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connectors.sql.databricks_delta_tables import (
+    CONNECTOR_TYPE,
+    DatabrickDeltaTablesAccessConfig,
+    DatabrickDeltaTablesConnectionConfig,
+    DatabrickDeltaTablesUploader,
+    DatabrickDeltaTablesUploaderConfig,
+    DatabrickDeltaTablesUploadStager,
+)
+
+CATALOG = "utic-dev-tech-fixtures"
+
+
+class EnvData(BaseModel):
+    server_hostname: str
+    http_path: str
+    access_token: SecretStr
+
+
+def get_env_data() -> EnvData:
+    return EnvData(
+        server_hostname=os.environ["DATABRICKS_SERVER_HOSTNAME"],
+        http_path=os.environ["DATABRICKS_HTTP_PATH"],
+        access_token=os.environ["DATABRICKS_ACCESS_TOKEN"],
+    )
+
+
+def get_destination_schema(new_table_name: str) -> str:
+    p = Path(env_setup_path / "sql" / "databricks_delta_tables" / "destination" / "schema.sql")
+    with p.open() as f:
+        data_lines = f.readlines()
+    data_lines[0] = data_lines[0].replace("elements", new_table_name)
+    data = "".join([line.strip() for line in data_lines])
+    return data
+
+
+@contextmanager
+def get_connection() -> DeltaTableConnection:
+    env_data = get_env_data()
+    with connect(
+        server_hostname=env_data.server_hostname,
+        http_path=env_data.http_path,
+        access_token=env_data.access_token.get_secret_value(),
+    ) as connection:
+        yield connection
+
+
+@contextmanager
+def get_cursor() -> DeltaTableCursor:
+    with get_connection() as connection:
+        with connection.cursor() as cursor:
+            cursor.execute(f"USE CATALOG '{CATALOG}'")
+            yield cursor
+
+
+@pytest.fixture
+def destination_table() -> str:
+    random_id = str(uuid4())[:8]
+    table_name = f"elements_{random_id}"
+    destination_schema = get_destination_schema(new_table_name=table_name)
+    with get_cursor() as cursor:
+        logger.info(f"creating table: {table_name}")
+        cursor.execute(f"DROP TABLE IF EXISTS {table_name}")
+        cursor.execute(destination_schema)
+
+    yield table_name
+    with get_cursor() as cursor:
+        logger.info(f"dropping table: {table_name}")
+        cursor.execute(f"DROP TABLE IF EXISTS {table_name}")
+
+
+def validate_destination(expected_num_elements: int, table_name: str, retries=30, interval=1):
+    with get_cursor() as cursor:
+        for i in range(retries):
+            cursor.execute(f"SELECT COUNT(*) FROM {table_name}")
+            count = cursor.fetchone()[0]
+            if count == expected_num_elements:
+                break
+            logger.info(f"retry attempt {i}: expected {expected_num_elements} != count {count}")
+            time.sleep(interval)
+        assert (
+            count == expected_num_elements
+        ), f"dest check failed: got {count}, expected {expected_num_elements}"
+
+
+@pytest.mark.asyncio
+@pytest.mark.skip("Resources take too long to spin up to run in CI")
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
+@requires_env("DATABRICKS_SERVER_HOSTNAME", "DATABRICKS_HTTP_PATH", "DATABRICKS_ACCESS_TOKEN")
+async def test_databricks_delta_tables_destination(
+    upload_file: Path, temp_dir: Path, destination_table: str
+):
+    env_data = get_env_data()
+    mock_file_data = FileData(
+        identifier="mock file data",
+        connector_type=CONNECTOR_TYPE,
+        source_identifiers=SourceIdentifiers(filename=upload_file.name, fullpath=upload_file.name),
+    )
+    stager = DatabrickDeltaTablesUploadStager()
+    staged_path = stager.run(
+        elements_filepath=upload_file,
+        file_data=mock_file_data,
+        output_dir=temp_dir,
+        output_filename=upload_file.name,
+    )
+
+    assert staged_path.suffix == upload_file.suffix
+
+    uploader = DatabrickDeltaTablesUploader(
+        connection_config=DatabrickDeltaTablesConnectionConfig(
+            access_config=DatabrickDeltaTablesAccessConfig(
+                token=env_data.access_token.get_secret_value()
+            ),
+            http_path=env_data.http_path,
+            server_hostname=env_data.server_hostname,
+        ),
+        upload_config=DatabrickDeltaTablesUploaderConfig(
+            catalog=CATALOG, database="default", table_name=destination_table
+        ),
+    )
+    with staged_path.open("r") as f:
+        staged_data = json.load(f)
+    expected_num_elements = len(staged_data)
+    uploader.precheck()
+    uploader.run(path=staged_path, file_data=mock_file_data)
+    validate_destination(expected_num_elements=expected_num_elements, table_name=destination_table)

test/integration/connectors/sql/test_postgres.py

@@ -5,7 +5,12 @@ import pytest
 from _pytest.fixtures import TopRequest
 from psycopg2 import connect
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
+from test.integration.connectors.utils.constants import (
+    DESTINATION_TAG,
+    SOURCE_TAG,
+    SQL_TAG,
+    env_setup_path,
+)
 from test.integration.connectors.utils.docker_compose import docker_compose_context
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
@@ -51,7 +56,7 @@ def source_database_setup() -> str:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, SQL_TAG)
 async def test_postgres_source(temp_dir: Path, source_database_setup: str):
     connect_params = {
         "host": "localhost",
@@ -115,7 +120,7 @@ def validate_destination(
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "sql")
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 async def test_postgres_destination(upload_file: Path, temp_dir: Path):
     # the postgres destination connector doesn't leverage the file data but is required as an input,
     # mocking it with arbitrary values to meet the base requirements:
@@ -179,6 +184,7 @@ async def test_postgres_destination(upload_file: Path, temp_dir: Path):
         )
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_postgres_stager(
     request: TopRequest,

test/integration/connectors/sql/test_singlestore.py

@@ -5,7 +5,12 @@ import pytest
 import singlestoredb as s2
 from _pytest.fixtures import TopRequest
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
+from test.integration.connectors.utils.constants import (
+    DESTINATION_TAG,
+    SOURCE_TAG,
+    SQL_TAG,
+    env_setup_path,
+)
 from test.integration.connectors.utils.docker_compose import docker_compose_context
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
@@ -54,7 +59,7 @@ def source_database_setup() -> dict:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, SQL_TAG)
 async def test_singlestore_source(temp_dir: Path, source_database_setup: dict):
 
     connection_config = SingleStoreConnectionConfig(
@@ -101,7 +106,7 @@ def validate_destination(
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "sql")
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 async def test_singlestore_destination(upload_file: Path, temp_dir: Path):
     mock_file_data = FileData(
         identifier="mock file data",
@@ -160,6 +165,7 @@ async def test_singlestore_destination(upload_file: Path, temp_dir: Path):
         )
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_singlestore_stager(
     request: TopRequest,

test/integration/connectors/sql/test_snowflake.py

@@ -6,7 +6,12 @@ import pytest
 import snowflake.connector as sf
 from _pytest.fixtures import TopRequest
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
+from test.integration.connectors.utils.constants import (
+    DESTINATION_TAG,
+    SOURCE_TAG,
+    SQL_TAG,
+    env_setup_path,
+)
 from test.integration.connectors.utils.docker import container_context
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
@@ -109,7 +114,7 @@ def destination_database_setup() -> dict:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, SQL_TAG)
 @requires_env("LOCALSTACK_AUTH_TOKEN")
 async def test_snowflake_source(temp_dir: Path, source_database_setup: dict):
     connection_config = SnowflakeConnectionConfig(
@@ -163,7 +168,7 @@ def validate_destination(
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "sql")
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 @requires_env("LOCALSTACK_AUTH_TOKEN")
 async def test_snowflake_destination(
     upload_file: Path, temp_dir: Path, destination_database_setup: dict
@@ -222,6 +227,7 @@ async def test_snowflake_destination(
     )
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_snowflake_stager(
     request: TopRequest,

test/integration/connectors/sql/test_sqlite.py

@@ -6,7 +6,12 @@ from pathlib import Path
 import pytest
 from _pytest.fixtures import TopRequest
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
+from test.integration.connectors.utils.constants import (
+    DESTINATION_TAG,
+    SOURCE_TAG,
+    SQL_TAG,
+    env_setup_path,
+)
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
     stager_validation,
@@ -52,7 +57,7 @@ def source_database_setup() -> Path:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, SQL_TAG)
 async def test_sqlite_source(source_database_setup: Path, temp_dir: Path):
     connection_config = SQLiteConnectionConfig(database_path=source_database_setup)
     indexer = SQLiteIndexer(
@@ -110,7 +115,7 @@ def validate_destination(db_path: Path, expected_num_elements: int):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "sql")
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 async def test_sqlite_destination(
     upload_file: Path, temp_dir: Path, destination_database_setup: Path
 ):
@@ -146,6 +151,7 @@ async def test_sqlite_destination(
     validate_destination(db_path=destination_database_setup, expected_num_elements=len(staged_data))
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_sqlite_stager(
     request: TopRequest,

test/integration/connectors/test_astradb.py

@@ -9,7 +9,7 @@ from _pytest.fixtures import TopRequest
 from astrapy import Collection
 from astrapy import DataAPIClient as AstraDBClient
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, VECTOR_DB_TAG
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
     stager_validation,
@@ -49,9 +49,9 @@ def connection_config() -> AstraDBConnectionConfig:
     )
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, VECTOR_DB_TAG)
 @requires_env("ASTRA_DB_APPLICATION_TOKEN", "ASTRA_DB_API_ENDPOINT")
-def test_precheck_succeeds(connection_config: AstraDBConnectionConfig):
+def test_precheck_succeeds_indexer(connection_config: AstraDBConnectionConfig):
     indexer = AstraDBIndexer(
         connection_config=connection_config,
         index_config=AstraDBIndexerConfig(collection_name=EXISTENT_COLLECTION_NAME),
@@ -64,19 +64,34 @@ def test_precheck_succeeds(connection_config: AstraDBConnectionConfig):
     uploader.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @requires_env("ASTRA_DB_APPLICATION_TOKEN", "ASTRA_DB_API_ENDPOINT")
-def test_precheck_fails(connection_config: AstraDBConnectionConfig):
+def test_precheck_succeeds_uploader(connection_config: AstraDBConnectionConfig):
+    uploader = AstraDBUploader(
+        connection_config=connection_config,
+        upload_config=AstraDBUploaderConfig(collection_name=EXISTENT_COLLECTION_NAME),
+    )
+    uploader.precheck()
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, VECTOR_DB_TAG)
+@requires_env("ASTRA_DB_APPLICATION_TOKEN", "ASTRA_DB_API_ENDPOINT")
+def test_precheck_fails_indexer(connection_config: AstraDBConnectionConfig):
     indexer = AstraDBIndexer(
         connection_config=connection_config,
         index_config=AstraDBIndexerConfig(collection_name=NONEXISTENT_COLLECTION_NAME),
     )
+    with pytest.raises(expected_exception=SourceConnectionError):
+        indexer.precheck()
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
+@requires_env("ASTRA_DB_APPLICATION_TOKEN", "ASTRA_DB_API_ENDPOINT")
+def test_precheck_fails_uploader(connection_config: AstraDBConnectionConfig):
     uploader = AstraDBUploader(
         connection_config=connection_config,
         upload_config=AstraDBUploaderConfig(collection_name=NONEXISTENT_COLLECTION_NAME),
     )
-    with pytest.raises(expected_exception=SourceConnectionError):
-        indexer.precheck()
     with pytest.raises(expected_exception=DestinationConnectionError):
         uploader.precheck()
 
@@ -117,7 +132,7 @@ def collection(upload_file: Path) -> Collection:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, VECTOR_DB_TAG)
 @requires_env("ASTRA_DB_API_ENDPOINT", "ASTRA_DB_APPLICATION_TOKEN")
 async def test_astra_search_source(
     tmp_path: Path,
@@ -151,7 +166,7 @@ async def test_astra_search_source(
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @requires_env("ASTRA_DB_API_ENDPOINT", "ASTRA_DB_APPLICATION_TOKEN")
 async def test_astra_search_destination(
     upload_file: Path,
@@ -201,6 +216,7 @@ async def test_astra_search_destination(
     )
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_astra_stager(
     request: TopRequest,