unstructured-ingest 0.7.2__py3-none-any.whl → 1.0.1__py3-none-any.whl

test/integration/connectors/elasticsearch/test_opensearch.py

@@ -1,326 +0,0 @@
-import json
-import tempfile
-import time
-from contextlib import contextmanager
-from pathlib import Path
-from typing import Generator
-
-import pandas as pd
-import pytest
-from _pytest.fixtures import TopRequest
-from opensearchpy import Document, Keyword, OpenSearch, Text
-
-from test.integration.connectors.utils.constants import DESTINATION_TAG, NOSQL_TAG, SOURCE_TAG
-from test.integration.connectors.utils.docker import HealthCheck, container_context
-from test.integration.connectors.utils.validation.destination import (
-    StagerValidationConfigs,
-    stager_validation,
-)
-from test.integration.connectors.utils.validation.source import (
-    SourceValidationConfigs,
-    source_connector_validation,
-)
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.error import (
-    DestinationConnectionError,
-    SourceConnectionError,
-)
-from unstructured_ingest.processes.connectors.elasticsearch.opensearch import (
-    CONNECTOR_TYPE,
-    OpenSearchAccessConfig,
-    OpenSearchConnectionConfig,
-    OpenSearchDownloader,
-    OpenSearchDownloaderConfig,
-    OpenSearchIndexer,
-    OpenSearchIndexerConfig,
-    OpenSearchUploader,
-    OpenSearchUploaderConfig,
-    OpenSearchUploadStager,
-    OpenSearchUploadStagerConfig,
-)
-
-SOURCE_INDEX_NAME = "movies"
-DESTINATION_INDEX_NAME = "elements"
-
-
-class Movie(Document):
-    title = Text(fields={"raw": Keyword()})
-    year = Text()
-    director = Text()
-    cast = Text()
-    genre = Text()
-    wiki_page = Text()
-    ethnicity = Text()
-    plot = Text()
-
-    class Index:
-        name = SOURCE_INDEX_NAME
-
-    def save(self, **kwargs):
-        return super(Movie, self).save(**kwargs)
-
-
-@contextmanager
-def get_client() -> Generator[OpenSearch, None, None]:
-    with OpenSearch(
-        hosts=[{"host": "localhost", "port": 9200}],
-        http_auth=("admin", "admin"),
-        use_ssl=True,
-        verify_certs=False,
-        ssl_show_warn=False,
-    ) as client:
-        yield client
-
-
-def get_index_count(client: OpenSearch, index_name: str) -> int:
-    count_resp = client.cat.count(index=index_name, params={"format": "json"})
-    return int(count_resp[0]["count"])
-
-
-def wait_for_write(
-    client: OpenSearch, index_name: str, expected_count: int, timeout: int = 30, interval: int = 1
-) -> None:
-    current_count = get_index_count(client, index_name)
-    start = time.time()
-    while time.time() - start < timeout:
-        print(f"waiting for current count ({current_count}) to match expected {expected_count}")
-        time.sleep(interval)
-        current_count = get_index_count(client, index_name)
-        if current_count == expected_count:
-            return
-    raise TimeoutError("Timed out while waiting for write to sync")
-
-
-def validate_count(
-    client: OpenSearch, index_name: str, expected_count: int, retries: int = 10, interval: int = 1
-) -> None:
-    current_count = get_index_count(client, index_name)
-    if current_count == expected_count:
-        return
-    tries = 0
-    while tries < retries:
-        print(
-            f"retrying validation to check if expected count "
-            f"{expected_count} will match current count {current_count}"
-        )
-        time.sleep(interval)
-        current_count = get_index_count(client, index_name)
-        if current_count == expected_count:
-            break
-    assert current_count == expected_count, (
-        f"Expected count ({expected_count}) doesn't match how "
-        f"much came back from index: {current_count}"
-    )
-
-
-@pytest.fixture
-def source_index(movies_dataframe: pd.DataFrame) -> str:
-    with container_context(
-        image="opensearchproject/opensearch:2.11.1",
-        ports={9200: 9200, 9600: 9600},
-        environment={"discovery.type": "single-node"},
-        healthcheck=HealthCheck(
-            test="curl --fail https://localhost:9200/_cat/health -ku 'admin:admin' >/dev/null || exit 1",  # noqa: E501
-            interval=1,
-        ),
-    ):
-        with get_client() as client:
-            Movie.init(using=client)
-            for i, row in movies_dataframe.iterrows():
-                movie = Movie(
-                    meta={"id": i},
-                    title=row["Title"],
-                    year=row["Release Year"],
-                    director=row["Director"],
-                    cast=row["Cast"],
-                    genre=row["Genre"],
-                    wiki_page=row["Wiki Page"],
-                    ethnicity=row["Origin/Ethnicity"],
-                    plot=row["Plot"],
-                )
-                movie.save(using=client)
-            wait_for_write(
-                client=client, index_name=SOURCE_INDEX_NAME, expected_count=len(movies_dataframe)
-            )
-        yield SOURCE_INDEX_NAME
-
-
-@pytest.fixture
-def destination_index(opensearch_elements_mapping: dict) -> str:
-    with container_context(
-        image="opensearchproject/opensearch:2.11.1",
-        ports={9200: 9200, 9600: 9600},
-        environment={"discovery.type": "single-node"},
-        healthcheck=HealthCheck(
-            test="curl --fail https://localhost:9200/_cat/health -ku 'admin:admin' >/dev/null || exit 1",  # noqa: E501
-            interval=1,
-        ),
-    ):
-        with get_client() as client:
-            response = client.indices.create(
-                index=DESTINATION_INDEX_NAME, body=opensearch_elements_mapping
-            )
-            if not response["acknowledged"]:
-                raise RuntimeError(f"failed to create index: {response}")
-        yield DESTINATION_INDEX_NAME
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
-async def test_opensearch_source(source_index: str, movies_dataframe: pd.DataFrame):
-    indexer_config = OpenSearchIndexerConfig(index_name=source_index)
-    with tempfile.TemporaryDirectory() as tempdir:
-        tempdir_path = Path(tempdir)
-        connection_config = OpenSearchConnectionConfig(
-            access_config=OpenSearchAccessConfig(password="admin"),
-            username="admin",
-            hosts=["http://localhost:9200"],
-            use_ssl=True,
-        )
-        download_config = OpenSearchDownloaderConfig(download_dir=tempdir_path)
-        indexer = OpenSearchIndexer(
-            connection_config=connection_config, index_config=indexer_config
-        )
-        downloader = OpenSearchDownloader(
-            connection_config=connection_config, download_config=download_config
-        )
-        expected_num_files = len(movies_dataframe)
-        await source_connector_validation(
-            indexer=indexer,
-            downloader=downloader,
-            configs=SourceValidationConfigs(
-                test_id=CONNECTOR_TYPE,
-                expected_num_files=expected_num_files,
-                expected_number_indexed_file_data=1,
-                validate_downloaded_files=True,
-            ),
-        )
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
-def test_opensearch_source_precheck_fail_no_cluster():
-    indexer_config = OpenSearchIndexerConfig(index_name="index")
-
-    connection_config = OpenSearchConnectionConfig(
-        access_config=OpenSearchAccessConfig(password="admin"),
-        username="admin",
-        hosts=["http://localhost:9200"],
-        use_ssl=True,
-    )
-    indexer = OpenSearchIndexer(connection_config=connection_config, index_config=indexer_config)
-    with pytest.raises(SourceConnectionError):
-        indexer.precheck()
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
-def test_opensearch_source_precheck_fail_no_index(source_index: str):
-    indexer_config = OpenSearchIndexerConfig(index_name="index")
-
-    connection_config = OpenSearchConnectionConfig(
-        access_config=OpenSearchAccessConfig(password="admin"),
-        username="admin",
-        hosts=["http://localhost:9200"],
-        use_ssl=True,
-    )
-    indexer = OpenSearchIndexer(connection_config=connection_config, index_config=indexer_config)
-    with pytest.raises(SourceConnectionError):
-        indexer.precheck()
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
-async def test_opensearch_destination(
-    upload_file: Path,
-    destination_index: str,
-    tmp_path: Path,
-):
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
-        connector_type=CONNECTOR_TYPE,
-        identifier="mock file data",
-    )
-    connection_config = OpenSearchConnectionConfig(
-        access_config=OpenSearchAccessConfig(password="admin"),
-        username="admin",
-        hosts=["http://localhost:9200"],
-        use_ssl=True,
-    )
-    stager = OpenSearchUploadStager(
-        upload_stager_config=OpenSearchUploadStagerConfig(index_name=destination_index)
-    )
-
-    uploader = OpenSearchUploader(
-        connection_config=connection_config,
-        upload_config=OpenSearchUploaderConfig(index_name=destination_index),
-    )
-    staged_filepath = stager.run(
-        elements_filepath=upload_file,
-        file_data=file_data,
-        output_dir=tmp_path,
-        output_filename=upload_file.name,
-    )
-    uploader.precheck()
-    uploader.run(path=staged_filepath, file_data=file_data)
-
-    # Run validation
-    with staged_filepath.open() as f:
-        staged_elements = json.load(f)
-    expected_count = len(staged_elements)
-    with get_client() as client:
-        validate_count(client=client, expected_count=expected_count, index_name=destination_index)
-
-    # Rerun and make sure the same documents get updated
-    uploader.run(path=staged_filepath, file_data=file_data)
-    with get_client() as client:
-        validate_count(client=client, expected_count=expected_count, index_name=destination_index)
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
-def test_opensearch_destination_precheck_fail():
-    connection_config = OpenSearchConnectionConfig(
-        access_config=OpenSearchAccessConfig(password="admin"),
-        username="admin",
-        hosts=["http://localhost:9200"],
-        use_ssl=True,
-    )
-    uploader = OpenSearchUploader(
-        connection_config=connection_config,
-        upload_config=OpenSearchUploaderConfig(index_name="index"),
-    )
-    with pytest.raises(DestinationConnectionError):
-        uploader.precheck()
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
-def test_opensearch_destination_precheck_fail_no_index(destination_index: str):
-    connection_config = OpenSearchConnectionConfig(
-        access_config=OpenSearchAccessConfig(password="admin"),
-        username="admin",
-        hosts=["http://localhost:9200"],
-        use_ssl=True,
-    )
-    uploader = OpenSearchUploader(
-        connection_config=connection_config,
-        upload_config=OpenSearchUploaderConfig(index_name="index"),
-    )
-    with pytest.raises(DestinationConnectionError):
-        uploader.precheck()
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
-@pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
-def test_opensearch_stager(
-    request: TopRequest,
-    upload_file_str: str,
-    tmp_path: Path,
-):
-    upload_file: Path = request.getfixturevalue(upload_file_str)
-    stager = OpenSearchUploadStager(
-        upload_stager_config=OpenSearchUploadStagerConfig(index_name="mock_index")
-    )
-    stager_validation(
-        configs=StagerValidationConfigs(test_id=CONNECTOR_TYPE, expected_count=22),
-        input_file=upload_file,
-        stager=stager,
-        tmp_dir=tmp_path,
-    )

test/integration/connectors/sql/test_databricks_delta_tables.py

@@ -1,170 +0,0 @@
-import json
-import os
-import time
-from contextlib import contextmanager
-from pathlib import Path
-from uuid import uuid4
-
-import pytest
-from databricks.sql import connect
-from databricks.sql.client import Connection as DeltaTableConnection
-from databricks.sql.client import Cursor as DeltaTableCursor
-from pydantic import BaseModel, Secret, SecretStr
-from pytest_mock import MockerFixture
-
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG, env_setup_path
-from test.integration.utils import requires_env
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.logger import logger
-from unstructured_ingest.processes.connectors.sql.databricks_delta_tables import (
-    CONNECTOR_TYPE,
-    DatabricksDeltaTablesAccessConfig,
-    DatabricksDeltaTablesConnectionConfig,
-    DatabricksDeltaTablesUploader,
-    DatabricksDeltaTablesUploaderConfig,
-    DatabricksDeltaTablesUploadStager,
-)
-
-CATALOG = "utic-dev-tech-fixtures"
-
-
-class EnvData(BaseModel):
-    server_hostname: str
-    http_path: str
-    access_token: SecretStr
-
-
-def get_env_data() -> EnvData:
-    return EnvData(
-        server_hostname=os.environ["DATABRICKS_SERVER_HOSTNAME"],
-        http_path=os.environ["DATABRICKS_HTTP_PATH"],
-        access_token=os.environ["DATABRICKS_ACCESS_TOKEN"],
-    )
-
-
-def get_destination_schema(new_table_name: str) -> str:
-    p = Path(env_setup_path / "sql" / "databricks_delta_tables" / "destination" / "schema.sql")
-    with p.open() as f:
-        data_lines = f.readlines()
-    data_lines[0] = data_lines[0].replace("elements", new_table_name)
-    data = "".join([line.strip() for line in data_lines])
-    return data
-
-
-@contextmanager
-def get_connection() -> DeltaTableConnection:
-    env_data = get_env_data()
-    with connect(
-        server_hostname=env_data.server_hostname,
-        http_path=env_data.http_path,
-        access_token=env_data.access_token.get_secret_value(),
-    ) as connection:
-        yield connection
-
-
-@contextmanager
-def get_cursor() -> DeltaTableCursor:
-    with get_connection() as connection:
-        with connection.cursor() as cursor:
-            cursor.execute(f"USE CATALOG '{CATALOG}'")
-            yield cursor
-
-
-@pytest.fixture
-def destination_table() -> str:
-    random_id = str(uuid4())[:8]
-    table_name = f"elements_{random_id}"
-    destination_schema = get_destination_schema(new_table_name=table_name)
-    with get_cursor() as cursor:
-        logger.info(f"creating table: {table_name}")
-        cursor.execute(f"DROP TABLE IF EXISTS {table_name}")
-        cursor.execute(destination_schema)
-
-    yield table_name
-    with get_cursor() as cursor:
-        logger.info(f"dropping table: {table_name}")
-        cursor.execute(f"DROP TABLE IF EXISTS {table_name}")
-
-
-def validate_destination(expected_num_elements: int, table_name: str, retries=30, interval=1):
-    with get_cursor() as cursor:
-        for i in range(retries):
-            cursor.execute(f"SELECT COUNT(*) FROM {table_name}")
-            count = cursor.fetchone()[0]
-            if count == expected_num_elements:
-                break
-            logger.info(f"retry attempt {i}: expected {expected_num_elements} != count {count}")
-            time.sleep(interval)
-        assert (
-            count == expected_num_elements
-        ), f"dest check failed: got {count}, expected {expected_num_elements}"
-
-
-@pytest.mark.asyncio
-@pytest.mark.skip("Resources take too long to spin up to run in CI")
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
-@requires_env("DATABRICKS_SERVER_HOSTNAME", "DATABRICKS_HTTP_PATH", "DATABRICKS_ACCESS_TOKEN")
-async def test_databricks_delta_tables_destination(
-    upload_file: Path, temp_dir: Path, destination_table: str
-):
-    env_data = get_env_data()
-    mock_file_data = FileData(
-        identifier="mock file data",
-        connector_type=CONNECTOR_TYPE,
-        source_identifiers=SourceIdentifiers(filename=upload_file.name, fullpath=upload_file.name),
-    )
-    stager = DatabricksDeltaTablesUploadStager()
-    staged_path = stager.run(
-        elements_filepath=upload_file,
-        file_data=mock_file_data,
-        output_dir=temp_dir,
-        output_filename=upload_file.name,
-    )
-
-    assert staged_path.suffix == upload_file.suffix
-
-    uploader = DatabricksDeltaTablesUploader(
-        connection_config=DatabricksDeltaTablesConnectionConfig(
-            access_config=DatabricksDeltaTablesAccessConfig(
-                token=env_data.access_token.get_secret_value()
-            ),
-            http_path=env_data.http_path,
-            server_hostname=env_data.server_hostname,
-        ),
-        upload_config=DatabricksDeltaTablesUploaderConfig(
-            catalog=CATALOG, database="default", table_name=destination_table
-        ),
-    )
-    with staged_path.open("r") as f:
-        staged_data = json.load(f)
-    expected_num_elements = len(staged_data)
-    uploader.precheck()
-    uploader.run(path=staged_path, file_data=mock_file_data)
-    validate_destination(expected_num_elements=expected_num_elements, table_name=destination_table)
-
-
-def test_get_credentials_provider_with_client_id_and_secret(mocker: MockerFixture):
-    access_config = DatabricksDeltaTablesAccessConfig(
-        client_id="test_client_id", client_secret="test_client_secret"
-    )
-    connection_config = DatabricksDeltaTablesConnectionConfig(
-        access_config=Secret(access_config),
-        server_hostname="test_server_hostname",
-        http_path="test_http_path",
-    )
-
-    credentials_provider = connection_config.get_credentials_provider()
-    assert credentials_provider is not False
-    assert type(credentials_provider).__name__ == "function"
-
-
-def test_get_credentials_provider_with_token(mocker: MockerFixture):
-    access_config = DatabricksDeltaTablesAccessConfig(token="test_token")
-    connection_config = DatabricksDeltaTablesConnectionConfig(
-        access_config=Secret(access_config),
-        server_hostname="test_server_hostname",
-        http_path="test_http_path",
-    )
-
-    credentials_provider = connection_config.get_credentials_provider()
-    assert credentials_provider is False

test/integration/connectors/sql/test_postgres.py

@@ -1,201 +0,0 @@
-import json
-from pathlib import Path
-
-import pytest
-from _pytest.fixtures import TopRequest
-from psycopg2 import connect
-
-from test.integration.connectors.utils.constants import (
-    DESTINATION_TAG,
-    SOURCE_TAG,
-    SQL_TAG,
-    env_setup_path,
-)
-from test.integration.connectors.utils.docker_compose import docker_compose_context
-from test.integration.connectors.utils.validation.destination import (
-    StagerValidationConfigs,
-    stager_validation,
-)
-from test.integration.connectors.utils.validation.source import (
-    SourceValidationConfigs,
-    source_connector_validation,
-)
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.processes.connectors.sql.postgres import (
-    CONNECTOR_TYPE,
-    PostgresAccessConfig,
-    PostgresConnectionConfig,
-    PostgresDownloader,
-    PostgresDownloaderConfig,
-    PostgresIndexer,
-    PostgresIndexerConfig,
-    PostgresUploader,
-    PostgresUploadStager,
-)
-
-SEED_DATA_ROWS = 10
-
-
-@pytest.fixture
-def source_database_setup() -> str:
-    db_name = "test_db"
-    with docker_compose_context(docker_compose_path=env_setup_path / "sql" / "postgres" / "source"):
-        connection = connect(
-            user="unstructured",
-            password="test",
-            dbname=db_name,
-            host="localhost",
-            port=5433,
-        )
-        with connection.cursor() as cursor:
-            for i in range(SEED_DATA_ROWS):
-                sql_statment = f"INSERT INTO cars (brand, price) VALUES " f"('brand_{i}', {i})"
-                cursor.execute(sql_statment)
-            connection.commit()
-        yield db_name
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, SQL_TAG)
-async def test_postgres_source(temp_dir: Path, source_database_setup: str):
-    connect_params = {
-        "host": "localhost",
-        "port": 5433,
-        "database": "test_db",
-        "user": "unstructured",
-        "password": "test",
-    }
-    connection_config = PostgresConnectionConfig(
-        host=connect_params["host"],
-        port=connect_params["port"],
-        database=connect_params["database"],
-        username=connect_params["user"],
-        access_config=PostgresAccessConfig(password=connect_params["password"]),
-    )
-    indexer = PostgresIndexer(
-        connection_config=connection_config,
-        index_config=PostgresIndexerConfig(table_name="cars", id_column="car_id", batch_size=6),
-    )
-    downloader = PostgresDownloader(
-        connection_config=connection_config,
-        download_config=PostgresDownloaderConfig(fields=["car_id", "brand"], download_dir=temp_dir),
-    )
-    await source_connector_validation(
-        indexer=indexer,
-        downloader=downloader,
-        configs=SourceValidationConfigs(
-            test_id="postgres",
-            expected_num_files=SEED_DATA_ROWS,
-            expected_number_indexed_file_data=2,
-            validate_downloaded_files=True,
-        ),
-    )
-
-
-def validate_destination(
-    connect_params: dict,
-    expected_num_elements: int,
-    test_embedding: list[float],
-    expected_text: str,
-):
-    # Run the following validations:
-    # * Check that the number of records in the table match the expected value
-    # * Given the embedding, make sure it matches the associated text it belongs to
-    with connect(**connect_params) as connection:
-        cursor = connection.cursor()
-        query = "select count(*) from elements;"
-        cursor.execute(query)
-        count = cursor.fetchone()[0]
-        assert (
-            count == expected_num_elements
-        ), f"dest check failed: got {count}, expected {expected_num_elements}"
-
-        cursor.execute("SELECT embeddings FROM elements order by text limit 1")
-        similarity_query = (
-            f"SELECT text FROM elements ORDER BY embeddings <-> '{test_embedding}' LIMIT 1;"
-        )
-        cursor.execute(similarity_query)
-        res = cursor.fetchone()
-        assert res[0] == expected_text
-
-
-@pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
-async def test_postgres_destination(upload_file: Path, temp_dir: Path):
-    # the postgres destination connector doesn't leverage the file data but is required as an input,
-    # mocking it with arbitrary values to meet the base requirements:
-    mock_file_data = FileData(
-        identifier="mock file data",
-        connector_type=CONNECTOR_TYPE,
-        source_identifiers=SourceIdentifiers(filename=upload_file.name, fullpath=upload_file.name),
-    )
-    with docker_compose_context(
-        docker_compose_path=env_setup_path / "sql" / "postgres" / "destination"
-    ):
-        stager = PostgresUploadStager()
-        staged_path = stager.run(
-            elements_filepath=upload_file,
-            file_data=mock_file_data,
-            output_dir=temp_dir,
-            output_filename=upload_file.name,
-        )
-
-        # The stager should append the `.json` suffix to the output filename passed in.
-        assert staged_path.suffix == upload_file.suffix
-
-        connect_params = {
-            "host": "localhost",
-            "port": 5433,
-            "database": "elements",
-            "user": "unstructured",
-            "password": "test",
-        }
-
-        uploader = PostgresUploader(
-            connection_config=PostgresConnectionConfig(
-                host=connect_params["host"],
-                port=connect_params["port"],
-                database=connect_params["database"],
-                username=connect_params["user"],
-                access_config=PostgresAccessConfig(password=connect_params["password"]),
-            )
-        )
-        uploader.precheck()
-        uploader.run(path=staged_path, file_data=mock_file_data)
-
-        with staged_path.open("r") as f:
-            staged_data = json.load(f)
-
-        sample_element = staged_data[0]
-        expected_num_elements = len(staged_data)
-        validate_destination(
-            connect_params=connect_params,
-            expected_num_elements=expected_num_elements,
-            expected_text=sample_element["text"],
-            test_embedding=sample_element["embeddings"],
-        )
-
-        uploader.run(path=staged_path, file_data=mock_file_data)
-        validate_destination(
-            connect_params=connect_params,
-            expected_num_elements=expected_num_elements,
-            expected_text=sample_element["text"],
-            test_embedding=sample_element["embeddings"],
-        )
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
-@pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
-def test_postgres_stager(
-    request: TopRequest,
-    upload_file_str: str,
-    tmp_path: Path,
-):
-    upload_file: Path = request.getfixturevalue(upload_file_str)
-    stager = PostgresUploadStager()
-    stager_validation(
-        configs=StagerValidationConfigs(test_id=CONNECTOR_TYPE, expected_count=22),
-        input_file=upload_file,
-        stager=stager,
-        tmp_dir=tmp_path,
-    )