unstructured-ingest 0.3.8__tar.gz → 0.3.9__tar.gz

{unstructured_ingest-0.3.8/unstructured_ingest.egg-info → unstructured_ingest-0.3.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.3.8
+Version: 0.3.9
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,13 +22,14 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: opentelemetry-sdk
-Requires-Dist: pandas
-Requires-Dist: python-dateutil
-Requires-Dist: dataclasses_json
 Requires-Dist: tqdm
 Requires-Dist: click
+Requires-Dist: python-dateutil
+Requires-Dist: ndjson
 Requires-Dist: pydantic>=2.7
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: pandas
+Requires-Dist: dataclasses_json
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
@@ -81,13 +82,13 @@ Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: requests; extra == "confluence"
 Requires-Dist: atlassian-python-api; extra == "confluence"
+Requires-Dist: requests; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: deltalake; extra == "delta-table"
 Requires-Dist: boto3; extra == "delta-table"
+Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord-py; extra == "discord"
 Provides-Extra: dropbox
@@ -99,18 +100,18 @@ Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
 Requires-Dist: gcsfs; extra == "gcs"
-Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: fsspec; extra == "gcs"
+Requires-Dist: bs4; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
+Requires-Dist: requests; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -123,15 +124,18 @@ Provides-Extra: milvus
 Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
+Provides-Extra: neo4j
+Requires-Dist: cymple; extra == "neo4j"
+Requires-Dist: neo4j; extra == "neo4j"
 Provides-Extra: notion
 Requires-Dist: backoff; extra == "notion"
-Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Provides-Extra: onedrive
-Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
+Requires-Dist: bs4; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
@@ -174,8 +178,8 @@ Requires-Dist: requests; extra == "vectara"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: openai; extra == "embed-octoai"
 Requires-Dist: tiktoken; extra == "embed-octoai"
+Requires-Dist: openai; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -183,8 +187,8 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: openai; extra == "openai"
 Requires-Dist: tiktoken; extra == "openai"
+Requires-Dist: openai; extra == "openai"
 Provides-Extra: bedrock
 Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: togetherai

{unstructured_ingest-0.3.8 → unstructured_ingest-0.3.9}/setup.py

@@ -108,6 +108,7 @@ connectors_reqs = {
     "lancedb": load_requirements("requirements/connectors/lancedb.in"),
     "milvus": load_requirements("requirements/connectors/milvus.in"),
     "mongodb": load_requirements("requirements/connectors/mongodb.in"),
+    "neo4j": load_requirements("requirements/connectors/neo4j.in"),
     "notion": load_requirements("requirements/connectors/notion.in"),
     "onedrive": load_requirements("requirements/connectors/onedrive.in"),
     "opensearch": load_requirements("requirements/connectors/opensearch.in"),

{unstructured_ingest-0.3.8 → unstructured_ingest-0.3.9}/test/integration/chunkers/test_chunkers.py

@@ -29,14 +29,3 @@ async def test_chunker_api(chunker_file: Path, strategy: str):
     chunker = Chunker(config=chunker_config)
     results = await chunker.run_async(elements_filepath=chunker_file)
     assert results
-
-
-@pytest.mark.parametrize("chunker_file", chunker_files, ids=[path.name for path in chunker_files])
-@pytest.mark.parametrize("strategy", ["basic", "by_title"])
-def test_chunker_basic(chunker_file: Path, strategy: str):
-    chunker_config = ChunkerConfig(
-        chunking_strategy=strategy,
-    )
-    chunker = Chunker(config=chunker_config)
-    results = chunker.run(elements_filepath=chunker_file)
-    assert results

{unstructured_ingest-0.3.8 → unstructured_ingest-0.3.9}/test/integration/connectors/conftest.py

@@ -6,7 +6,7 @@ import pytest
 
 from unstructured_ingest.v2.logger import logger
 
-FILENAME = "DA-1p-with-duplicate-pages.pdf.json"
+FILENAME = Path("DA-1p-with-duplicate-pages.pdf.json")
 
 
 @pytest.fixture
@@ -19,6 +19,16 @@ def upload_file() -> Path:
     return upload_file
 
 
+@pytest.fixture
+def upload_file_ndjson() -> Path:
+    int_test_dir = Path(__file__).parent
+    assets_dir = int_test_dir / "assets"
+    upload_file = assets_dir / FILENAME.with_suffix(".ndjson")
+    assert upload_file.exists()
+    assert upload_file.is_file()
+    return upload_file
+
+
 @pytest.fixture
 def temp_dir() -> Generator[Path, None, None]:
     with tempfile.TemporaryDirectory() as temp_dir:

{unstructured_ingest-0.3.8 → unstructured_ingest-0.3.9}/test/integration/connectors/databricks_tests/test_volumes_native.py

@@ -11,8 +11,8 @@ from databricks.sdk import WorkspaceClient
 from databricks.sdk.errors.platform import NotFound
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG
-from test.integration.connectors.utils.validation import (
-    ValidationConfigs,
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
     source_connector_validation,
 )
 from test.integration.utils import requires_env
@@ -82,7 +82,7 @@ async def test_volumes_native_source():
         await source_connector_validation(
             indexer=indexer,
             downloader=downloader,
-            configs=ValidationConfigs(
+            configs=SourceValidationConfigs(
                 test_id="databricks_volumes_native",
                 expected_num_files=1,
             ),
@@ -156,6 +156,7 @@ async def test_volumes_native_destination(upload_file: Path):
                 catalog=env_data.catalog,
             ),
         )
+        uploader.precheck()
         if uploader.is_async():
             await uploader.run_async(path=upload_file, file_data=file_data)
         else:

unstructured_ingest-0.3.9/test/integration/connectors/duckdb/conftest.py

@@ -0,0 +1,14 @@
+from pathlib import Path
+
+import pytest
+
+int_test_dir = Path(__file__).parent
+assets_dir = int_test_dir / "assets"
+
+
+@pytest.fixture
+def duckdb_schema() -> Path:
+    schema_file = assets_dir / "duckdb-schema.sql"
+    assert schema_file.exists()
+    assert schema_file.is_file()
+    return schema_file

unstructured_ingest-0.3.9/test/integration/connectors/duckdb/test_duckdb.py

@@ -0,0 +1,89 @@
+import json
+from pathlib import Path
+
+import duckdb
+import pytest
+from _pytest.fixtures import TopRequest
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.validation.destination import (
+    StagerValidationConfigs,
+    stager_validation,
+)
+from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.duckdb.duckdb import (
+    CONNECTOR_TYPE,
+    DuckDBConnectionConfig,
+    DuckDBUploader,
+    DuckDBUploaderConfig,
+    DuckDBUploadStager,
+)
+
+
+@pytest.fixture
+def provisioned_db_file(duckdb_schema: Path, temp_dir: Path) -> Path:
+    db_path = Path(temp_dir) / "temp_duck.db"
+    with duckdb.connect(database=db_path) as duckdb_connection:
+        with duckdb_schema.open("r") as f:
+            query = f.read()
+        duckdb_connection.execute(query)
+        duckdb_connection.close()
+    return db_path
+
+
+def validate_duckdb_destination(db_path: Path, expected_num_elements: int):
+    conn = None
+    try:
+        conn = duckdb.connect(db_path)
+        _results = conn.sql("select count(*) from elements").fetchall()
+        _count = _results[0][0]
+        assert (
+            _count == expected_num_elements
+        ), f"dest check failed: got {_count}, expected {expected_num_elements}"
+        conn.close()
+    finally:
+        if conn:
+            conn.close()
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "duckdb")
+def test_duckdb_destination(upload_file: Path, provisioned_db_file: Path, temp_dir: Path):
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock-file-data",
+    )
+
+    stager = DuckDBUploadStager()
+    staged_path = stager.run(
+        elements_filepath=upload_file,
+        file_data=file_data,
+        output_dir=temp_dir,
+        output_filename=upload_file.name,
+    )
+
+    connection_config = DuckDBConnectionConfig(database=str(provisioned_db_file))
+    upload_config = DuckDBUploaderConfig()
+    uploader = DuckDBUploader(connection_config=connection_config, upload_config=upload_config)
+
+    uploader.run(path=staged_path, file_data=file_data)
+
+    with staged_path.open() as f:
+        data = json.load(f)
+    validate_duckdb_destination(db_path=provisioned_db_file, expected_num_elements=len(data))
+
+
+@pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
+def test_duckdb_stager(
+    request: TopRequest,
+    upload_file_str: str,
+    tmp_path: Path,
+):
+    upload_file: Path = request.getfixturevalue(upload_file_str)
+    stager = DuckDBUploadStager()
+    stager_validation(
+        configs=StagerValidationConfigs(test_id=CONNECTOR_TYPE, expected_count=22),
+        input_file=upload_file,
+        stager=stager,
+        tmp_dir=tmp_path,
+    )

unstructured_ingest-0.3.9/test/integration/connectors/duckdb/test_motherduck.py

@@ -0,0 +1,95 @@
+import os
+import uuid
+from pathlib import Path
+from typing import Generator
+
+import duckdb
+import pandas as pd
+import pytest
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.duckdb.motherduck import (
+    CONNECTOR_TYPE,
+    MotherDuckAccessConfig,
+    MotherDuckConnectionConfig,
+    MotherDuckUploader,
+    MotherDuckUploaderConfig,
+    MotherDuckUploadStager,
+)
+
+
+@pytest.fixture
+def md_token() -> str:
+    motherduck_token = os.getenv("MOTHERDUCK_TOKEN", None)
+    assert motherduck_token
+    return motherduck_token
+
+
+@pytest.fixture
+def provisioned_db(md_token: str, duckdb_schema: Path) -> Generator[str, None, None]:
+    database_name = f"test_{str(uuid.uuid4()).replace('-', '_')}"
+    try:
+        with duckdb.connect(f"md:?motherduck_token={md_token}") as md_conn:
+            with duckdb_schema.open("r") as f:
+                query = f.read()
+            md_conn.execute(f"CREATE DATABASE {database_name}")
+            md_conn.execute(f"USE {database_name}")
+            md_conn.execute(query)
+            md_conn.close()
+        yield database_name
+    finally:
+        with duckdb.connect(f"md:?motherduck_token={md_token}") as md_conn:
+            md_conn.execute(f"DROP DATABASE {database_name}")
+            md_conn.close()
+
+
+def validate_motherduck_destination(database: str, expected_num_elements: int, md_token: str):
+    conn = None
+    try:
+        conn = duckdb.connect(f"md:?motherduck_token={md_token}")
+        conn.execute(f"USE {database}")
+        _results = conn.sql("select count(*) from elements").fetchall()
+        _count = _results[0][0]
+        assert (
+            _count == expected_num_elements
+        ), f"dest check failed: got {_count}, expected {expected_num_elements}"
+        conn.close()
+    finally:
+        if conn:
+            conn.close()
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@requires_env("MOTHERDUCK_TOKEN")
+def test_motherduck_destination(
+    md_token: str, upload_file: Path, provisioned_db: str, temp_dir: Path
+):
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=CONNECTOR_TYPE,
+        identifier="mock-file-data",
+    )
+
+    stager = MotherDuckUploadStager()
+    staged_path = stager.run(
+        elements_filepath=upload_file,
+        file_data=file_data,
+        output_dir=temp_dir,
+        output_filename=upload_file.name,
+    )
+
+    access_config = MotherDuckAccessConfig(md_token=md_token)
+    connection_config = MotherDuckConnectionConfig(
+        database=provisioned_db, access_config=access_config
+    )
+    upload_config = MotherDuckUploaderConfig()
+    uploader = MotherDuckUploader(connection_config=connection_config, upload_config=upload_config)
+
+    uploader.run(path=staged_path, file_data=file_data)
+
+    staged_df = pd.read_json(staged_path, orient="records", lines=True)
+    validate_motherduck_destination(
+        database=provisioned_db, expected_num_elements=len(staged_df), md_token=md_token
+    )

{unstructured_ingest-0.3.8 → unstructured_ingest-0.3.9}/test/integration/connectors/elasticsearch/test_elasticsearch.py

@@ -5,16 +5,20 @@ import time
 from contextlib import contextmanager
 from pathlib import Path
 from typing import Generator
-
+from test.integration.connectors.utils.validation.destination import (
+    StagerValidationConfigs,
+    stager_validation,
+)
 import pandas as pd
 import pytest
+from _pytest.fixtures import TopRequest
 from elasticsearch import Elasticsearch as ElasticsearchClient
 from elasticsearch.helpers import bulk
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG
 from test.integration.connectors.utils.docker import HealthCheck, container_context
-from test.integration.connectors.utils.validation import (
-    ValidationConfigs,
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
     source_connector_validation,
 )
 from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
@@ -194,7 +198,7 @@ async def test_elasticsearch_source(source_index: str, movies_dataframe: pd.Data
         await source_connector_validation(
             indexer=indexer,
             downloader=downloader,
-            configs=ValidationConfigs(
+            configs=SourceValidationConfigs(
                 test_id=CONNECTOR_TYPE,
                 expected_num_files=expected_num_files,
                 expected_number_indexed_file_data=1,
@@ -306,3 +310,21 @@ def test_elasticsearch_destination_precheck_fail_no_index(destination_index: str
     )
     with pytest.raises(DestinationConnectionError):
         uploader.precheck()
+
+
+@pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
+def test_elasticsearch_stager(
+    request: TopRequest,
+    upload_file_str: str,
+    tmp_path: Path,
+):
+    upload_file: Path = request.getfixturevalue(upload_file_str)
+    stager = ElasticsearchUploadStager(
+        upload_stager_config=ElasticsearchUploadStagerConfig(index_name="mock_index")
+    )
+    stager_validation(
+        configs=StagerValidationConfigs(test_id=CONNECTOR_TYPE, expected_count=22),
+        input_file=upload_file,
+        stager=stager,
+        tmp_dir=tmp_path,
+    )

{unstructured_ingest-0.3.8 → unstructured_ingest-0.3.9}/test/integration/connectors/elasticsearch/test_opensearch.py

@@ -7,12 +7,17 @@ from typing import Generator
 
 import pandas as pd
 import pytest
+from _pytest.fixtures import TopRequest
 from opensearchpy import Document, Keyword, OpenSearch, Text
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG
 from test.integration.connectors.utils.docker import HealthCheck, container_context
-from test.integration.connectors.utils.validation import (
-    ValidationConfigs,
+from test.integration.connectors.utils.validation.destination import (
+    StagerValidationConfigs,
+    stager_validation,
+)
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
     source_connector_validation,
 )
 from unstructured_ingest.error import (
@@ -183,7 +188,7 @@ async def test_opensearch_source(source_index: str, movies_dataframe: pd.DataFra
         await source_connector_validation(
             indexer=indexer,
             downloader=downloader,
-            configs=ValidationConfigs(
+            configs=SourceValidationConfigs(
                 test_id=CONNECTOR_TYPE,
                 expected_num_files=expected_num_files,
                 expected_number_indexed_file_data=1,
@@ -300,3 +305,21 @@ def test_opensearch_destination_precheck_fail_no_index(destination_index: str):
     )
     with pytest.raises(DestinationConnectionError):
         uploader.precheck()
+
+
+@pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
+def test_opensearch_stager(
+    request: TopRequest,
+    upload_file_str: str,
+    tmp_path: Path,
+):
+    upload_file: Path = request.getfixturevalue(upload_file_str)
+    stager = OpenSearchUploadStager(
+        upload_stager_config=OpenSearchUploadStagerConfig(index_name="mock_index")
+    )
+    stager_validation(
+        configs=StagerValidationConfigs(test_id=CONNECTOR_TYPE, expected_count=22),
+        input_file=upload_file,
+        stager=stager,
+        tmp_dir=tmp_path,
+    )

unstructured_ingest-0.3.9/test/integration/connectors/sql/test_postgres.py

@@ -0,0 +1,191 @@
+import json
+from pathlib import Path
+
+import pytest
+from _pytest.fixtures import TopRequest
+from psycopg2 import connect
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG, env_setup_path
+from test.integration.connectors.utils.docker_compose import docker_compose_context
+from test.integration.connectors.utils.validation.destination import (
+    StagerValidationConfigs,
+    stager_validation,
+)
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
+from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.processes.connectors.sql.postgres import (
+    CONNECTOR_TYPE,
+    PostgresAccessConfig,
+    PostgresConnectionConfig,
+    PostgresDownloader,
+    PostgresDownloaderConfig,
+    PostgresIndexer,
+    PostgresIndexerConfig,
+    PostgresUploader,
+    PostgresUploadStager,
+)
+
+SEED_DATA_ROWS = 20
+
+
+@pytest.fixture
+def source_database_setup() -> str:
+    db_name = "test_db"
+    with docker_compose_context(docker_compose_path=env_setup_path / "sql" / "postgres" / "source"):
+        connection = connect(
+            user="unstructured",
+            password="test",
+            dbname=db_name,
+            host="localhost",
+            port=5433,
+        )
+        with connection.cursor() as cursor:
+            for i in range(SEED_DATA_ROWS):
+                sql_statment = f"INSERT INTO cars (brand, price) VALUES " f"('brand_{i}', {i})"
+                cursor.execute(sql_statment)
+            connection.commit()
+        yield db_name
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "sql")
+async def test_postgres_source(temp_dir: Path, source_database_setup: str):
+    connect_params = {
+        "host": "localhost",
+        "port": 5433,
+        "database": "test_db",
+        "user": "unstructured",
+        "password": "test",
+    }
+    connection_config = PostgresConnectionConfig(
+        host=connect_params["host"],
+        port=connect_params["port"],
+        database=connect_params["database"],
+        username=connect_params["user"],
+        access_config=PostgresAccessConfig(password=connect_params["password"]),
+    )
+    indexer = PostgresIndexer(
+        connection_config=connection_config,
+        index_config=PostgresIndexerConfig(table_name="cars", id_column="car_id", batch_size=5),
+    )
+    downloader = PostgresDownloader(
+        connection_config=connection_config,
+        download_config=PostgresDownloaderConfig(fields=["car_id", "brand"], download_dir=temp_dir),
+    )
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="postgres",
+            expected_num_files=SEED_DATA_ROWS,
+            expected_number_indexed_file_data=4,
+            validate_downloaded_files=True,
+        ),
+    )
+
+
+def validate_destination(
+    connect_params: dict,
+    expected_num_elements: int,
+    test_embedding: list[float],
+    expected_text: str,
+):
+    # Run the following validations:
+    # * Check that the number of records in the table match the expected value
+    # * Given the embedding, make sure it matches the associated text it belongs to
+    with connect(**connect_params) as connection:
+        cursor = connection.cursor()
+        query = "select count(*) from elements;"
+        cursor.execute(query)
+        count = cursor.fetchone()[0]
+        assert (
+            count == expected_num_elements
+        ), f"dest check failed: got {count}, expected {expected_num_elements}"
+
+        cursor.execute("SELECT embeddings FROM elements order by text limit 1")
+        similarity_query = (
+            f"SELECT text FROM elements ORDER BY embeddings <-> '{test_embedding}' LIMIT 1;"
+        )
+        cursor.execute(similarity_query)
+        res = cursor.fetchone()
+        assert res[0] == expected_text
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "sql")
+async def test_postgres_destination(upload_file: Path, temp_dir: Path):
+    # the postgres destination connector doesn't leverage the file data but is required as an input,
+    # mocking it with arbitrary values to meet the base requirements:
+    mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
+    with docker_compose_context(
+        docker_compose_path=env_setup_path / "sql" / "postgres" / "destination"
+    ):
+        stager = PostgresUploadStager()
+        staged_path = stager.run(
+            elements_filepath=upload_file,
+            file_data=mock_file_data,
+            output_dir=temp_dir,
+            output_filename=upload_file.name,
+        )
+
+        # The stager should append the `.json` suffix to the output filename passed in.
+        assert staged_path.suffix == upload_file.suffix
+
+        connect_params = {
+            "host": "localhost",
+            "port": 5433,
+            "database": "elements",
+            "user": "unstructured",
+            "password": "test",
+        }
+
+        uploader = PostgresUploader(
+            connection_config=PostgresConnectionConfig(
+                host=connect_params["host"],
+                port=connect_params["port"],
+                database=connect_params["database"],
+                username=connect_params["user"],
+                access_config=PostgresAccessConfig(password=connect_params["password"]),
+            )
+        )
+        uploader.precheck()
+        uploader.run(path=staged_path, file_data=mock_file_data)
+
+        with staged_path.open("r") as f:
+            staged_data = json.load(f)
+
+        sample_element = staged_data[0]
+        expected_num_elements = len(staged_data)
+        validate_destination(
+            connect_params=connect_params,
+            expected_num_elements=expected_num_elements,
+            expected_text=sample_element["text"],
+            test_embedding=sample_element["embeddings"],
+        )
+
+        uploader.run(path=staged_path, file_data=mock_file_data)
+        validate_destination(
+            connect_params=connect_params,
+            expected_num_elements=expected_num_elements,
+            expected_text=sample_element["text"],
+            test_embedding=sample_element["embeddings"],
+        )
+
+
+@pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
+def test_postgres_stager(
+    request: TopRequest,
+    upload_file_str: str,
+    tmp_path: Path,
+):
+    upload_file: Path = request.getfixturevalue(upload_file_str)
+    stager = PostgresUploadStager()
+    stager_validation(
+        configs=StagerValidationConfigs(test_id=CONNECTOR_TYPE, expected_count=22),
+        input_file=upload_file,
+        stager=stager,
+        tmp_dir=tmp_path,
+    )