unstructured-ingest 0.3.13__py3-none-any.whl → 0.3.15__py3-none-any.whl

test/integration/connectors/test_azure_ai_search.py

@@ -23,9 +23,7 @@ from azure.search.documents.indexes.models import (
     VectorSearchProfile,
 )
 
-from test.integration.connectors.utils.constants import (
-    DESTINATION_TAG,
-)
+from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
     stager_validation,
@@ -195,7 +193,7 @@ def validate_count(
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @requires_env("AZURE_SEARCH_API_KEY")
 async def test_azure_ai_search_destination(
     upload_file: Path,
@@ -239,6 +237,7 @@ async def test_azure_ai_search_destination(
         validate_count(search_client=search_client, expected_count=expected_count)
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_azure_ai_search_stager(
     request: TopRequest,

test/integration/connectors/test_chroma.py

@@ -5,9 +5,7 @@ import chromadb
 import pytest
 from _pytest.fixtures import TopRequest
 
-from test.integration.connectors.utils.constants import (
-    DESTINATION_TAG,
-)
+from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
 from test.integration.connectors.utils.docker import HealthCheck, container_context
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
@@ -27,7 +25,7 @@ from unstructured_ingest.v2.processes.connectors.chroma import (
 @pytest.fixture
 def chroma_instance():
     with container_context(
-        image="chromadb/chroma:latest",
+        image="chromadb/chroma:0.6.2",
         ports={8000: 8000},
         name="chroma_int_test",
         healthcheck=HealthCheck(
@@ -64,7 +62,7 @@ def validate_collection(collection_name: str, num_embeddings: int):
     )
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 def test_chroma_destination(
     upload_file: Path,
     chroma_instance,
@@ -104,7 +102,7 @@ def test_chroma_destination(
 
 
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "stager")
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "stager", VECTOR_DB_TAG)
 def test_chroma_stager(
     request: TopRequest,
     upload_file_str: str,

test/integration/connectors/test_confluence.py

@@ -2,9 +2,7 @@ import os
 
 import pytest
 
-from test.integration.connectors.utils.constants import (
-    SOURCE_TAG,
-)
+from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
@@ -22,7 +20,7 @@ from unstructured_ingest.v2.processes.connectors.confluence import (
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
 @requires_env("CONFLUENCE_USER_EMAIL", "CONFLUENCE_API_TOKEN")
 async def test_confluence_source(temp_dir):
     # Retrieve environment variables
@@ -69,7 +67,7 @@ async def test_confluence_source(temp_dir):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, UNCATEGORIZED_TAG)
 @requires_env("CONFLUENCE_USER_EMAIL", "CONFLUENCE_API_TOKEN")
 async def test_confluence_source_large(temp_dir):
     # Retrieve environment variables

test/integration/connectors/test_delta_table.py

@@ -6,9 +6,7 @@ import pytest
 from deltalake import DeltaTable
 from fsspec import get_filesystem_class
 
-from test.integration.connectors.utils.constants import (
-    DESTINATION_TAG,
-)
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.delta_table import (
@@ -25,7 +23,7 @@ multiprocessing.set_start_method("spawn")
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 async def test_delta_table_destination_local(upload_file: Path, temp_dir: Path):
     destination_path = str(temp_dir)
     connection_config = DeltaTableConnectionConfig(
@@ -81,7 +79,7 @@ def get_aws_credentials() -> dict:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 @requires_env("S3_INGEST_TEST_ACCESS_KEY", "S3_INGEST_TEST_SECRET_KEY")
 async def test_delta_table_destination_s3(upload_file: Path, temp_dir: Path):
     aws_credentials = get_aws_credentials()
@@ -140,7 +138,7 @@ async def test_delta_table_destination_s3(upload_file: Path, temp_dir: Path):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
 @requires_env("S3_INGEST_TEST_ACCESS_KEY", "S3_INGEST_TEST_SECRET_KEY")
 async def test_delta_table_destination_s3_bad_creds(upload_file: Path, temp_dir: Path):
     aws_credentials = {

test/integration/connectors/test_lancedb.py

@@ -11,7 +11,7 @@ import pytest_asyncio
 from lancedb import AsyncConnection
 from upath import UPath
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.lancedb.aws import (
@@ -106,7 +106,7 @@ async def connection_with_uri(request, tmp_path: Path):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @pytest.mark.parametrize("connection_with_uri", ["local", "s3", "gcs", "az"], indirect=True)
 async def test_lancedb_destination(
     upload_file: Path,
@@ -164,7 +164,7 @@ async def test_lancedb_destination(
 
 
 class TestPrecheck:
-    @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+    @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
     @pytest.mark.parametrize("connection_with_uri", ["local", "s3", "gcs", "az"], indirect=True)
     def test_succeeds(
         self,

test/integration/connectors/test_milvus.py

@@ -13,7 +13,11 @@ from pymilvus import (
 )
 from pymilvus.milvus_client import IndexParams
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, env_setup_path
+from test.integration.connectors.utils.constants import (
+    DESTINATION_TAG,
+    VECTOR_DB_TAG,
+    env_setup_path,
+)
 from test.integration.connectors.utils.docker import healthcheck_wait
 from test.integration.connectors.utils.docker_compose import docker_compose_context
 from test.integration.connectors.utils.validation.destination import (
@@ -112,7 +116,7 @@ def validate_count(
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 async def test_milvus_destination(
     upload_file: Path,
     collection: str,
@@ -150,7 +154,7 @@ async def test_milvus_destination(
         validate_count(client=client, expected_count=expected_count)
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 def test_precheck_succeeds(collection: str):
     uploader = MilvusUploader(
         connection_config=MilvusConnectionConfig(uri=DB_URI),
@@ -159,7 +163,7 @@ def test_precheck_succeeds(collection: str):
     uploader.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 def test_precheck_fails_on_nonexistent_collection(collection: str):
     uploader = MilvusUploader(
         connection_config=MilvusConnectionConfig(uri=DB_URI),
@@ -174,7 +178,7 @@ def test_precheck_fails_on_nonexistent_collection(collection: str):
         uploader.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 def test_precheck_fails_on_nonexisting_db(collection: str):
     uploader = MilvusUploader(
         connection_config=MilvusConnectionConfig(uri=DB_URI),
@@ -187,6 +191,7 @@ def test_precheck_fails_on_nonexisting_db(collection: str):
         uploader.precheck()
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_milvus_stager(
     request: TopRequest,

test/integration/connectors/test_mongodb.py

@@ -13,7 +13,7 @@ from pymongo.database import Database
 from pymongo.mongo_client import MongoClient
 from pymongo.operations import SearchIndexModel
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, NOSQL_TAG, SOURCE_TAG
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
@@ -180,7 +180,7 @@ def validate_collection_vector(
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 @requires_env("MONGODB_URI", "MONGODB_DATABASE")
 async def test_mongodb_source(temp_dir: Path):
     env_data = get_env_data()
@@ -205,7 +205,7 @@ async def test_mongodb_source(temp_dir: Path):
     )
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 def test_mongodb_indexer_precheck_fail_no_host():
     indexer_config = MongoDBIndexerConfig(
         database="non-existent-database", collection="non-existent-database"
@@ -218,7 +218,7 @@ def test_mongodb_indexer_precheck_fail_no_host():
         indexer.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 @requires_env("MONGODB_URI", "MONGODB_DATABASE")
 def test_mongodb_indexer_precheck_fail_no_database():
     env_data = get_env_data()
@@ -233,7 +233,7 @@ def test_mongodb_indexer_precheck_fail_no_database():
         indexer.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 @requires_env("MONGODB_URI", "MONGODB_DATABASE")
 def test_mongodb_indexer_precheck_fail_no_collection():
     env_data = get_env_data()
@@ -249,7 +249,7 @@ def test_mongodb_indexer_precheck_fail_no_collection():
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 @requires_env("MONGODB_URI", "MONGODB_DATABASE")
 async def test_mongodb_destination(
     upload_file: Path,
@@ -289,7 +289,7 @@ async def test_mongodb_destination(
     validate_collection_count(collection=destination_collection, expected_records=expected_records)
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 def test_mongodb_uploader_precheck_fail_no_host():
     upload_config = MongoDBUploaderConfig(
         database="database",
@@ -303,7 +303,7 @@ def test_mongodb_uploader_precheck_fail_no_host():
         uploader.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 @requires_env("MONGODB_URI", "MONGODB_DATABASE")
 def test_mongodb_uploader_precheck_fail_no_database():
     env_data = get_env_data()
@@ -319,7 +319,7 @@ def test_mongodb_uploader_precheck_fail_no_database():
         uploader.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 @requires_env("MONGODB_URI", "MONGODB_DATABASE")
 def test_mongodb_uploader_precheck_fail_no_collection():
     env_data = get_env_data()

test/integration/connectors/test_neo4j.py

@@ -9,7 +9,7 @@ from neo4j import AsyncGraphDatabase, Driver, GraphDatabase
 from neo4j.exceptions import ServiceUnavailable
 from pytest_check import check
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, GRAPH_DB_TAG
 from test.integration.connectors.utils.docker import container_context
 from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.chunking import elements_from_base64_gzipped_json
@@ -51,7 +51,7 @@ def _neo4j_server():
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(DESTINATION_TAG, CONNECTOR_TYPE)
+@pytest.mark.tags(DESTINATION_TAG, CONNECTOR_TYPE, GRAPH_DB_TAG)
 async def test_neo4j_destination(upload_file: Path, tmp_path: Path):
     stager = Neo4jUploadStager()
     uploader = Neo4jUploader(
@@ -104,7 +104,7 @@ async def test_neo4j_destination(upload_file: Path, tmp_path: Path):
     await validate_uploaded_graph(modified_upload_file)
 
 
-@pytest.mark.tags(DESTINATION_TAG, CONNECTOR_TYPE)
+@pytest.mark.tags(DESTINATION_TAG, CONNECTOR_TYPE, GRAPH_DB_TAG)
 class TestPrecheck:
     @pytest.fixture
     def configured_uploader(self) -> Neo4jUploader:
@@ -199,13 +199,15 @@ async def validate_uploaded_graph(upload_file: Path):
     try:
         nodes_count = len((await driver.execute_query("MATCH (n) RETURN n"))[0])
         chunk_nodes_count = len(
-            (await driver.execute_query(f"MATCH (n: {Label.CHUNK}) RETURN n"))[0]
+            (await driver.execute_query(f"MATCH (n: {Label.CHUNK.value}) RETURN n"))[0]
         )
         document_nodes_count = len(
-            (await driver.execute_query(f"MATCH (n: {Label.DOCUMENT}) RETURN n"))[0]
+            (await driver.execute_query(f"MATCH (n: {Label.DOCUMENT.value}) RETURN n"))[0]
         )
         element_nodes_count = len(
-            (await driver.execute_query(f"MATCH (n: {Label.UNSTRUCTURED_ELEMENT}) RETURN n"))[0]
+            (await driver.execute_query(f"MATCH (n: {Label.UNSTRUCTURED_ELEMENT.value}) RETURN n"))[
+                0
+            ]
         )
         with check:
             assert nodes_count == expected_nodes_count
@@ -217,12 +219,18 @@ async def validate_uploaded_graph(upload_file: Path):
             assert element_nodes_count == expected_element_count
 
         records, _, _ = await driver.execute_query(
-            f"MATCH ()-[r:{Relationship.PART_OF_DOCUMENT}]->(:{Label.DOCUMENT}) RETURN r"
+            f"""
+            MATCH ()-[r:{Relationship.PART_OF_DOCUMENT.value}]->(:{Label.DOCUMENT.value})
+            RETURN r
+            """
         )
         part_of_document_count = len(records)
 
         records, _, _ = await driver.execute_query(
-            f"MATCH (:{Label.CHUNK})-[r:{Relationship.NEXT_CHUNK}]->(:{Label.CHUNK}) RETURN r"
+            f"""
+            MATCH (:{Label.CHUNK.value})-[r:{Relationship.NEXT_CHUNK.value}]->(:{Label.CHUNK.value})
+            RETURN r
+            """
         )
         next_chunk_count = len(records)
 

test/integration/connectors/test_notion.py

@@ -1,5 +1,8 @@
 import os
 
+import pytest
+
+from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     get_all_file_data,
@@ -8,6 +11,7 @@ from test.integration.connectors.utils.validation.source import (
 )
 from unstructured_ingest.v2.interfaces import Downloader, Indexer
 from unstructured_ingest.v2.processes.connectors.notion.connector import (
+    CONNECTOR_TYPE,
     NotionAccessConfig,
     NotionConnectionConfig,
     NotionDownloader,
@@ -17,6 +21,7 @@ from unstructured_ingest.v2.processes.connectors.notion.connector import (
 )
 
 
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
 def test_notion_source_database(temp_dir):
     # Retrieve environment variables
     notion_api_key = os.environ["NOTION_API_KEY"]
@@ -55,6 +60,7 @@ def test_notion_source_database(temp_dir):
     )
 
 
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
 def test_notion_source_page(temp_dir):
     # Retrieve environment variables
     notion_api_key = os.environ["NOTION_API_KEY"]
@@ -93,6 +99,7 @@ def test_notion_source_page(temp_dir):
     )
 
 
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
 def source_connector_validation(
     indexer: Indexer,
     downloader: Downloader,

test/integration/connectors/test_onedrive.py

@@ -5,9 +5,7 @@ from pathlib import Path
 import pytest
 from office365.graph_client import GraphClient
 
-from test.integration.connectors.utils.constants import (
-    DESTINATION_TAG,
-)
+from test.integration.connectors.utils.constants import BLOB_STORAGE_TAG, DESTINATION_TAG
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.onedrive import (
@@ -67,7 +65,7 @@ def get_connection_config():
     return connection_config
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, BLOB_STORAGE_TAG)
 @requires_env("MS_CLIENT_CRED", "MS_CLIENT_ID", "MS_TENANT_ID", "MS_USER_PNAME")
 @pytest.mark.xfail(
     reason="Issues with test setup on the provider side."

test/integration/connectors/test_pinecone.py

@@ -12,9 +12,7 @@ from _pytest.fixtures import TopRequest
 from pinecone import Pinecone, ServerlessSpec
 from pinecone.core.openapi.shared.exceptions import NotFoundException
 
-from test.integration.connectors.utils.constants import (
-    DESTINATION_TAG,
-)
+from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
     stager_validation,
@@ -109,11 +107,15 @@ def pinecone_index() -> Generator[str, None, None]:
 
 
 def validate_pinecone_index(
-    index_name: str, expected_num_of_vectors: int, retries=30, interval=1
+    index_name: str,
+    expected_num_of_vectors: int,
+    retries=30,
+    interval=1,
+    namespace: str = "default",
 ) -> None:
     # Because there's a delay for the index to catch up to the recent writes, add in a retry
     pinecone = Pinecone(api_key=get_api_key())
-    index = pinecone.Index(name=index_name)
+    index = pinecone.Index(name=index_name, namespace=namespace)
     vector_count = -1
     for i in range(retries):
         index_stats = index.describe_index_stats()
@@ -133,13 +135,15 @@ def validate_pinecone_index(
 
 @requires_env(API_KEY)
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 async def test_pinecone_destination(pinecone_index: str, upload_file: Path, temp_dir: Path):
+
     file_data = FileData(
         source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
         connector_type=CONNECTOR_TYPE,
         identifier="pinecone_mock_id",
     )
+
     connection_config = PineconeConnectionConfig(
         index_name=pinecone_index,
         access_config=PineconeAccessConfig(api_key=get_api_key()),
@@ -176,7 +180,7 @@ async def test_pinecone_destination(pinecone_index: str, upload_file: Path, temp
 
 @requires_env(API_KEY)
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @pytest.mark.skip(reason="TODO: get this to work")
 async def test_pinecone_destination_large_index(
     pinecone_index: str, upload_file: Path, temp_dir: Path
@@ -227,7 +231,67 @@ async def test_pinecone_destination_large_index(
 
 
 @requires_env(API_KEY)
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
+async def test_pinecone_destination_namespace(
+    pinecone_index: str, upload_file: Path, temp_dir: Path
+):
+    """
+    tests namespace functionality of destination connector.
+    """
+
+    # creates a file data structure.
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=CONNECTOR_TYPE,
+        identifier="pinecone_mock_id",
+    )
+
+    connection_config = PineconeConnectionConfig(
+        index_name=pinecone_index,
+        access_config=PineconeAccessConfig(api_key=get_api_key()),
+    )
+
+    stager_config = PineconeUploadStagerConfig()
+
+    stager = PineconeUploadStager(upload_stager_config=stager_config)
+    new_upload_file = stager.run(
+        elements_filepath=upload_file,
+        output_dir=temp_dir,
+        output_filename=upload_file.name,
+        file_data=file_data,
+    )
+
+    # here add namespace defintion
+    upload_config = PineconeUploaderConfig()
+    namespace_test_name = "user-1"
+    upload_config.namespace = namespace_test_name
+    uploader = PineconeUploader(connection_config=connection_config, upload_config=upload_config)
+    uploader.precheck()
+
+    uploader.run(path=new_upload_file, file_data=file_data)
+    with new_upload_file.open() as f:
+        staged_content = json.load(f)
+    expected_num_of_vectors = len(staged_content)
+    logger.info("validating first upload")
+    validate_pinecone_index(
+        index_name=pinecone_index,
+        expected_num_of_vectors=expected_num_of_vectors,
+        namespace=namespace_test_name,
+    )
+
+    # Rerun uploader and make sure no duplicates exist
+    uploader.run(path=new_upload_file, file_data=file_data)
+    logger.info("validating second upload")
+    validate_pinecone_index(
+        index_name=pinecone_index,
+        expected_num_of_vectors=expected_num_of_vectors,
+        namespace=namespace_test_name,
+    )
+
+
+@requires_env(API_KEY)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 def test_large_metadata(pinecone_index: str, tmp_path: Path, upload_file: Path):
     stager = PineconeUploadStager()
     uploader = PineconeUploader(
@@ -272,6 +336,7 @@ def test_large_metadata(pinecone_index: str, tmp_path: Path, upload_file: Path):
     validate_pinecone_index(pinecone_index, 1, interval=5)
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_pinecone_stager(
     request: TopRequest,

test/integration/connectors/test_qdrant.py

@@ -9,7 +9,7 @@ import pytest
 from _pytest.fixtures import TopRequest
 from qdrant_client import AsyncQdrantClient
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
 from test.integration.connectors.utils.docker import container_context
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
@@ -75,7 +75,7 @@ async def validate_upload(client: AsyncQdrantClient, upload_file: Path):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(LOCAL_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant")
+@pytest.mark.tags(LOCAL_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant", VECTOR_DB_TAG)
 async def test_qdrant_destination_local(upload_file: Path, tmp_path: Path):
     connection_kwargs = {"path": str(tmp_path / "qdrant")}
     async with qdrant_client(connection_kwargs) as client:
@@ -117,7 +117,7 @@ def docker_context():
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(SERVER_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant")
+@pytest.mark.tags(SERVER_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant", VECTOR_DB_TAG)
 async def test_qdrant_destination_server(upload_file: Path, tmp_path: Path, docker_context):
     connection_kwargs = {"location": "http://localhost:6333"}
     async with qdrant_client(connection_kwargs) as client:
@@ -153,7 +153,7 @@ async def test_qdrant_destination_server(upload_file: Path, tmp_path: Path, dock
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(SERVER_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant")
+@pytest.mark.tags(SERVER_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant", VECTOR_DB_TAG)
 @requires_env("QDRANT_API_KEY", "QDRANT_SERVER_URL")
 async def test_qdrant_destination_cloud(upload_file: Path, tmp_path: Path):
     server_url = os.environ["QDRANT_SERVER_URL"]
@@ -197,6 +197,7 @@ async def test_qdrant_destination_cloud(upload_file: Path, tmp_path: Path):
         await validate_upload(client=client, upload_file=upload_file)
 
 
+@pytest.mark.tags(SERVER_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant", VECTOR_DB_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_qdrant_stager(
     request: TopRequest,

test/integration/connectors/test_redis.py

@@ -9,7 +9,7 @@ import pytest
 from redis import exceptions as redis_exceptions
 from redis.asyncio import Redis, from_url
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, NOSQL_TAG
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.redisdb import (
@@ -96,7 +96,7 @@ async def redis_destination_test(
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(REDIS_CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(REDIS_CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 @requires_env("AZURE_REDIS_INGEST_TEST_PASSWORD")
 async def test_redis_destination_azure_with_password(upload_file: Path, tmp_path: Path):
     connection_kwargs = {
@@ -110,7 +110,7 @@ async def test_redis_destination_azure_with_password(upload_file: Path, tmp_path
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(REDIS_CONNECTOR_TYPE, DESTINATION_TAG, "redis")
+@pytest.mark.tags(REDIS_CONNECTOR_TYPE, DESTINATION_TAG, "redis", NOSQL_TAG)
 @requires_env("AZURE_REDIS_INGEST_TEST_PASSWORD")
 async def test_redis_destination_azure_with_uri(upload_file: Path, tmp_path: Path):
     connection_kwargs = {}