unstructured-ingest 0.3.13__py3-none-any.whl → 0.3.14__py3-none-any.whl

test/integration/connectors/test_lancedb.py

@@ -11,7 +11,7 @@ import pytest_asyncio
 from lancedb import AsyncConnection
 from upath import UPath
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
 from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.lancedb.aws import (
@@ -106,7 +106,7 @@ async def connection_with_uri(request, tmp_path: Path):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @pytest.mark.parametrize("connection_with_uri", ["local", "s3", "gcs", "az"], indirect=True)
 async def test_lancedb_destination(
     upload_file: Path,
@@ -164,7 +164,7 @@ async def test_lancedb_destination(
 
 
 class TestPrecheck:
-    @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+    @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
     @pytest.mark.parametrize("connection_with_uri", ["local", "s3", "gcs", "az"], indirect=True)
     def test_succeeds(
         self,

test/integration/connectors/test_milvus.py

@@ -13,7 +13,11 @@ from pymilvus import (
 )
 from pymilvus.milvus_client import IndexParams
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, env_setup_path
+from test.integration.connectors.utils.constants import (
+    DESTINATION_TAG,
+    VECTOR_DB_TAG,
+    env_setup_path,
+)
 from test.integration.connectors.utils.docker import healthcheck_wait
 from test.integration.connectors.utils.docker_compose import docker_compose_context
 from test.integration.connectors.utils.validation.destination import (
@@ -112,7 +116,7 @@ def validate_count(
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 async def test_milvus_destination(
     upload_file: Path,
     collection: str,
@@ -150,7 +154,7 @@ async def test_milvus_destination(
         validate_count(client=client, expected_count=expected_count)
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 def test_precheck_succeeds(collection: str):
     uploader = MilvusUploader(
         connection_config=MilvusConnectionConfig(uri=DB_URI),
@@ -159,7 +163,7 @@ def test_precheck_succeeds(collection: str):
     uploader.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 def test_precheck_fails_on_nonexistent_collection(collection: str):
     uploader = MilvusUploader(
         connection_config=MilvusConnectionConfig(uri=DB_URI),
@@ -174,7 +178,7 @@ def test_precheck_fails_on_nonexistent_collection(collection: str):
         uploader.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 def test_precheck_fails_on_nonexisting_db(collection: str):
     uploader = MilvusUploader(
         connection_config=MilvusConnectionConfig(uri=DB_URI),
@@ -187,6 +191,7 @@ def test_precheck_fails_on_nonexisting_db(collection: str):
         uploader.precheck()
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_milvus_stager(
     request: TopRequest,

test/integration/connectors/test_mongodb.py

@@ -13,7 +13,7 @@ from pymongo.database import Database
 from pymongo.mongo_client import MongoClient
 from pymongo.operations import SearchIndexModel
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG, SOURCE_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, NOSQL_TAG, SOURCE_TAG
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
@@ -180,7 +180,7 @@ def validate_collection_vector(
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 @requires_env("MONGODB_URI", "MONGODB_DATABASE")
 async def test_mongodb_source(temp_dir: Path):
     env_data = get_env_data()
@@ -205,7 +205,7 @@ async def test_mongodb_source(temp_dir: Path):
     )
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 def test_mongodb_indexer_precheck_fail_no_host():
     indexer_config = MongoDBIndexerConfig(
         database="non-existent-database", collection="non-existent-database"
@@ -218,7 +218,7 @@ def test_mongodb_indexer_precheck_fail_no_host():
         indexer.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 @requires_env("MONGODB_URI", "MONGODB_DATABASE")
 def test_mongodb_indexer_precheck_fail_no_database():
     env_data = get_env_data()
@@ -233,7 +233,7 @@ def test_mongodb_indexer_precheck_fail_no_database():
         indexer.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, NOSQL_TAG)
 @requires_env("MONGODB_URI", "MONGODB_DATABASE")
 def test_mongodb_indexer_precheck_fail_no_collection():
     env_data = get_env_data()
@@ -249,7 +249,7 @@ def test_mongodb_indexer_precheck_fail_no_collection():
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 @requires_env("MONGODB_URI", "MONGODB_DATABASE")
 async def test_mongodb_destination(
     upload_file: Path,
@@ -289,7 +289,7 @@ async def test_mongodb_destination(
     validate_collection_count(collection=destination_collection, expected_records=expected_records)
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 def test_mongodb_uploader_precheck_fail_no_host():
     upload_config = MongoDBUploaderConfig(
         database="database",
@@ -303,7 +303,7 @@ def test_mongodb_uploader_precheck_fail_no_host():
         uploader.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 @requires_env("MONGODB_URI", "MONGODB_DATABASE")
 def test_mongodb_uploader_precheck_fail_no_database():
     env_data = get_env_data()
@@ -319,7 +319,7 @@ def test_mongodb_uploader_precheck_fail_no_database():
         uploader.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 @requires_env("MONGODB_URI", "MONGODB_DATABASE")
 def test_mongodb_uploader_precheck_fail_no_collection():
     env_data = get_env_data()

test/integration/connectors/test_neo4j.py

@@ -9,7 +9,7 @@ from neo4j import AsyncGraphDatabase, Driver, GraphDatabase
 from neo4j.exceptions import ServiceUnavailable
 from pytest_check import check
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, GRAPH_DB_TAG
 from test.integration.connectors.utils.docker import container_context
 from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.chunking import elements_from_base64_gzipped_json
@@ -51,7 +51,7 @@ def _neo4j_server():
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(DESTINATION_TAG, CONNECTOR_TYPE)
+@pytest.mark.tags(DESTINATION_TAG, CONNECTOR_TYPE, GRAPH_DB_TAG)
 async def test_neo4j_destination(upload_file: Path, tmp_path: Path):
     stager = Neo4jUploadStager()
     uploader = Neo4jUploader(
@@ -104,7 +104,7 @@ async def test_neo4j_destination(upload_file: Path, tmp_path: Path):
     await validate_uploaded_graph(modified_upload_file)
 
 
-@pytest.mark.tags(DESTINATION_TAG, CONNECTOR_TYPE)
+@pytest.mark.tags(DESTINATION_TAG, CONNECTOR_TYPE, GRAPH_DB_TAG)
 class TestPrecheck:
     @pytest.fixture
     def configured_uploader(self) -> Neo4jUploader:
@@ -199,13 +199,15 @@ async def validate_uploaded_graph(upload_file: Path):
     try:
         nodes_count = len((await driver.execute_query("MATCH (n) RETURN n"))[0])
         chunk_nodes_count = len(
-            (await driver.execute_query(f"MATCH (n: {Label.CHUNK}) RETURN n"))[0]
+            (await driver.execute_query(f"MATCH (n: {Label.CHUNK.value}) RETURN n"))[0]
         )
         document_nodes_count = len(
-            (await driver.execute_query(f"MATCH (n: {Label.DOCUMENT}) RETURN n"))[0]
+            (await driver.execute_query(f"MATCH (n: {Label.DOCUMENT.value}) RETURN n"))[0]
         )
         element_nodes_count = len(
-            (await driver.execute_query(f"MATCH (n: {Label.UNSTRUCTURED_ELEMENT}) RETURN n"))[0]
+            (await driver.execute_query(f"MATCH (n: {Label.UNSTRUCTURED_ELEMENT.value}) RETURN n"))[
+                0
+            ]
         )
         with check:
             assert nodes_count == expected_nodes_count
@@ -217,12 +219,18 @@ async def validate_uploaded_graph(upload_file: Path):
             assert element_nodes_count == expected_element_count
 
         records, _, _ = await driver.execute_query(
-            f"MATCH ()-[r:{Relationship.PART_OF_DOCUMENT}]->(:{Label.DOCUMENT}) RETURN r"
+            f"""
+            MATCH ()-[r:{Relationship.PART_OF_DOCUMENT.value}]->(:{Label.DOCUMENT.value})
+            RETURN r
+            """
         )
         part_of_document_count = len(records)
 
         records, _, _ = await driver.execute_query(
-            f"MATCH (:{Label.CHUNK})-[r:{Relationship.NEXT_CHUNK}]->(:{Label.CHUNK}) RETURN r"
+            f"""
+            MATCH (:{Label.CHUNK.value})-[r:{Relationship.NEXT_CHUNK.value}]->(:{Label.CHUNK.value})
+            RETURN r
+            """
         )
         next_chunk_count = len(records)
 

test/integration/connectors/test_notion.py

@@ -1,5 +1,8 @@
 import os
 
+import pytest
+
+from test.integration.connectors.utils.constants import SOURCE_TAG, UNCATEGORIZED_TAG
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     get_all_file_data,
@@ -8,6 +11,7 @@ from test.integration.connectors.utils.validation.source import (
 )
 from unstructured_ingest.v2.interfaces import Downloader, Indexer
 from unstructured_ingest.v2.processes.connectors.notion.connector import (
+    CONNECTOR_TYPE,
     NotionAccessConfig,
     NotionConnectionConfig,
     NotionDownloader,
@@ -17,6 +21,7 @@ from unstructured_ingest.v2.processes.connectors.notion.connector import (
 )
 
 
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
 def test_notion_source_database(temp_dir):
     # Retrieve environment variables
     notion_api_key = os.environ["NOTION_API_KEY"]
@@ -55,6 +60,7 @@ def test_notion_source_database(temp_dir):
     )
 
 
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
 def test_notion_source_page(temp_dir):
     # Retrieve environment variables
     notion_api_key = os.environ["NOTION_API_KEY"]
@@ -93,6 +99,7 @@ def test_notion_source_page(temp_dir):
     )
 
 
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
 def source_connector_validation(
     indexer: Indexer,
     downloader: Downloader,

test/integration/connectors/test_onedrive.py

@@ -5,9 +5,7 @@ from pathlib import Path
 import pytest
 from office365.graph_client import GraphClient
 
-from test.integration.connectors.utils.constants import (
-    DESTINATION_TAG,
-)
+from test.integration.connectors.utils.constants import BLOB_STORAGE_TAG, DESTINATION_TAG
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.onedrive import (
@@ -67,7 +65,7 @@ def get_connection_config():
     return connection_config
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, BLOB_STORAGE_TAG)
 @requires_env("MS_CLIENT_CRED", "MS_CLIENT_ID", "MS_TENANT_ID", "MS_USER_PNAME")
 @pytest.mark.xfail(
     reason="Issues with test setup on the provider side."

test/integration/connectors/test_pinecone.py

@@ -12,9 +12,7 @@ from _pytest.fixtures import TopRequest
 from pinecone import Pinecone, ServerlessSpec
 from pinecone.core.openapi.shared.exceptions import NotFoundException
 
-from test.integration.connectors.utils.constants import (
-    DESTINATION_TAG,
-)
+from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
     stager_validation,
@@ -133,7 +131,7 @@ def validate_pinecone_index(
 
 @requires_env(API_KEY)
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 async def test_pinecone_destination(pinecone_index: str, upload_file: Path, temp_dir: Path):
     file_data = FileData(
         source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
@@ -176,7 +174,7 @@ async def test_pinecone_destination(pinecone_index: str, upload_file: Path, temp
 
 @requires_env(API_KEY)
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @pytest.mark.skip(reason="TODO: get this to work")
 async def test_pinecone_destination_large_index(
     pinecone_index: str, upload_file: Path, temp_dir: Path
@@ -227,7 +225,7 @@ async def test_pinecone_destination_large_index(
 
 
 @requires_env(API_KEY)
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 def test_large_metadata(pinecone_index: str, tmp_path: Path, upload_file: Path):
     stager = PineconeUploadStager()
     uploader = PineconeUploader(
@@ -272,6 +270,7 @@ def test_large_metadata(pinecone_index: str, tmp_path: Path, upload_file: Path):
     validate_pinecone_index(pinecone_index, 1, interval=5)
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_pinecone_stager(
     request: TopRequest,

test/integration/connectors/test_qdrant.py

@@ -9,7 +9,7 @@ import pytest
 from _pytest.fixtures import TopRequest
 from qdrant_client import AsyncQdrantClient
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
 from test.integration.connectors.utils.docker import container_context
 from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
@@ -75,7 +75,7 @@ async def validate_upload(client: AsyncQdrantClient, upload_file: Path):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(LOCAL_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant")
+@pytest.mark.tags(LOCAL_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant", VECTOR_DB_TAG)
 async def test_qdrant_destination_local(upload_file: Path, tmp_path: Path):
     connection_kwargs = {"path": str(tmp_path / "qdrant")}
     async with qdrant_client(connection_kwargs) as client:
@@ -117,7 +117,7 @@ def docker_context():
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(SERVER_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant")
+@pytest.mark.tags(SERVER_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant", VECTOR_DB_TAG)
 async def test_qdrant_destination_server(upload_file: Path, tmp_path: Path, docker_context):
     connection_kwargs = {"location": "http://localhost:6333"}
     async with qdrant_client(connection_kwargs) as client:
@@ -153,7 +153,7 @@ async def test_qdrant_destination_server(upload_file: Path, tmp_path: Path, dock
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(SERVER_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant")
+@pytest.mark.tags(SERVER_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant", VECTOR_DB_TAG)
 @requires_env("QDRANT_API_KEY", "QDRANT_SERVER_URL")
 async def test_qdrant_destination_cloud(upload_file: Path, tmp_path: Path):
     server_url = os.environ["QDRANT_SERVER_URL"]
@@ -197,6 +197,7 @@ async def test_qdrant_destination_cloud(upload_file: Path, tmp_path: Path):
         await validate_upload(client=client, upload_file=upload_file)
 
 
+@pytest.mark.tags(SERVER_CONNECTOR_TYPE, DESTINATION_TAG, "qdrant", VECTOR_DB_TAG)
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_qdrant_stager(
     request: TopRequest,

test/integration/connectors/test_redis.py

@@ -9,7 +9,7 @@ import pytest
 from redis import exceptions as redis_exceptions
 from redis.asyncio import Redis, from_url
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, NOSQL_TAG
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.redisdb import (
@@ -96,7 +96,7 @@ async def redis_destination_test(
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(REDIS_CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(REDIS_CONNECTOR_TYPE, DESTINATION_TAG, NOSQL_TAG)
 @requires_env("AZURE_REDIS_INGEST_TEST_PASSWORD")
 async def test_redis_destination_azure_with_password(upload_file: Path, tmp_path: Path):
     connection_kwargs = {
@@ -110,7 +110,7 @@ async def test_redis_destination_azure_with_password(upload_file: Path, tmp_path
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(REDIS_CONNECTOR_TYPE, DESTINATION_TAG, "redis")
+@pytest.mark.tags(REDIS_CONNECTOR_TYPE, DESTINATION_TAG, "redis", NOSQL_TAG)
 @requires_env("AZURE_REDIS_INGEST_TEST_PASSWORD")
 async def test_redis_destination_azure_with_uri(upload_file: Path, tmp_path: Path):
     connection_kwargs = {}

test/integration/connectors/test_s3.py

@@ -6,6 +6,7 @@ from pathlib import Path
 import pytest
 
 from test.integration.connectors.utils.constants import (
+    BLOB_STORAGE_TAG,
     DESTINATION_TAG,
     SOURCE_TAG,
     env_setup_path,
@@ -47,7 +48,7 @@ def anon_connection_config() -> S3ConnectionConfig:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
 async def test_s3_source(anon_connection_config: S3ConnectionConfig):
     indexer_config = S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/small-pdf-set/")
     with tempfile.TemporaryDirectory() as tempdir:
@@ -70,7 +71,7 @@ async def test_s3_source(anon_connection_config: S3ConnectionConfig):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
 async def test_s3_source_special_char(anon_connection_config: S3ConnectionConfig):
     indexer_config = S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/special-characters/")
     with tempfile.TemporaryDirectory() as tempdir:
@@ -92,7 +93,7 @@ async def test_s3_source_special_char(anon_connection_config: S3ConnectionConfig
         )
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
 def test_s3_source_no_access(anon_connection_config: S3ConnectionConfig):
     indexer_config = S3IndexerConfig(remote_url="s3://utic-ingest-test-fixtures/destination/")
     indexer = S3Indexer(connection_config=anon_connection_config, index_config=indexer_config)
@@ -100,7 +101,7 @@ def test_s3_source_no_access(anon_connection_config: S3ConnectionConfig):
         indexer.precheck()
 
 
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
 def test_s3_source_no_bucket(anon_connection_config: S3ConnectionConfig):
     indexer_config = S3IndexerConfig(remote_url="s3://fake-bucket")
     indexer = S3Indexer(connection_config=anon_connection_config, index_config=indexer_config)
@@ -109,7 +110,7 @@ def test_s3_source_no_bucket(anon_connection_config: S3ConnectionConfig):
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "minio")
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, "minio", BLOB_STORAGE_TAG)
 async def test_s3_minio_source(anon_connection_config: S3ConnectionConfig):
     anon_connection_config.endpoint_url = "http://localhost:9000"
     indexer_config = S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/")
@@ -149,7 +150,7 @@ def get_aws_credentials() -> dict:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, BLOB_STORAGE_TAG)
 @requires_env("S3_INGEST_TEST_ACCESS_KEY", "S3_INGEST_TEST_SECRET_KEY")
 async def test_s3_destination(upload_file: Path):
     aws_credentials = get_aws_credentials()

test/integration/connectors/test_vectara.py

@@ -8,7 +8,7 @@ from uuid import uuid4
 import pytest
 import requests
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, NOSQL_TAG
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.logger import logger
@@ -211,7 +211,7 @@ def corpora_util() -> Generator[str, None, None]:
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(VECTARA_CONNECTOR_TYPE, DESTINATION_TAG, "vectara")
+@pytest.mark.tags(VECTARA_CONNECTOR_TYPE, DESTINATION_TAG, "vectara", NOSQL_TAG)
 @requires_env("VECTARA_OAUTH_CLIENT_ID", "VECTARA_OAUTH_SECRET", "VECTARA_CUSTOMER_ID")
 async def test_vectara_destination(
     upload_file: Path, tmp_path: Path, corpora_util: str, retries=30, interval=10

test/integration/connectors/utils/constants.py

@@ -2,6 +2,12 @@ from pathlib import Path
 
 SOURCE_TAG = "source"
 DESTINATION_TAG = "destination"
+BLOB_STORAGE_TAG = "blob_storage"
+SQL_TAG = "sql"
+NOSQL_TAG = "nosql"
+VECTOR_DB_TAG = "vector_db"
+GRAPH_DB_TAG = "graph_db"
+UNCATEGORIZED_TAG = "uncategorized"
 
 env_setup_path = Path(__file__).parents[1] / "env_setup"
 expected_results_path = Path(__file__).parents[1] / "expected_results"

test/integration/connectors/utils/docker.py

@@ -44,7 +44,7 @@ def get_container(
     docker_client: docker.DockerClient,
     image: str,
     ports: dict,
-    name: Optional[str] = "connector_test",
+    name: Optional[str] = None,
     environment: Optional[dict] = None,
     volumes: Optional[dict] = None,
     healthcheck: Optional[HealthCheck] = None,
@@ -115,7 +115,7 @@ def container_context(
     healthcheck: Optional[HealthCheck] = None,
     healthcheck_retries: int = 30,
     docker_client: Optional[docker.DockerClient] = None,
-    name: Optional[str] = "connector_test",
+    name: Optional[str] = None,
 ):
     docker_client = docker_client or docker.from_env()
     print(f"pulling image {image}")

test/integration/connectors/weaviate/test_cloud.py

@@ -1,12 +1,15 @@
 import pytest
 from pydantic import ValidationError
 
+from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
 from unstructured_ingest.v2.processes.connectors.weaviate.cloud import (
+    CONNECTOR_TYPE,
     CloudWeaviateAccessConfig,
     CloudWeaviateConnectionConfig,
 )
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 def test_weaviate_failing_connection_config():
     with pytest.raises(ValidationError):
         CloudWeaviateConnectionConfig(
@@ -16,6 +19,7 @@ def test_weaviate_failing_connection_config():
         )
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 def test_weaviate_connection_config_happy_path():
     CloudWeaviateConnectionConfig(
         access_config=CloudWeaviateAccessConfig(
@@ -25,6 +29,7 @@ def test_weaviate_connection_config_happy_path():
     )
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 def test_weaviate_connection_config_anonymous():
     CloudWeaviateConnectionConfig(
         access_config=CloudWeaviateAccessConfig(api_key="my key", password="password"),

test/integration/connectors/weaviate/test_local.py

@@ -7,7 +7,7 @@ import requests
 import weaviate
 from weaviate.client import WeaviateClient
 
-from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
 from test.integration.connectors.utils.docker import container_context
 from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.weaviate.local import (
@@ -74,7 +74,7 @@ def run_uploader_and_validate(
 
 
 @pytest.mark.asyncio
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
 def test_weaviate_local_destination(upload_file: Path, collection: str, tmp_path: Path):
     file_data = FileData(
         source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.3.13"  # pragma: no cover
+__version__ = "0.3.14"  # pragma: no cover

unstructured_ingest/v2/processes/connectors/neo4j.py

@@ -105,7 +105,7 @@ class Neo4jUploadStager(UploadStager):
         output_filepath.parent.mkdir(parents=True, exist_ok=True)
 
         with open(output_filepath, "w") as file:
-            json.dump(_GraphData.from_nx(nx_graph).model_dump(), file, indent=4)
+            file.write(_GraphData.from_nx(nx_graph).model_dump_json())
 
         return output_filepath
 
@@ -196,7 +196,7 @@ class _GraphData(BaseModel):
 
 
 class _Node(BaseModel):
-    model_config = ConfigDict(use_enum_values=True)
+    model_config = ConfigDict()
 
     id_: str = Field(default_factory=lambda: str(uuid.uuid4()))
     labels: list[Label] = Field(default_factory=list)
@@ -207,20 +207,20 @@ class _Node(BaseModel):
 
 
 class _Edge(BaseModel):
-    model_config = ConfigDict(use_enum_values=True)
+    model_config = ConfigDict()
 
     source_id: str
     destination_id: str
     relationship: Relationship
 
 
-class Label(str, Enum):
+class Label(Enum):
     UNSTRUCTURED_ELEMENT = "UnstructuredElement"
     CHUNK = "Chunk"
     DOCUMENT = "Document"
 
 
-class Relationship(str, Enum):
+class Relationship(Enum):
     PART_OF_DOCUMENT = "PART_OF_DOCUMENT"
     PART_OF_CHUNK = "PART_OF_CHUNK"
     NEXT_CHUNK = "NEXT_CHUNK"
@@ -263,14 +263,14 @@ class Neo4jUploader(Uploader):
     async def _create_uniqueness_constraints(self, client: AsyncDriver) -> None:
         for label in Label:
             logger.info(
-                f"Adding id uniqueness constraint for nodes labeled '{label}'"
+                f"Adding id uniqueness constraint for nodes labeled '{label.value}'"
                 " if it does not already exist."
             )
-            constraint_name = f"{label.lower()}_id"
+            constraint_name = f"{label.value.lower()}_id"
             await client.execute_query(
                 f"""
                 CREATE CONSTRAINT {constraint_name} IF NOT EXISTS
-                FOR (n: {label}) REQUIRE n.id IS UNIQUE
+                FOR (n: {label.value}) REQUIRE n.id IS UNIQUE
                 """
             )
 
@@ -278,8 +278,8 @@ class Neo4jUploader(Uploader):
         logger.info(f"Deleting old data for the record '{file_data.identifier}' (if present).")
         _, summary, _ = await client.execute_query(
             f"""
-            MATCH (n: {Label.DOCUMENT} {{id: $identifier}})
-            MATCH (n)--(m: {Label.CHUNK}|{Label.UNSTRUCTURED_ELEMENT})
+            MATCH (n: {Label.DOCUMENT.value} {{id: $identifier}})
+            MATCH (n)--(m: {Label.CHUNK.value}|{Label.UNSTRUCTURED_ELEMENT.value})
             DETACH DELETE m""",
             identifier=file_data.identifier,
         )
@@ -349,7 +349,7 @@ class Neo4jUploader(Uploader):
 
     @staticmethod
     def _create_nodes_query(nodes: list[_Node], labels: tuple[Label, ...]) -> tuple[str, dict]:
-        labels_string = ", ".join(labels)
+        labels_string = ", ".join([label.value for label in labels])
         logger.info(f"Preparing MERGE query for {len(nodes)} nodes labeled '{labels_string}'.")
         query_string = f"""
             UNWIND $nodes AS node
@@ -366,7 +366,7 @@ class Neo4jUploader(Uploader):
             UNWIND $edges AS edge
             MATCH (u {{id: edge.source}})
             MATCH (v {{id: edge.destination}})
-            MERGE (u)-[:{relationship}]->(v)
+            MERGE (u)-[:{relationship.value}]->(v)
             """
         parameters = {
             "edges": [