unstructured-ingest 0.5.23__py3-none-any.whl → 0.6.0__py3-none-any.whl

test/integration/connectors/databricks/test_volumes_native.py

@@ -21,7 +21,6 @@ from test.integration.connectors.utils.validation.source import (
 )
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.errors import UserAuthError, UserError
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.databricks.volumes_native import (
     CONNECTOR_TYPE,
     DatabricksNativeVolumesAccessConfig,
@@ -33,6 +32,7 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes_native impor
     DatabricksNativeVolumesUploader,
     DatabricksNativeVolumesUploaderConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @dataclass

test/integration/connectors/duckdb/test_duckdb.py

@@ -10,7 +10,6 @@ from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
     stager_validation,
 )
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.duckdb.duckdb import (
     CONNECTOR_TYPE,
     DuckDBConnectionConfig,
@@ -18,6 +17,7 @@ from unstructured_ingest.v2.processes.connectors.duckdb.duckdb import (
     DuckDBUploaderConfig,
     DuckDBUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture

test/integration/connectors/duckdb/test_motherduck.py

@@ -9,7 +9,6 @@ import pytest
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.duckdb.motherduck import (
     CONNECTOR_TYPE,
     MotherDuckAccessConfig,
@@ -18,6 +17,7 @@ from unstructured_ingest.v2.processes.connectors.duckdb.motherduck import (
     MotherDuckUploaderConfig,
     MotherDuckUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture

test/integration/connectors/elasticsearch/test_elasticsearch.py

@@ -22,7 +22,7 @@ from test.integration.connectors.utils.validation.source import (
     source_connector_validation,
 )
 from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.elasticsearch.elasticsearch import (
     CONNECTOR_TYPE,
     ElasticsearchAccessConfig,

test/integration/connectors/elasticsearch/test_opensearch.py

@@ -24,7 +24,6 @@ from unstructured_ingest.error import (
     DestinationConnectionError,
     SourceConnectionError,
 )
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.elasticsearch.opensearch import (
     CONNECTOR_TYPE,
     OpenSearchAccessConfig,
@@ -38,6 +37,7 @@ from unstructured_ingest.v2.processes.connectors.elasticsearch.opensearch import
     OpenSearchUploadStager,
     OpenSearchUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 SOURCE_INDEX_NAME = "movies"
 DESTINATION_INDEX_NAME = "elements"

test/integration/connectors/sql/test_databricks_delta_tables.py

@@ -14,7 +14,6 @@ from pytest_mock import MockerFixture
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG, env_setup_path
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connectors.sql.databricks_delta_tables import (
     CONNECTOR_TYPE,
@@ -24,6 +23,7 @@ from unstructured_ingest.v2.processes.connectors.sql.databricks_delta_tables imp
     DatabricksDeltaTablesUploaderConfig,
     DatabricksDeltaTablesUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 CATALOG = "utic-dev-tech-fixtures"
 

test/integration/connectors/sql/test_postgres.py

@@ -20,7 +20,6 @@ from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
 )
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.sql.postgres import (
     CONNECTOR_TYPE,
     PostgresAccessConfig,
@@ -32,6 +31,7 @@ from unstructured_ingest.v2.processes.connectors.sql.postgres import (
     PostgresUploader,
     PostgresUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 SEED_DATA_ROWS = 10
 

test/integration/connectors/sql/test_singlestore.py

@@ -20,7 +20,6 @@ from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
 )
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.sql.singlestore import (
     CONNECTOR_TYPE,
     SingleStoreAccessConfig,
@@ -33,6 +32,7 @@ from unstructured_ingest.v2.processes.connectors.sql.singlestore import (
     SingleStoreUploaderConfig,
     SingleStoreUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 SEED_DATA_ROWS = 10
 

test/integration/connectors/sql/test_snowflake.py

@@ -22,7 +22,6 @@ from test.integration.connectors.utils.validation.source import (
     source_connector_validation,
 )
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.sql.snowflake import (
     CONNECTOR_TYPE,
     SnowflakeAccessConfig,
@@ -34,6 +33,7 @@ from unstructured_ingest.v2.processes.connectors.sql.snowflake import (
     SnowflakeUploader,
     SnowflakeUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 SEED_DATA_ROWS = 20
 

test/integration/connectors/sql/test_sqlite.py

@@ -20,7 +20,6 @@ from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
 )
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.sql.sqlite import (
     CONNECTOR_TYPE,
     SQLiteConnectionConfig,
@@ -31,6 +30,7 @@ from unstructured_ingest.v2.processes.connectors.sql.sqlite import (
     SQLiteUploader,
     SQLiteUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 SEED_DATA_ROWS = 10
 

test/integration/connectors/test_astradb.py

@@ -20,7 +20,6 @@ from test.integration.connectors.utils.validation.source import (
     source_connector_validation,
 )
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.astradb import (
     CONNECTOR_TYPE,
     AstraDBAccessConfig,
@@ -36,6 +35,7 @@ from unstructured_ingest.v2.processes.connectors.astradb import (
     DestinationConnectionError,
     SourceConnectionError,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 EXISTENT_COLLECTION_NAME = "ingest_test_src"
 NONEXISTENT_COLLECTION_NAME = "nonexistant"

test/integration/connectors/test_azure_ai_search.py

@@ -29,7 +29,6 @@ from test.integration.connectors.utils.validation.destination import (
     stager_validation,
 )
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.azure_ai_search import (
     CONNECTOR_TYPE,
     RECORD_ID_LABEL,
@@ -40,6 +39,7 @@ from unstructured_ingest.v2.processes.connectors.azure_ai_search import (
     AzureAISearchUploadStager,
     AzureAISearchUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 repo_path = Path(__file__).parent.resolve()
 

test/integration/connectors/test_chroma.py

@@ -27,7 +27,7 @@ from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
     stager_validation,
 )
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.chroma import (
     CONNECTOR_TYPE,
     ChromaConnectionConfig,

test/integration/connectors/test_delta_table.py

@@ -8,7 +8,6 @@ from fsspec import get_filesystem_class
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.delta_table import (
     CONNECTOR_TYPE,
     DeltaTableAccessConfig,
@@ -18,6 +17,7 @@ from unstructured_ingest.v2.processes.connectors.delta_table import (
     DeltaTableUploadStager,
     DeltaTableUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 multiprocessing.set_start_method("spawn")
 

test/integration/connectors/test_lancedb.py

@@ -13,7 +13,6 @@ from upath import UPath
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
 from unstructured_ingest.v2.constants import RECORD_ID_LABEL
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.lancedb.aws import (
     LanceDBAwsAccessConfig,
     LanceDBAwsConnectionConfig,
@@ -39,6 +38,7 @@ from unstructured_ingest.v2.processes.connectors.lancedb.local import (
     LanceDBLocalConnectionConfig,
     LanceDBLocalUploader,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 DATABASE_NAME = "database"
 TABLE_NAME = "elements"

test/integration/connectors/test_milvus.py

@@ -25,7 +25,6 @@ from test.integration.connectors.utils.validation.destination import (
     stager_validation,
 )
 from unstructured_ingest.error import DestinationConnectionError
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.milvus import (
     CONNECTOR_TYPE,
     MilvusConnectionConfig,
@@ -33,6 +32,7 @@ from unstructured_ingest.v2.processes.connectors.milvus import (
     MilvusUploaderConfig,
     MilvusUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 DB_NAME = "test_database"
 EXISTENT_COLLECTION_NAME = "test_collection"

test/integration/connectors/test_mongodb.py

@@ -20,7 +20,6 @@ from test.integration.connectors.utils.validation.source import (
 )
 from test.integration.utils import requires_env
 from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.mongodb import (
     CONNECTOR_TYPE,
     MongoDBAccessConfig,
@@ -32,6 +31,7 @@ from unstructured_ingest.v2.processes.connectors.mongodb import (
     MongoDBUploader,
     MongoDBUploaderConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 SOURCE_COLLECTION = "sample-mongodb-data"
 

test/integration/connectors/test_neo4j.py

@@ -13,11 +13,6 @@ from test.integration.connectors.utils.constants import DESTINATION_TAG, GRAPH_D
 from test.integration.connectors.utils.docker import container_context
 from unstructured_ingest.error import DestinationConnectionError
 from unstructured_ingest.utils.chunking import elements_from_base64_gzipped_json
-from unstructured_ingest.v2.interfaces.file_data import (
-    FileData,
-    FileDataSourceMetadata,
-    SourceIdentifiers,
-)
 from unstructured_ingest.v2.processes.connectors.neo4j import (
     CONNECTOR_TYPE,
     Label,
@@ -28,6 +23,11 @@ from unstructured_ingest.v2.processes.connectors.neo4j import (
     Neo4jUploadStager,
     Relationship,
 )
+from unstructured_ingest.v2.types.file_data import (
+    FileData,
+    FileDataSourceMetadata,
+    SourceIdentifiers,
+)
 
 USERNAME = "neo4j"
 PASSWORD = "password"

test/integration/connectors/test_onedrive.py

@@ -15,7 +15,6 @@ from test.integration.connectors.utils.validation.source import (
     source_connector_validation,
 )
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.onedrive import (
     CONNECTOR_TYPE,
     OnedriveAccessConfig,
@@ -27,6 +26,7 @@ from unstructured_ingest.v2.processes.connectors.onedrive import (
     OnedriveUploader,
     OnedriveUploaderConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture

test/integration/connectors/test_pinecone.py

@@ -19,7 +19,6 @@ from test.integration.connectors.utils.validation.destination import (
 )
 from test.integration.utils import requires_env
 from unstructured_ingest.error import DestinationConnectionError
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connectors.pinecone import (
     CONNECTOR_TYPE,
@@ -31,6 +30,7 @@ from unstructured_ingest.v2.processes.connectors.pinecone import (
     PineconeUploadStager,
     PineconeUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 METADATA_BYTES_LIMIT = (
     40960  # 40KB https://docs.pinecone.io/reference/quotas-and-limits#hard-limits

test/integration/connectors/test_qdrant.py

@@ -16,7 +16,6 @@ from test.integration.connectors.utils.validation.destination import (
     stager_validation,
 )
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.qdrant.cloud import (
     CloudQdrantAccessConfig,
     CloudQdrantConnectionConfig,
@@ -45,6 +44,7 @@ from unstructured_ingest.v2.processes.connectors.qdrant.server import (
     ServerQdrantUploadStager,
     ServerQdrantUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 COLLECTION_NAME = f"test-coll-{uuid.uuid4().hex[:12]}"
 VECTORS_CONFIG = {"size": 384, "distance": "Cosine"}

test/integration/connectors/test_redis.py

@@ -11,7 +11,6 @@ from redis.asyncio import Redis, from_url
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, NOSQL_TAG
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.redisdb import (
     CONNECTOR_TYPE as REDIS_CONNECTOR_TYPE,
 )
@@ -21,6 +20,7 @@ from unstructured_ingest.v2.processes.connectors.redisdb import (
     RedisUploader,
     RedisUploaderConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 async def delete_record(client: Redis, element_id: str, key_prefix: str) -> None:

test/integration/connectors/test_s3.py

@@ -18,7 +18,6 @@ from test.integration.connectors.utils.validation.source import (
 )
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.errors import UserAuthError, UserError
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.fsspec.s3 import (
     CONNECTOR_TYPE,
     S3AccessConfig,
@@ -30,6 +29,7 @@ from unstructured_ingest.v2.processes.connectors.fsspec.s3 import (
     S3Uploader,
     S3UploaderConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 def validate_predownload_file_data(file_data: FileData):

test/integration/connectors/test_vectara.py

@@ -1,6 +1,7 @@
 import json
 import os
 import time
+from functools import lru_cache
 from pathlib import Path
 from typing import Generator
 from uuid import uuid4
@@ -10,7 +11,6 @@ import requests
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, NOSQL_TAG
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.logger import logger
 from unstructured_ingest.v2.processes.connectors.vectara import (
     CONNECTOR_TYPE as VECTARA_CONNECTOR_TYPE,
@@ -23,26 +23,32 @@ from unstructured_ingest.v2.processes.connectors.vectara import (
     VectaraUploadStager,
     VectaraUploadStagerConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
-def validate_upload(response: dict, expected_data: dict):
+def validate_upload(document: dict, expected_data: dict):
+    logger.info(f"validating document: {document}")
     element_id = expected_data["element_id"]
     expected_text = expected_data["text"]
     filename = expected_data["metadata"]["filename"]
     filetype = expected_data["metadata"]["filetype"]
     page_number = expected_data["metadata"]["page_number"]
 
-    response = response["search_results"][0]
-
-    assert response is not None
-    assert response["text"] == expected_text
-    assert response["part_metadata"]["element_id"] == element_id
-    assert response["part_metadata"]["filename"] == filename
-    assert response["part_metadata"]["filetype"] == filetype
-    assert response["part_metadata"]["page_number"] == page_number
+    assert document is not None
+    speech_parts = document["parts"]
+    assert speech_parts
+    first_part = speech_parts[0]
+    assert first_part["text"] == expected_text
+    part_metadata = first_part["metadata"]
+    assert part_metadata
+    assert part_metadata["element_id"] == element_id
+    assert part_metadata["filename"] == filename
+    assert part_metadata["filetype"] == filetype
+    assert part_metadata["page_number"] == page_number
 
 
 @requires_env("VECTARA_OAUTH_CLIENT_ID", "VECTARA_OAUTH_SECRET", "VECTARA_CUSTOMER_ID")
+@lru_cache()
 def _get_jwt_token():
     """Connect to the server and get a JWT token."""
     customer_id = os.environ["VECTARA_CUSTOMER_ID"]
@@ -65,23 +71,12 @@ def _get_jwt_token():
     return response_json.get("access_token")
 
 
-def query_data(corpus_key: str, element_id: str) -> dict:
+def list_documents(corpus_key: str) -> list[str]:
 
-    url = f"https://api.vectara.io/v2/corpora/{corpus_key}/query"
+    url = f"https://api.vectara.io/v2/corpora/{corpus_key}/documents"
 
     # the query below requires the corpus to have filter attributes for element_id
 
-    data = json.dumps(
-        {
-            "query": "string",
-            "search": {
-                "metadata_filter": f"part.element_id = '{element_id}'",
-                "lexical_interpolation": 1,
-                "limit": 10,
-            },
-        }
-    )
-
     jwt_token = _get_jwt_token()
     headers = {
         "Content-Type": "application/json",
@@ -90,11 +85,26 @@ def query_data(corpus_key: str, element_id: str) -> dict:
         "X-source": "unstructured",
     }
 
-    response = requests.post(url, headers=headers, data=data)
+    response = requests.get(url, headers=headers)
     response.raise_for_status()
     response_json = response.json()
+    documents = response_json.get("documents", [])
+    return documents
+
 
-    return response_json
+def fetch_document(corpus_key: str, documents_id: str) -> dict:
+    url = f"https://api.vectara.io/v2/corpora/{corpus_key}/documents/{documents_id}"
+    jwt_token = _get_jwt_token()
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        "Authorization": f"Bearer {jwt_token}",
+        "X-source": "unstructured",
+    }
+
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+    return response.json()
 
 
 def create_corpora(corpus_key: str, corpus_name: str) -> None:
@@ -148,8 +158,8 @@ def delete_corpora(corpus_key: str) -> None:
     response.raise_for_status()
 
 
-def list_corpora() -> list:
-    url = "https://api.vectara.io/v2/corpora?limit=100"
+def get_metadata(corpus_key: str):
+    url = f"https://api.vectara.io/v2/corpora/{corpus_key}"
     jwt_token = _get_jwt_token()
     headers = {
         "Content-Type": "application/json",
@@ -159,35 +169,28 @@ def list_corpora() -> list:
     }
     response = requests.get(url, headers=headers)
     response.raise_for_status()
-    response_json = response.json()
-    if response_json.get("corpora"):
-        return [item["key"] for item in response_json.get("corpora")]
-    else:
-        return []
+    return response.json()
 
 
 def wait_for_ready(corpus_key: str, timeout=60, interval=2) -> None:
-    def is_ready_status():
-        corpora_list = list_corpora()
-        return corpus_key in corpora_list
-
     start = time.time()
-    is_ready = is_ready_status()
-    while not is_ready and time.time() - start < timeout:
-        time.sleep(interval)
-        is_ready = is_ready_status()
-    if not is_ready:
-        raise TimeoutError("time out waiting for corpus to be ready")
+    while time.time() - start < timeout:
+        try:
+            get_metadata(corpus_key)
+            return
+        except requests.HTTPError:
+            time.sleep(interval)
+    raise TimeoutError("time out waiting for corpus to be ready")
 
 
 def wait_for_delete(corpus_key: str, timeout=60, interval=2) -> None:
     start = time.time()
     while time.time() - start < timeout:
-        corpora_list = list_corpora()
-        if corpus_key not in corpora_list:
+        try:
+            get_metadata(corpus_key)
+            time.sleep(interval)
+        except requests.HTTPError:
             return
-        time.sleep(interval)
-
     raise TimeoutError("time out waiting for corpus to delete")
 
 
@@ -210,11 +213,23 @@ def corpora_util() -> Generator[str, None, None]:
         wait_for_delete(corpus_key=corpus_key)
 
 
+def wait_for_doc_meta(corpus_key: str, timeout=60, interval=1) -> list[str]:
+    start = time.time()
+    while time.time() - start < timeout:
+        all_document_meta = list_documents(corpus_key)
+        if not all_document_meta:
+            time.sleep(interval)
+            continue
+        else:
+            return all_document_meta
+    raise TimeoutError("time out waiting for document to be ready")
+
+
 @pytest.mark.asyncio
 @pytest.mark.tags(VECTARA_CONNECTOR_TYPE, DESTINATION_TAG, "vectara", NOSQL_TAG)
 @requires_env("VECTARA_OAUTH_CLIENT_ID", "VECTARA_OAUTH_SECRET", "VECTARA_CUSTOMER_ID")
 async def test_vectara_destination(
-    upload_file: Path, tmp_path: Path, corpora_util: str, retries=30, interval=10
+    upload_file: Path, tmp_path: Path, corpora_util: str, retries=30, interval=1
 ):
     corpus_key = corpora_util
     connection_kwargs = {
@@ -231,7 +246,7 @@ async def test_vectara_destination(
         identifier="mock-file-data",
     )
 
-    stager_config = VectaraUploadStagerConfig(batch_size=10)
+    stager_config = VectaraUploadStagerConfig()
     stager = VectaraUploadStager(upload_stager_config=stager_config)
     new_upload_file = stager.run(
         elements_filepath=upload_file,
@@ -260,11 +275,8 @@ async def test_vectara_destination(
         elements = json.load(upload_fp)
     first_element = elements[0]
 
-    for i in range(retries):
-        response = query_data(corpus_key, first_element["element_id"])
-        if not response["search_results"]:
-            time.sleep(interval)
-        else:
-            break
-
-    validate_upload(response=response, expected_data=first_element)
+    all_document_meta = wait_for_doc_meta(corpus_key)
+    assert len(all_document_meta) == 1
+    document_meta = all_document_meta[0]
+    document = fetch_document(corpus_key=corpus_key, documents_id=document_meta["id"])
+    validate_upload(document=document, expected_data=first_element)

test/integration/connectors/utils/validation/destination.py

@@ -4,7 +4,8 @@ from pathlib import Path
 
 from test.integration.connectors.utils.validation.utils import ValidationConfig
 from unstructured_ingest.utils.data_prep import get_data
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers, UploadStager
+from unstructured_ingest.v2.interfaces import UploadStager
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 class StagerValidationConfigs(ValidationConfig):

test/integration/connectors/utils/validation/source.py

@@ -8,7 +8,8 @@ from deepdiff import DeepDiff
 from pydantic import Field
 
 from test.integration.connectors.utils.validation.utils import ValidationConfig
-from unstructured_ingest.v2.interfaces import Downloader, FileData, Indexer
+from unstructured_ingest.v2.interfaces import Downloader, Indexer
+from unstructured_ingest.v2.types.file_data import FileData
 
 NONSTANDARD_METADATA_FIELDS = {
     "additional_metadata.@microsoft.graph.downloadUrl": [

test/integration/connectors/weaviate/test_local.py

@@ -9,7 +9,6 @@ from weaviate.client import WeaviateClient
 
 from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
 from test.integration.connectors.utils.docker import container_context
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.weaviate.local import (
     CONNECTOR_TYPE,
     LocalWeaviateConnectionConfig,
@@ -17,6 +16,7 @@ from unstructured_ingest.v2.processes.connectors.weaviate.local import (
     LocalWeaviateUploaderConfig,
     LocalWeaviateUploadStager,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 COLLECTION_NAME = "elements"
 

test/unit/test_html.py

@@ -5,7 +5,7 @@ from bs4 import BeautifulSoup
 from pytest_mock import MockerFixture
 
 from unstructured_ingest.utils.html import HtmlMixin
-from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 def test_extract_images(mocker: MockerFixture):

test/unit/v2/connectors/ibm_watsonx/test_ibm_watsonx_s3.py

@@ -8,7 +8,6 @@ from pyiceberg.exceptions import CommitFailedException
 from pytest_mock import MockerFixture
 
 from unstructured_ingest.v2.errors import ProviderError, UserError
-from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.ibm_watsonx import IBM_WATSONX_S3_CONNECTOR_TYPE
 from unstructured_ingest.v2.processes.connectors.ibm_watsonx.ibm_watsonx_s3 import (
     IbmWatsonxAccessConfig,
@@ -16,6 +15,7 @@ from unstructured_ingest.v2.processes.connectors.ibm_watsonx.ibm_watsonx_s3 impo
     IbmWatsonxUploader,
     IbmWatsonxUploaderConfig,
 )
+from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture