unstructured-ingest 0.3.9__tar.gz → 0.3.11__tar.gz

{unstructured_ingest-0.3.9/unstructured_ingest.egg-info → unstructured_ingest-0.3.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.3.9
+Version: 0.3.11
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,14 +22,14 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: tqdm
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: dataclasses_json
+Requires-Dist: ndjson
 Requires-Dist: click
+Requires-Dist: tqdm
 Requires-Dist: python-dateutil
-Requires-Dist: ndjson
 Requires-Dist: pydantic>=2.7
-Requires-Dist: opentelemetry-sdk
 Requires-Dist: pandas
-Requires-Dist: dataclasses_json
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
@@ -67,16 +67,16 @@ Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: adlfs; extra == "azure"
 Requires-Dist: fsspec; extra == "azure"
+Requires-Dist: adlfs; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
-Requires-Dist: requests; extra == "biomed"
 Requires-Dist: bs4; extra == "biomed"
+Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: fsspec; extra == "box"
 Requires-Dist: boxfs; extra == "box"
+Requires-Dist: fsspec; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
@@ -92,8 +92,8 @@ Requires-Dist: deltalake; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord-py; extra == "discord"
 Provides-Extra: dropbox
-Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Requires-Dist: fsspec; extra == "dropbox"
+Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
@@ -110,8 +110,8 @@ Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: hubspot-api-client; extra == "hubspot"
+Requires-Dist: urllib3; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -126,21 +126,22 @@ Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
 Requires-Dist: cymple; extra == "neo4j"
+Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: neo4j; extra == "neo4j"
 Provides-Extra: notion
+Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
-Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
-Requires-Dist: htmlBuilder; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
 Provides-Extra: onedrive
-Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Requires-Dist: msal; extra == "onedrive"
 Requires-Dist: bs4; extra == "onedrive"
+Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
-Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Requires-Dist: msal; extra == "outlook"
+Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
 Provides-Extra: pinecone
 Requires-Dist: pinecone-client>=3.7.1; extra == "pinecone"
 Provides-Extra: postgres
@@ -153,18 +154,18 @@ Provides-Extra: s3
 Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
 Provides-Extra: sharepoint
-Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
+Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: fsspec; extra == "sftp"
+Requires-Dist: paramiko; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
+Requires-Dist: psycopg2-binary; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate
@@ -178,8 +179,8 @@ Requires-Dist: requests; extra == "vectara"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: tiktoken; extra == "embed-octoai"
 Requires-Dist: openai; extra == "embed-octoai"
+Requires-Dist: tiktoken; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -187,8 +188,8 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
+Requires-Dist: tiktoken; extra == "openai"
 Provides-Extra: bedrock
 Requires-Dist: boto3; extra == "bedrock"
 Provides-Extra: togetherai

{unstructured_ingest-0.3.9/test/integration/connectors/databricks_tests → unstructured_ingest-0.3.11/test/integration/connectors/databricks}/test_volumes_native.py

@@ -1,10 +1,10 @@
 import json
 import os
-import tempfile
 import uuid
 from contextlib import contextmanager
 from dataclasses import dataclass
 from pathlib import Path
+from unittest import mock
 
 import pytest
 from databricks.sdk import WorkspaceClient
@@ -31,11 +31,15 @@ from unstructured_ingest.v2.processes.connectors.databricks.volumes_native impor
 
 
 @dataclass
-class EnvData:
+class BaseEnvData:
     host: str
+    catalog: str
+
+
+@dataclass
+class BasicAuthEnvData(BaseEnvData):
     client_id: str
     client_secret: str
-    catalog: str
 
     def get_connection_config(self) -> DatabricksNativeVolumesConnectionConfig:
         return DatabricksNativeVolumesConnectionConfig(
@@ -47,8 +51,21 @@ class EnvData:
         )
 
 
-def get_env_data() -> EnvData:
-    return EnvData(
+@dataclass
+class PATEnvData(BaseEnvData):
+    token: str
+
+    def get_connection_config(self) -> DatabricksNativeVolumesConnectionConfig:
+        return DatabricksNativeVolumesConnectionConfig(
+            host=self.host,
+            access_config=DatabricksNativeVolumesAccessConfig(
+                token=self.token,
+            ),
+        )
+
+
+def get_basic_auth_env_data() -> BasicAuthEnvData:
+    return BasicAuthEnvData(
         host=os.environ["DATABRICKS_HOST"],
         client_id=os.environ["DATABRICKS_CLIENT_ID"],
         client_secret=os.environ["DATABRICKS_CLIENT_SECRET"],
@@ -56,23 +73,30 @@ def get_env_data() -> EnvData:
     )
 
 
+def get_pat_env_data() -> PATEnvData:
+    return PATEnvData(
+        host=os.environ["DATABRICKS_HOST"],
+        catalog=os.environ["DATABRICKS_CATALOG"],
+        token=os.environ["DATABRICKS_PAT"],
+    )
+
+
 @pytest.mark.asyncio
 @pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
 @requires_env(
     "DATABRICKS_HOST", "DATABRICKS_CLIENT_ID", "DATABRICKS_CLIENT_SECRET", "DATABRICKS_CATALOG"
 )
-async def test_volumes_native_source():
-    env_data = get_env_data()
-    indexer_config = DatabricksNativeVolumesIndexerConfig(
-        recursive=True,
-        volume="test-platform",
-        volume_path="databricks-volumes-test-input",
-        catalog=env_data.catalog,
-    )
-    connection_config = env_data.get_connection_config()
-    with tempfile.TemporaryDirectory() as tempdir:
-        tempdir_path = Path(tempdir)
-        download_config = DatabricksNativeVolumesDownloaderConfig(download_dir=tempdir_path)
+async def test_volumes_native_source(tmp_path: Path):
+    env_data = get_basic_auth_env_data()
+    with mock.patch.dict(os.environ, clear=True):
+        indexer_config = DatabricksNativeVolumesIndexerConfig(
+            recursive=True,
+            volume="test-platform",
+            volume_path="databricks-volumes-test-input",
+            catalog=env_data.catalog,
+        )
+        connection_config = env_data.get_connection_config()
+        download_config = DatabricksNativeVolumesDownloaderConfig(download_dir=tmp_path)
         indexer = DatabricksNativeVolumesIndexer(
             connection_config=connection_config, index_config=indexer_config
         )
@@ -89,12 +113,44 @@ async def test_volumes_native_source():
         )
 
 
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@requires_env("DATABRICKS_HOST", "DATABRICKS_PAT", "DATABRICKS_CATALOG")
+async def test_volumes_native_source_pat(tmp_path: Path):
+    env_data = get_pat_env_data()
+    with mock.patch.dict(os.environ, clear=True):
+        indexer_config = DatabricksNativeVolumesIndexerConfig(
+            recursive=True,
+            volume="test-platform",
+            volume_path="databricks-volumes-test-input",
+            catalog=env_data.catalog,
+        )
+        connection_config = env_data.get_connection_config()
+        download_config = DatabricksNativeVolumesDownloaderConfig(download_dir=tmp_path)
+        indexer = DatabricksNativeVolumesIndexer(
+            connection_config=connection_config, index_config=indexer_config
+        )
+        downloader = DatabricksNativeVolumesDownloader(
+            connection_config=connection_config, download_config=download_config
+        )
+        await source_connector_validation(
+            indexer=indexer,
+            downloader=downloader,
+            configs=SourceValidationConfigs(
+                test_id="databricks_volumes_native_pat",
+                expected_num_files=1,
+            ),
+        )
+
+
 def _get_volume_path(catalog: str, volume: str, volume_path: str):
     return f"/Volumes/{catalog}/default/{volume}/{volume_path}"
 
 
 @contextmanager
-def databricks_destination_context(env_data: EnvData, volume: str, volume_path) -> WorkspaceClient:
+def databricks_destination_context(
+    env_data: BasicAuthEnvData, volume: str, volume_path
+) -> WorkspaceClient:
     client = WorkspaceClient(
         host=env_data.host, client_id=env_data.client_id, client_secret=env_data.client_secret
     )
@@ -137,7 +193,7 @@ def validate_upload(client: WorkspaceClient, catalog: str, volume: str, volume_p
     "DATABRICKS_HOST", "DATABRICKS_CLIENT_ID", "DATABRICKS_CLIENT_SECRET", "DATABRICKS_CATALOG"
 )
 async def test_volumes_native_destination(upload_file: Path):
-    env_data = get_env_data()
+    env_data = get_basic_auth_env_data()
     volume_path = f"databricks-volumes-test-output-{uuid.uuid4()}"
     file_data = FileData(
         source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),

{unstructured_ingest-0.3.9 → unstructured_ingest-0.3.11}/test/integration/connectors/sql/test_postgres.py

@@ -15,7 +15,7 @@ from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
 )
-from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.sql.postgres import (
     CONNECTOR_TYPE,
     PostgresAccessConfig,
@@ -28,7 +28,7 @@ from unstructured_ingest.v2.processes.connectors.sql.postgres import (
     PostgresUploadStager,
 )
 
-SEED_DATA_ROWS = 20
+SEED_DATA_ROWS = 10
 
 
 @pytest.fixture
@@ -69,7 +69,7 @@ async def test_postgres_source(temp_dir: Path, source_database_setup: str):
     )
     indexer = PostgresIndexer(
         connection_config=connection_config,
-        index_config=PostgresIndexerConfig(table_name="cars", id_column="car_id", batch_size=5),
+        index_config=PostgresIndexerConfig(table_name="cars", id_column="car_id", batch_size=6),
     )
     downloader = PostgresDownloader(
         connection_config=connection_config,
@@ -81,7 +81,7 @@ async def test_postgres_source(temp_dir: Path, source_database_setup: str):
         configs=SourceValidationConfigs(
             test_id="postgres",
             expected_num_files=SEED_DATA_ROWS,
-            expected_number_indexed_file_data=4,
+            expected_number_indexed_file_data=2,
             validate_downloaded_files=True,
         ),
     )
@@ -119,7 +119,11 @@ def validate_destination(
 async def test_postgres_destination(upload_file: Path, temp_dir: Path):
     # the postgres destination connector doesn't leverage the file data but is required as an input,
     # mocking it with arbitrary values to meet the base requirements:
-    mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
+    mock_file_data = FileData(
+        identifier="mock file data",
+        connector_type=CONNECTOR_TYPE,
+        source_identifiers=SourceIdentifiers(filename=upload_file.name, fullpath=upload_file.name),
+    )
     with docker_compose_context(
         docker_compose_path=env_setup_path / "sql" / "postgres" / "destination"
     ):

{unstructured_ingest-0.3.9 → unstructured_ingest-0.3.11}/test/integration/connectors/sql/test_singlestore.py

@@ -15,7 +15,7 @@ from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
 )
-from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.sql.singlestore import (
     CONNECTOR_TYPE,
     SingleStoreAccessConfig,
@@ -29,7 +29,7 @@ from unstructured_ingest.v2.processes.connectors.sql.singlestore import (
     SingleStoreUploadStager,
 )
 
-SEED_DATA_ROWS = 20
+SEED_DATA_ROWS = 10
 
 
 @pytest.fixture
@@ -66,7 +66,7 @@ async def test_singlestore_source(temp_dir: Path, source_database_setup: dict):
     )
     indexer = SingleStoreIndexer(
         connection_config=connection_config,
-        index_config=SingleStoreIndexerConfig(table_name="cars", id_column="car_id", batch_size=5),
+        index_config=SingleStoreIndexerConfig(table_name="cars", id_column="car_id", batch_size=6),
     )
     downloader = SingleStoreDownloader(
         connection_config=connection_config,
@@ -80,7 +80,7 @@ async def test_singlestore_source(temp_dir: Path, source_database_setup: dict):
         configs=SourceValidationConfigs(
             test_id="singlestore",
             expected_num_files=SEED_DATA_ROWS,
-            expected_number_indexed_file_data=4,
+            expected_number_indexed_file_data=2,
             validate_downloaded_files=True,
         ),
     )
@@ -103,7 +103,11 @@ def validate_destination(
 @pytest.mark.asyncio
 @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, "sql")
 async def test_singlestore_destination(upload_file: Path, temp_dir: Path):
-    mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
+    mock_file_data = FileData(
+        identifier="mock file data",
+        connector_type=CONNECTOR_TYPE,
+        source_identifiers=SourceIdentifiers(filename=upload_file.name, fullpath=upload_file.name),
+    )
     with docker_compose_context(
         docker_compose_path=env_setup_path / "sql" / "singlestore" / "destination"
     ):

{unstructured_ingest-0.3.9 → unstructured_ingest-0.3.11}/test/integration/connectors/sql/test_snowflake.py

@@ -17,7 +17,7 @@ from test.integration.connectors.utils.validation.source import (
     source_connector_validation,
 )
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.sql.snowflake import (
     CONNECTOR_TYPE,
     SnowflakeAccessConfig,
@@ -170,7 +170,11 @@ async def test_snowflake_destination(
 ):
     # the postgres destination connector doesn't leverage the file data but is required as an input,
     # mocking it with arbitrary values to meet the base requirements:
-    mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
+    mock_file_data = FileData(
+        identifier="mock file data",
+        connector_type=CONNECTOR_TYPE,
+        source_identifiers=SourceIdentifiers(filename=upload_file.name, fullpath=upload_file.name),
+    )
     init_db_destination()
     stager = SnowflakeUploadStager()
     staged_path = stager.run(

{unstructured_ingest-0.3.9 → unstructured_ingest-0.3.11}/test/integration/connectors/sql/test_sqlite.py

@@ -15,7 +15,7 @@ from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
 )
-from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.sql.sqlite import (
     CONNECTOR_TYPE,
     SQLiteConnectionConfig,
@@ -27,7 +27,7 @@ from unstructured_ingest.v2.processes.connectors.sql.sqlite import (
     SQLiteUploadStager,
 )
 
-SEED_DATA_ROWS = 20
+SEED_DATA_ROWS = 10
 
 
 @pytest.fixture
@@ -57,7 +57,7 @@ async def test_sqlite_source(source_database_setup: Path, temp_dir: Path):
     connection_config = SQLiteConnectionConfig(database_path=source_database_setup)
     indexer = SQLiteIndexer(
         connection_config=connection_config,
-        index_config=SQLiteIndexerConfig(table_name="cars", id_column="car_id", batch_size=5),
+        index_config=SQLiteIndexerConfig(table_name="cars", id_column="car_id", batch_size=6),
     )
     downloader = SQLiteDownloader(
         connection_config=connection_config,
@@ -69,7 +69,7 @@ async def test_sqlite_source(source_database_setup: Path, temp_dir: Path):
         configs=SourceValidationConfigs(
             test_id="sqlite",
             expected_num_files=SEED_DATA_ROWS,
-            expected_number_indexed_file_data=4,
+            expected_number_indexed_file_data=2,
             validate_downloaded_files=True,
         ),
     )
@@ -116,7 +116,11 @@ async def test_sqlite_destination(
 ):
     # the sqlite destination connector doesn't leverage the file data but is required as an input,
     # mocking it with arbitrary values to meet the base requirements:
-    mock_file_data = FileData(identifier="mock file data", connector_type=CONNECTOR_TYPE)
+    mock_file_data = FileData(
+        identifier="mock file data",
+        connector_type=CONNECTOR_TYPE,
+        source_identifiers=SourceIdentifiers(filename=upload_file.name, fullpath=upload_file.name),
+    )
     stager = SQLiteUploadStager()
     staged_path = stager.run(
         elements_filepath=upload_file,

{unstructured_ingest-0.3.9 → unstructured_ingest-0.3.11}/test/integration/connectors/test_astradb.py

@@ -14,12 +14,18 @@ from test.integration.connectors.utils.validation.destination import (
     StagerValidationConfigs,
     stager_validation,
 )
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.astradb import (
     CONNECTOR_TYPE,
     AstraDBAccessConfig,
     AstraDBConnectionConfig,
+    AstraDBDownloader,
+    AstraDBDownloaderConfig,
     AstraDBIndexer,
     AstraDBIndexerConfig,
     AstraDBUploader,
@@ -110,6 +116,40 @@ def collection(upload_file: Path) -> Collection:
         astra_db.drop_collection(collection)
 
 
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+@requires_env("ASTRA_DB_API_ENDPOINT", "ASTRA_DB_APPLICATION_TOKEN")
+async def test_astra_search_source(
+    tmp_path: Path,
+):
+    env_data = get_env_data()
+    collection_name = "ingest_test_src"
+    connection_config = AstraDBConnectionConfig(
+        access_config=AstraDBAccessConfig(token=env_data.token, api_endpoint=env_data.api_endpoint)
+    )
+    indexer = AstraDBIndexer(
+        index_config=AstraDBIndexerConfig(
+            collection_name=collection_name,
+        ),
+        connection_config=connection_config,
+    )
+    downloader = AstraDBDownloader(
+        connection_config=connection_config,
+        download_config=AstraDBDownloaderConfig(download_dir=tmp_path),
+    )
+
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id=CONNECTOR_TYPE,
+            expected_num_files=5,
+            expected_number_indexed_file_data=1,
+            validate_downloaded_files=True,
+        ),
+    )
+
+
 @pytest.mark.asyncio
 @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
 @requires_env("ASTRA_DB_API_ENDPOINT", "ASTRA_DB_APPLICATION_TOKEN")

{unstructured_ingest-0.3.9 → unstructured_ingest-0.3.11}/test/integration/connectors/test_kafka.py

@@ -122,7 +122,7 @@ async def test_kafka_source_local(kafka_seed_topic: str):
             indexer=indexer,
             downloader=downloader,
             configs=SourceValidationConfigs(
-                test_id="kafka", expected_num_files=5, validate_downloaded_files=True
+                test_id="kafka-local", expected_num_files=5, validate_downloaded_files=True
             ),
         )
 
@@ -204,7 +204,7 @@ async def test_kafka_source_cloud(kafka_seed_topic_cloud: int):
             indexer=indexer,
             downloader=downloader,
             configs=SourceValidationConfigs(
-                test_id="kafka",
+                test_id="kafka-cloud",
                 exclude_fields_extend=["connector_type"],
                 expected_num_files=expected_messages,
                 validate_downloaded_files=True,

{unstructured_ingest-0.3.9 → unstructured_ingest-0.3.11}/test/integration/connectors/test_mongodb.py

@@ -197,7 +197,10 @@ async def test_mongodb_source(temp_dir: Path):
         indexer=indexer,
         downloader=downloader,
         configs=SourceValidationConfigs(
-            test_id=CONNECTOR_TYPE, expected_num_files=4, validate_downloaded_files=True
+            test_id=CONNECTOR_TYPE,
+            expected_num_files=4,
+            validate_downloaded_files=True,
+            expected_number_indexed_file_data=1,
         ),
     )
 

{unstructured_ingest-0.3.9 → unstructured_ingest-0.3.11}/test/integration/connectors/utils/validation/source.py

@@ -1,14 +1,13 @@
 import json
 import os
 import shutil
-from dataclasses import replace
 from pathlib import Path
 from typing import Callable, Optional
 
 from deepdiff import DeepDiff
 from pydantic import Field
 
-from test.integration.connectors.utils.validation.utils import ValidationConfig, reset_dir
+from test.integration.connectors.utils.validation.utils import ValidationConfig
 from unstructured_ingest.v2.interfaces import Downloader, FileData, Indexer
 
 
@@ -92,7 +91,7 @@ def check_contents(
         file_data_path = expected_output_dir / f"{file_data.identifier}.json"
         with file_data_path.open("r") as file:
             expected_file_data_contents = json.load(file)
-        current_file_data_contents = file_data.to_dict()
+        current_file_data_contents = file_data.model_dump()
         expected_file_data_contents = configs.omit_ignored_fields(expected_file_data_contents)
         current_file_data_contents = configs.omit_ignored_fields(current_file_data_contents)
         diff = DeepDiff(expected_file_data_contents, current_file_data_contents)
@@ -160,9 +159,11 @@ def update_fixtures(
     save_filedata: bool = True,
 ):
     # Rewrite the current file data
+    if not output_dir.exists():
+        output_dir.mkdir(parents=True)
     if save_filedata:
         file_data_output_path = output_dir / "file_data"
-        reset_dir(dir_path=file_data_output_path)
+        shutil.rmtree(path=file_data_output_path, ignore_errors=True)
         print(
             f"Writing {len(all_file_data)} file data to "
             f"saved fixture location {file_data_output_path}"
@@ -171,7 +172,7 @@ def update_fixtures(
         for file_data in all_file_data:
             file_data_path = file_data_output_path / f"{file_data.identifier}.json"
             with file_data_path.open(mode="w") as f:
-                json.dump(file_data.to_dict(), f, indent=2)
+                json.dump(file_data.model_dump(), f, indent=2)
 
     # Record file structure of download directory
     download_files = get_files(dir_path=download_dir)
@@ -183,7 +184,7 @@ def update_fixtures(
     # If applicable, save raw downloads
     if save_downloads:
         raw_download_output_path = output_dir / "downloads"
-        reset_dir(raw_download_output_path)
+        shutil.rmtree(path=raw_download_output_path, ignore_errors=True)
         print(
             f"Writing {len(download_files)} downloaded files to "
             f"saved fixture location {raw_download_output_path}"
@@ -213,7 +214,10 @@ def run_all_validations(
     if configs.validate_file_data:
         run_expected_results_validation(
             expected_output_dir=test_output_dir / "file_data",
-            all_file_data=postdownload_file_data,
+            all_file_data=get_all_file_data(
+                all_predownload_file_data=predownload_file_data,
+                all_postdownload_file_data=postdownload_file_data,
+            ),
             configs=configs,
         )
     download_files = get_files(dir_path=download_dir)
@@ -229,6 +233,19 @@ def run_all_validations(
         )
 
 
+def get_all_file_data(
+    all_postdownload_file_data: list[FileData], all_predownload_file_data: list[FileData]
+) -> list[FileData]:
+    all_file_data = all_postdownload_file_data
+    indexed_file_data = [
+        fd
+        for fd in all_predownload_file_data
+        if fd.identifier not in [f.identifier for f in all_file_data]
+    ]
+    all_file_data += indexed_file_data
+    return all_file_data
+
+
 async def source_connector_validation(
     indexer: Indexer,
     downloader: Downloader,
@@ -246,7 +263,7 @@ async def source_connector_validation(
     test_output_dir = configs.test_output_dir()
     for file_data in indexer.run():
         assert file_data
-        predownload_file_data = replace(file_data)
+        predownload_file_data = file_data.model_copy(deep=True)
         all_predownload_file_data.append(predownload_file_data)
         if downloader.is_async():
             resp = await downloader.run_async(file_data=file_data)
@@ -254,10 +271,10 @@ async def source_connector_validation(
             resp = downloader.run(file_data=file_data)
         if isinstance(resp, list):
             for r in resp:
-                postdownload_file_data = replace(r["file_data"])
+                postdownload_file_data = r["file_data"].model_copy(deep=True)
                 all_postdownload_file_data.append(postdownload_file_data)
         else:
-            postdownload_file_data = replace(resp["file_data"])
+            postdownload_file_data = resp["file_data"].model_copy(deep=True)
             all_postdownload_file_data.append(postdownload_file_data)
     if not overwrite_fixtures:
         print("Running validation")
@@ -273,7 +290,10 @@ async def source_connector_validation(
         update_fixtures(
             output_dir=test_output_dir,
             download_dir=download_dir,
-            all_file_data=all_postdownload_file_data,
+            all_file_data=get_all_file_data(
+                all_predownload_file_data=all_predownload_file_data,
+                all_postdownload_file_data=all_postdownload_file_data,
+            ),
             save_downloads=configs.validate_downloaded_files,
             save_filedata=configs.validate_file_data,
         )

unstructured_ingest-0.3.11/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.3.11"  # pragma: no cover