unstructured-ingest 0.4.7__tar.gz → 0.5.1__tar.gz

{unstructured_ingest-0.4.7/unstructured_ingest.egg-info → unstructured_ingest-0.5.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.4.7
+Version: 0.5.1
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -23,12 +23,12 @@ Requires-Python: >=3.9.0,<3.14
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
 Requires-Dist: opentelemetry-sdk
-Requires-Dist: click
+Requires-Dist: python-dateutil
 Requires-Dist: tqdm
-Requires-Dist: pandas
 Requires-Dist: pydantic>=2.7
+Requires-Dist: pandas
 Requires-Dist: dataclasses_json
-Requires-Dist: python-dateutil
+Requires-Dist: click
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
@@ -74,33 +74,33 @@ Provides-Extra: biomed
 Requires-Dist: bs4; extra == "biomed"
 Requires-Dist: requests; extra == "biomed"
 Provides-Extra: box
-Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
+Requires-Dist: boxfs; extra == "box"
 Provides-Extra: chroma
 Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: requests; extra == "confluence"
 Requires-Dist: atlassian-python-api; extra == "confluence"
+Requires-Dist: requests; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: boto3; extra == "delta-table"
 Requires-Dist: deltalake; extra == "delta-table"
+Requires-Dist: boto3; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord.py; extra == "discord"
 Provides-Extra: dropbox
-Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Requires-Dist: fsspec; extra == "dropbox"
+Requires-Dist: dropboxdrivefs; extra == "dropbox"
 Provides-Extra: duckdb
 Requires-Dist: duckdb; extra == "duckdb"
 Provides-Extra: elasticsearch
 Requires-Dist: elasticsearch[async]; extra == "elasticsearch"
 Provides-Extra: gcs
+Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
 Requires-Dist: gcsfs; extra == "gcs"
-Requires-Dist: fsspec; extra == "gcs"
 Provides-Extra: github
 Requires-Dist: requests; extra == "github"
 Requires-Dist: pygithub>1.58.0; extra == "github"
@@ -109,8 +109,8 @@ Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: urllib3; extra == "hubspot"
 Requires-Dist: hubspot-api-client; extra == "hubspot"
+Requires-Dist: urllib3; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -124,14 +124,14 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
+Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: neo4j; extra == "neo4j"
-Requires-Dist: networkx; extra == "neo4j"
 Provides-Extra: notion
+Requires-Dist: httpx; extra == "notion"
+Requires-Dist: notion-client; extra == "notion"
 Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
-Requires-Dist: notion-client; extra == "notion"
-Requires-Dist: httpx; extra == "notion"
 Provides-Extra: onedrive
 Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
@@ -152,21 +152,21 @@ Requires-Dist: praw; extra == "reddit"
 Provides-Extra: redis
 Requires-Dist: redis; extra == "redis"
 Provides-Extra: s3
-Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
+Requires-Dist: s3fs; extra == "s3"
 Provides-Extra: sharepoint
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
 Requires-Dist: msal; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: paramiko; extra == "sftp"
 Requires-Dist: fsspec; extra == "sftp"
+Requires-Dist: paramiko; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: psycopg2-binary; extra == "snowflake"
 Requires-Dist: snowflake-connector-python; extra == "snowflake"
+Requires-Dist: psycopg2-binary; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate
@@ -179,17 +179,17 @@ Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: vectara
 Requires-Dist: aiofiles; extra == "vectara"
-Requires-Dist: requests; extra == "vectara"
 Requires-Dist: httpx; extra == "vectara"
+Requires-Dist: requests; extra == "vectara"
 Provides-Extra: vastdb
-Requires-Dist: vastdb; extra == "vastdb"
 Requires-Dist: pyarrow; extra == "vastdb"
 Requires-Dist: ibis; extra == "vastdb"
+Requires-Dist: vastdb; extra == "vastdb"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai
-Requires-Dist: tiktoken; extra == "embed-octoai"
 Requires-Dist: openai; extra == "embed-octoai"
+Requires-Dist: tiktoken; extra == "embed-octoai"
 Provides-Extra: embed-vertexai
 Requires-Dist: vertexai; extra == "embed-vertexai"
 Provides-Extra: embed-voyageai
@@ -197,11 +197,11 @@ Requires-Dist: voyageai; extra == "embed-voyageai"
 Provides-Extra: embed-mixedbreadai
 Requires-Dist: mixedbread-ai; extra == "embed-mixedbreadai"
 Provides-Extra: openai
-Requires-Dist: tiktoken; extra == "openai"
 Requires-Dist: openai; extra == "openai"
+Requires-Dist: tiktoken; extra == "openai"
 Provides-Extra: bedrock
-Requires-Dist: aioboto3; extra == "bedrock"
 Requires-Dist: boto3; extra == "bedrock"
+Requires-Dist: aioboto3; extra == "bedrock"
 Provides-Extra: togetherai
 Requires-Dist: together; extra == "togetherai"
 

{unstructured_ingest-0.4.7 → unstructured_ingest-0.5.1}/setup.py

@@ -142,7 +142,7 @@ embed_reqs = {
     "embed-voyageai": load_requirements("requirements/embed/voyageai.in"),
     "embed-mixedbreadai": load_requirements("requirements/embed/mixedbreadai.in"),
     "openai": load_requirements("requirements/embed/openai.in"),
-    "bedrock": load_requirements("requirements/embed/aws-bedrock.in"),
+    "bedrock": load_requirements("requirements/embed/bedrock.in"),
     "togetherai": load_requirements("requirements/embed/togetherai.in"),
 }
 

unstructured_ingest-0.5.1/test/integration/connectors/sql/test_vastdb.py

@@ -0,0 +1,34 @@
+from pathlib import Path
+
+import pytest
+from _pytest.fixtures import TopRequest
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG, SQL_TAG
+from test.integration.connectors.utils.validation.destination import (
+    StagerValidationConfigs,
+    stager_validation,
+)
+from unstructured_ingest.v2.processes.connectors.sql.vastdb import (
+    CONNECTOR_TYPE,
+    VastdbUploadStager,
+    VastdbUploadStagerConfig,
+)
+
+
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, SQL_TAG)
+@pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
+def test_vast_stager(
+    request: TopRequest,
+    upload_file_str: str,
+    tmp_path: Path,
+):
+    upload_file: Path = request.getfixturevalue(upload_file_str)
+    stager = VastdbUploadStager(
+        upload_stager_config=VastdbUploadStagerConfig(rename_columns_map={"page_number": "page"})
+    )
+    stager_validation(
+        configs=StagerValidationConfigs(test_id=CONNECTOR_TYPE, expected_count=22),
+        input_file=upload_file,
+        stager=stager,
+        tmp_dir=tmp_path,
+    )

unstructured_ingest-0.5.1/test/integration/connectors/test_google_drive.py

@@ -0,0 +1,116 @@
+import os
+
+import pytest
+
+from test.integration.connectors.utils.constants import (
+    SOURCE_TAG,
+    UNCATEGORIZED_TAG,
+)
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    get_all_file_data,
+    run_all_validations,
+    update_fixtures,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.interfaces import Downloader, Indexer
+from unstructured_ingest.v2.processes.connectors.google_drive import (
+    CONNECTOR_TYPE,
+    GoogleDriveAccessConfig,
+    GoogleDriveConnectionConfig,
+    GoogleDriveDownloader,
+    GoogleDriveDownloaderConfig,
+    GoogleDriveIndexer,
+    GoogleDriveIndexerConfig,
+)
+
+
+@requires_env("GOOGLE_DRIVE_SERVICE_KEY")
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE)
+def test_google_drive_source(temp_dir):
+    # Retrieve environment variables
+    service_account_key = os.environ["GOOGLE_DRIVE_SERVICE_KEY"]
+
+    # Create connection and indexer configurations
+    access_config = GoogleDriveAccessConfig(service_account_key=service_account_key)
+    connection_config = GoogleDriveConnectionConfig(
+        drive_id="1XidSOO76VpZ4m0i3gJN2m1X0Obol3UAi",
+        access_config=access_config,
+    )
+    index_config = GoogleDriveIndexerConfig(recursive=True)
+
+    download_config = GoogleDriveDownloaderConfig(download_dir=temp_dir)
+
+    # Instantiate indexer and downloader
+    indexer = GoogleDriveIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = GoogleDriveDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+    )
+
+    # Run the source connector validation
+    source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="google_drive_source",
+            expected_num_files=1,
+            validate_downloaded_files=True,
+        ),
+    )
+
+
+@pytest.mark.tags(SOURCE_TAG, CONNECTOR_TYPE, UNCATEGORIZED_TAG)
+def source_connector_validation(
+    indexer: Indexer,
+    downloader: Downloader,
+    configs: SourceValidationConfigs,
+    overwrite_fixtures: bool = os.getenv("OVERWRITE_FIXTURES", "False").lower() == "true",
+) -> None:
+    # Run common validations on the process of running a source connector, supporting dynamic
+    # validators that get passed in along with comparisons on the saved expected values.
+    # If overwrite_fixtures is st to True, will ignore all validators but instead overwrite the
+    # expected values with what gets generated by this test.
+    all_predownload_file_data = []
+    all_postdownload_file_data = []
+    indexer.precheck()
+    download_dir = downloader.download_config.download_dir
+    test_output_dir = configs.test_output_dir()
+
+    for file_data in indexer.run():
+        assert file_data
+        predownload_file_data = file_data.model_copy(deep=True)
+        all_predownload_file_data.append(predownload_file_data)
+        resp = downloader.run(file_data=file_data)
+        if isinstance(resp, list):
+            for r in resp:
+                postdownload_file_data = r["file_data"].model_copy(deep=True)
+                all_postdownload_file_data.append(postdownload_file_data)
+        else:
+            postdownload_file_data = resp["file_data"].model_copy(deep=True)
+            all_postdownload_file_data.append(postdownload_file_data)
+
+    if not overwrite_fixtures:
+        print("Running validation")
+        run_all_validations(
+            configs=configs,
+            predownload_file_data=all_predownload_file_data,
+            postdownload_file_data=all_postdownload_file_data,
+            download_dir=download_dir,
+            test_output_dir=test_output_dir,
+        )
+    else:
+        print("Running fixtures update")
+        update_fixtures(
+            output_dir=test_output_dir,
+            download_dir=download_dir,
+            all_file_data=get_all_file_data(
+                all_predownload_file_data=all_predownload_file_data,
+                all_postdownload_file_data=all_postdownload_file_data,
+            ),
+            save_downloads=configs.validate_downloaded_files,
+            save_filedata=configs.validate_file_data,
+        )

{unstructured_ingest-0.4.7 → unstructured_ingest-0.5.1}/test/integration/connectors/test_onedrive.py

@@ -5,13 +5,25 @@ from pathlib import Path
 import pytest
 from office365.graph_client import GraphClient
 
-from test.integration.connectors.utils.constants import BLOB_STORAGE_TAG, DESTINATION_TAG
+from test.integration.connectors.utils.constants import (
+    BLOB_STORAGE_TAG,
+    DESTINATION_TAG,
+    SOURCE_TAG,
+)
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
 from test.integration.utils import requires_env
 from unstructured_ingest.v2.interfaces import FileData, SourceIdentifiers
 from unstructured_ingest.v2.processes.connectors.onedrive import (
     CONNECTOR_TYPE,
     OnedriveAccessConfig,
     OnedriveConnectionConfig,
+    OnedriveDownloader,
+    OnedriveDownloaderConfig,
+    OnedriveIndexer,
+    OnedriveIndexerConfig,
     OnedriveUploader,
     OnedriveUploaderConfig,
 )
@@ -62,9 +74,46 @@ def get_connection_config():
     return connection_config
 
 
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
+@requires_env("MS_CLIENT_CRED", "MS_CLIENT_ID", "MS_TENANT_ID", "MS_USER_PNAME")
+async def test_onedrive_source(temp_dir):
+    connection_config = get_connection_config()
+    index_config = OnedriveIndexerConfig(recursive=True, path="eml")
+
+    download_config = OnedriveDownloaderConfig(download_dir=temp_dir)
+
+    # Instantiate indexer and downloader
+    indexer = OnedriveIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = OnedriveDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="onedrive",
+            expected_num_files=1,
+            validate_downloaded_files=True,
+            exclude_fields_extend=[
+                "metadata.date_created",
+                "metadata.date_modified",
+                "additional_metadata.LastModified",
+                "additional_metadata.@microsoft.graph.downloadUrl",
+            ],
+        ),
+    )
+
+
 @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, BLOB_STORAGE_TAG)
 @requires_env("MS_CLIENT_CRED", "MS_CLIENT_ID", "MS_TENANT_ID", "MS_USER_PNAME")
-def test_onedrive_destination(upload_file: Path, onedrive_test_folder: str):
+def xtest_onedrive_destination(upload_file: Path, onedrive_test_folder: str):
     """
     Integration test for the OneDrive destination connector.
 

unstructured_ingest-0.5.1/test/integration/connectors/test_sharepoint.py

@@ -0,0 +1,71 @@
+import os
+
+import pytest
+
+from test.integration.connectors.utils.constants import BLOB_STORAGE_TAG, SOURCE_TAG
+from test.integration.connectors.utils.validation.source import (
+    SourceValidationConfigs,
+    source_connector_validation,
+)
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.processes.connectors.sharepoint import (
+    CONNECTOR_TYPE,
+    SharepointAccessConfig,
+    SharepointConnectionConfig,
+    SharepointDownloader,
+    SharepointDownloaderConfig,
+    SharepointIndexer,
+    SharepointIndexerConfig,
+)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG, BLOB_STORAGE_TAG)
+@requires_env("SHAREPOINT_CLIENT_ID", "SHAREPOINT_CRED", "MS_TENANT_ID", "MS_USER_PNAME")
+async def test_sharepoint_source(temp_dir):
+    # Retrieve environment variables
+    site = "https://unstructuredio.sharepoint.com/sites/utic-platform-test-source"
+    client_id = os.environ["SHAREPOINT_CLIENT_ID"]
+    client_cred = os.environ["SHAREPOINT_CRED"]
+    user_pname = os.environ["MS_USER_PNAME"]
+    tenant = os.environ["MS_TENANT_ID"]
+
+    # Create connection and indexer configurations
+    access_config = SharepointAccessConfig(client_cred=client_cred)
+    connection_config = SharepointConnectionConfig(
+        client_id=client_id,
+        site=site,
+        tenant=tenant,
+        user_pname=user_pname,
+        access_config=access_config,
+    )
+    index_config = SharepointIndexerConfig(recursive=True)
+
+    download_config = SharepointDownloaderConfig(download_dir=temp_dir)
+
+    # Instantiate indexer and downloader
+    indexer = SharepointIndexer(
+        connection_config=connection_config,
+        index_config=index_config,
+    )
+    downloader = SharepointDownloader(
+        connection_config=connection_config,
+        download_config=download_config,
+    )
+
+    # Run the source connector validation
+    await source_connector_validation(
+        indexer=indexer,
+        downloader=downloader,
+        configs=SourceValidationConfigs(
+            test_id="sharepoint",
+            expected_num_files=4,
+            validate_downloaded_files=True,
+            exclude_fields_extend=[
+                "metadata.date_created",
+                "metadata.date_modified",
+                "additional_metadata.LastModified",
+                "additional_metadata.@microsoft.graph.downloadUrl",
+            ],
+        ),
+    )

{unstructured_ingest-0.4.7 → unstructured_ingest-0.5.1}/test/integration/connectors/utils/validation/source.py

@@ -10,6 +10,13 @@ from pydantic import Field
 from test.integration.connectors.utils.validation.utils import ValidationConfig
 from unstructured_ingest.v2.interfaces import Downloader, FileData, Indexer
 
+NONSTANDARD_METADATA_FIELDS = {
+    "additional_metadata.@microsoft.graph.downloadUrl": [
+        "additional_metadata",
+        "@microsoft.graph.downloadUrl",
+    ]
+}
+
 
 class SourceValidationConfigs(ValidationConfig):
     expected_number_indexed_file_data: Optional[int] = None
@@ -26,7 +33,7 @@ class SourceValidationConfigs(ValidationConfig):
     def get_exclude_fields(self) -> list[str]:
         exclude_fields = self.exclude_fields
         exclude_fields.extend(self.exclude_fields_extend)
-        return exclude_fields
+        return list(set(exclude_fields))
 
     def run_file_data_validation(
         self, predownload_file_data: FileData, postdownload_file_data: FileData
@@ -45,8 +52,13 @@ class SourceValidationConfigs(ValidationConfig):
         exclude_fields = self.get_exclude_fields()
         # Ignore fields that dynamically change every time the tests run
         copied_data = data.copy()
+
         for exclude_field in exclude_fields:
-            exclude_field_vals = exclude_field.split(".")
+            exclude_field_vals = (
+                NONSTANDARD_METADATA_FIELDS[exclude_field]
+                if exclude_field in NONSTANDARD_METADATA_FIELDS
+                else exclude_field.split(".")
+            )
             if len(exclude_field_vals) == 1:
                 current_val = copied_data
                 drop_field = exclude_field_vals[0]
@@ -261,21 +273,38 @@ async def source_connector_validation(
     indexer.precheck()
     download_dir = downloader.download_config.download_dir
     test_output_dir = configs.test_output_dir()
-    for file_data in indexer.run():
-        assert file_data
-        predownload_file_data = file_data.model_copy(deep=True)
-        all_predownload_file_data.append(predownload_file_data)
-        if downloader.is_async():
-            resp = await downloader.run_async(file_data=file_data)
-        else:
-            resp = downloader.run(file_data=file_data)
-        if isinstance(resp, list):
-            for r in resp:
-                postdownload_file_data = r["file_data"].model_copy(deep=True)
+    if indexer.is_async():
+        async for file_data in indexer.run_async():
+            assert file_data
+            predownload_file_data = file_data.model_copy(deep=True)
+            all_predownload_file_data.append(predownload_file_data)
+            if downloader.is_async():
+                resp = await downloader.run_async(file_data=file_data)
+            else:
+                resp = downloader.run(file_data=file_data)
+            if isinstance(resp, list):
+                for r in resp:
+                    postdownload_file_data = r["file_data"].model_copy(deep=True)
+                    all_postdownload_file_data.append(postdownload_file_data)
+            else:
+                postdownload_file_data = resp["file_data"].model_copy(deep=True)
+                all_postdownload_file_data.append(postdownload_file_data)
+    else:
+        for file_data in indexer.run():
+            assert file_data
+            predownload_file_data = file_data.model_copy(deep=True)
+            all_predownload_file_data.append(predownload_file_data)
+            if downloader.is_async():
+                resp = await downloader.run_async(file_data=file_data)
+            else:
+                resp = downloader.run(file_data=file_data)
+            if isinstance(resp, list):
+                for r in resp:
+                    postdownload_file_data = r["file_data"].model_copy(deep=True)
+                    all_postdownload_file_data.append(postdownload_file_data)
+            else:
+                postdownload_file_data = resp["file_data"].model_copy(deep=True)
                 all_postdownload_file_data.append(postdownload_file_data)
-        else:
-            postdownload_file_data = resp["file_data"].model_copy(deep=True)
-            all_postdownload_file_data.append(postdownload_file_data)
     if not overwrite_fixtures:
         print("Running validation")
         run_all_validations(

{unstructured_ingest-0.4.7 → unstructured_ingest-0.5.1}/test/integration/embedders/test_bedrock.py

@@ -31,7 +31,7 @@ def get_aws_credentials() -> dict:
 def test_bedrock_embedder(embedder_file: Path):
     aws_credentials = get_aws_credentials()
     embedder_config = EmbedderConfig(
-        embedding_provider="aws-bedrock",
+        embedding_provider="bedrock",
         embedding_aws_access_key_id=aws_credentials["aws_access_key_id"],
         embedding_aws_secret_access_key=aws_credentials["aws_secret_access_key"],
     )

{unstructured_ingest-0.4.7 → unstructured_ingest-0.5.1}/test/integration/partitioners/test_partitioner.py

@@ -1,4 +1,3 @@
-import json
 import os
 from pathlib import Path
 
@@ -15,6 +14,9 @@ all_partition_files = [path for path in assets_dir.iterdir() if path.is_file()]
 non_image_partition_files = [
     path for path in all_partition_files if path.suffix not in [".jpg", ".png", ".tif"]
 ]
+supported_fast_partition_files = [
+    path for path in non_image_partition_files if path.suffix != ".eml"
+]
 image_partition_files = [
     path for path in all_partition_files if path not in non_image_partition_files
 ]
@@ -33,18 +35,13 @@ async def test_partitioner_api_hi_res(partition_file: Path):
     )
     partitioner = Partitioner(config=partitioner_config)
     results = await partitioner.run_async(filename=partition_file)
-    results_dir = int_test_dir / "results"
-    results_dir.mkdir(exist_ok=True)
-    results_path = results_dir / f"{partition_file.name}.json"
-    with results_path.open("w") as f:
-        json.dump(results, f, indent=2)
     assert results
 
 
 @pytest.mark.parametrize(
     "partition_file",
-    non_image_partition_files,
-    ids=[path.name for path in non_image_partition_files],
+    supported_fast_partition_files,
+    ids=[path.name for path in supported_fast_partition_files],
 )
 @requires_env("UNSTRUCTURED_API_KEY", "UNSTRUCTURED_API_URL")
 @pytest.mark.asyncio
@@ -68,7 +65,11 @@ async def test_partitioner_api_fast_error(partition_file: Path):
     api_key = os.getenv("UNSTRUCTURED_API_KEY")
     api_url = os.getenv("UNSTRUCTURED_API_URL")
     partitioner_config = PartitionerConfig(
-        strategy="fast", partition_by_api=True, api_key=api_key, partition_endpoint=api_url
+        strategy="fast",
+        partition_by_api=True,
+        api_key=api_key,
+        partition_endpoint=api_url,
+        raise_unsupported_filetype=True,
     )
     partitioner = Partitioner(config=partitioner_config)
     with pytest.raises(UserError):

unstructured_ingest-0.5.1/test/unit/v2/connectors/motherduck/test_base.py

@@ -0,0 +1,74 @@
+from pathlib import Path
+
+import pytest
+from pytest_mock import MockerFixture
+
+from unstructured_ingest.v2.interfaces import FileData
+from unstructured_ingest.v2.interfaces.file_data import SourceIdentifiers
+from unstructured_ingest.v2.interfaces.upload_stager import UploadStagerConfig
+from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUploadStager
+
+
+@pytest.fixture
+def mock_instance() -> BaseDuckDBUploadStager:
+    return BaseDuckDBUploadStager(UploadStagerConfig())
+
+
+@pytest.mark.parametrize(
+    ("input_filepath", "output_filename", "expected"),
+    [
+        (
+            "/path/to/input_file.ndjson",
+            "output_file.ndjson",
+            "output_file.ndjson",
+        ),
+        ("input_file.txt", "output_file.json", "output_file.txt"),
+        ("/path/to/input_file.json", "output_file", "output_file.json"),
+    ],
+)
+def test_run_output_filename_suffix(
+    mocker: MockerFixture,
+    mock_instance: BaseDuckDBUploadStager,
+    input_filepath: str,
+    output_filename: str,
+    expected: str,
+):
+    output_dir = Path("/tmp/test/output_dir")
+
+    # Mocks
+    mock_get_data = mocker.patch(
+        "unstructured_ingest.v2.processes.connectors.duckdb.base.get_data",
+        return_value=[{"key": "value"}, {"key": "value2"}],
+    )
+    mock_conform_dict = mocker.patch.object(
+        BaseDuckDBUploadStager,
+        "conform_dict",
+        side_effect=lambda element_dict, file_data: element_dict,
+    )
+    mock_get_output_path = mocker.patch.object(
+        BaseDuckDBUploadStager, "get_output_path", return_value=output_dir / expected
+    )
+    mock_write_output = mocker.patch(
+        "unstructured_ingest.v2.processes.connectors.duckdb.base.write_data", return_value=None
+    )
+
+    # Act
+    result = mock_instance.run(
+        elements_filepath=Path(input_filepath),
+        file_data=FileData(
+            identifier="test",
+            connector_type="test",
+            source_identifiers=SourceIdentifiers(filename=input_filepath, fullpath=input_filepath),
+        ),
+        output_dir=output_dir,
+        output_filename=output_filename,
+    )
+
+    # Assert
+    mock_get_data.assert_called_once_with(path=Path(input_filepath))
+    assert mock_conform_dict.call_count == 2
+    mock_get_output_path.assert_called_once_with(output_filename=expected, output_dir=output_dir)
+    mock_write_output.assert_called_once_with(
+        path=output_dir / expected, data=[{"key": "value"}, {"key": "value2"}]
+    )
+    assert result.name == expected

unstructured_ingest-0.5.1/unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.5.1"  # pragma: no cover