unstructured-ingest 0.3.11__tar.gz → 0.3.12__tar.gz

{unstructured_ingest-0.3.11/unstructured_ingest.egg-info → unstructured_ingest-0.3.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: unstructured-ingest
-Version: 0.3.11
+Version: 0.3.12
 Summary: A library that prepares raw documents for downstream ML tasks.
 Home-page: https://github.com/Unstructured-IO/unstructured-ingest
 Author: Unstructured Technologies
@@ -22,14 +22,14 @@ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Requires-Python: >=3.9.0,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE.md
-Requires-Dist: opentelemetry-sdk
-Requires-Dist: dataclasses_json
 Requires-Dist: ndjson
-Requires-Dist: click
 Requires-Dist: tqdm
-Requires-Dist: python-dateutil
+Requires-Dist: opentelemetry-sdk
+Requires-Dist: click
+Requires-Dist: dataclasses_json
 Requires-Dist: pydantic>=2.7
 Requires-Dist: pandas
+Requires-Dist: python-dateutil
 Provides-Extra: remote
 Requires-Dist: unstructured-client>=0.26.1; extra == "remote"
 Provides-Extra: csv
@@ -67,13 +67,13 @@ Requires-Dist: pyairtable; extra == "airtable"
 Provides-Extra: astradb
 Requires-Dist: astrapy; extra == "astradb"
 Provides-Extra: azure
-Requires-Dist: fsspec; extra == "azure"
 Requires-Dist: adlfs; extra == "azure"
+Requires-Dist: fsspec; extra == "azure"
 Provides-Extra: azure-ai-search
 Requires-Dist: azure-search-documents; extra == "azure-ai-search"
 Provides-Extra: biomed
-Requires-Dist: bs4; extra == "biomed"
 Requires-Dist: requests; extra == "biomed"
+Requires-Dist: bs4; extra == "biomed"
 Provides-Extra: box
 Requires-Dist: boxfs; extra == "box"
 Requires-Dist: fsspec; extra == "box"
@@ -82,13 +82,13 @@ Requires-Dist: chromadb; extra == "chroma"
 Provides-Extra: clarifai
 Requires-Dist: clarifai; extra == "clarifai"
 Provides-Extra: confluence
-Requires-Dist: atlassian-python-api; extra == "confluence"
 Requires-Dist: requests; extra == "confluence"
+Requires-Dist: atlassian-python-api; extra == "confluence"
 Provides-Extra: couchbase
 Requires-Dist: couchbase; extra == "couchbase"
 Provides-Extra: delta-table
-Requires-Dist: boto3; extra == "delta-table"
 Requires-Dist: deltalake; extra == "delta-table"
+Requires-Dist: boto3; extra == "delta-table"
 Provides-Extra: discord
 Requires-Dist: discord-py; extra == "discord"
 Provides-Extra: dropbox
@@ -103,15 +103,15 @@ Requires-Dist: gcsfs; extra == "gcs"
 Requires-Dist: fsspec; extra == "gcs"
 Requires-Dist: bs4; extra == "gcs"
 Provides-Extra: github
-Requires-Dist: pygithub>1.58.0; extra == "github"
 Requires-Dist: requests; extra == "github"
+Requires-Dist: pygithub>1.58.0; extra == "github"
 Provides-Extra: gitlab
 Requires-Dist: python-gitlab; extra == "gitlab"
 Provides-Extra: google-drive
 Requires-Dist: google-api-python-client; extra == "google-drive"
 Provides-Extra: hubspot
-Requires-Dist: hubspot-api-client; extra == "hubspot"
 Requires-Dist: urllib3; extra == "hubspot"
+Requires-Dist: hubspot-api-client; extra == "hubspot"
 Provides-Extra: jira
 Requires-Dist: atlassian-python-api; extra == "jira"
 Provides-Extra: kafka
@@ -125,23 +125,23 @@ Requires-Dist: pymilvus; extra == "milvus"
 Provides-Extra: mongodb
 Requires-Dist: pymongo; extra == "mongodb"
 Provides-Extra: neo4j
-Requires-Dist: cymple; extra == "neo4j"
 Requires-Dist: networkx; extra == "neo4j"
 Requires-Dist: neo4j; extra == "neo4j"
+Requires-Dist: cymple; extra == "neo4j"
 Provides-Extra: notion
-Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: backoff; extra == "notion"
+Requires-Dist: htmlBuilder; extra == "notion"
 Requires-Dist: httpx; extra == "notion"
 Requires-Dist: notion-client; extra == "notion"
 Provides-Extra: onedrive
-Requires-Dist: msal; extra == "onedrive"
-Requires-Dist: bs4; extra == "onedrive"
 Requires-Dist: Office365-REST-Python-Client; extra == "onedrive"
+Requires-Dist: bs4; extra == "onedrive"
+Requires-Dist: msal; extra == "onedrive"
 Provides-Extra: opensearch
 Requires-Dist: opensearch-py; extra == "opensearch"
 Provides-Extra: outlook
-Requires-Dist: msal; extra == "outlook"
 Requires-Dist: Office365-REST-Python-Client; extra == "outlook"
+Requires-Dist: msal; extra == "outlook"
 Provides-Extra: pinecone
 Requires-Dist: pinecone-client>=3.7.1; extra == "pinecone"
 Provides-Extra: postgres
@@ -150,22 +150,24 @@ Provides-Extra: qdrant
 Requires-Dist: qdrant-client; extra == "qdrant"
 Provides-Extra: reddit
 Requires-Dist: praw; extra == "reddit"
+Provides-Extra: redis
+Requires-Dist: redis; extra == "redis"
 Provides-Extra: s3
-Requires-Dist: s3fs; extra == "s3"
 Requires-Dist: fsspec; extra == "s3"
+Requires-Dist: s3fs; extra == "s3"
 Provides-Extra: sharepoint
-Requires-Dist: msal; extra == "sharepoint"
 Requires-Dist: Office365-REST-Python-Client; extra == "sharepoint"
+Requires-Dist: msal; extra == "sharepoint"
 Provides-Extra: salesforce
 Requires-Dist: simple-salesforce; extra == "salesforce"
 Provides-Extra: sftp
-Requires-Dist: fsspec; extra == "sftp"
 Requires-Dist: paramiko; extra == "sftp"
+Requires-Dist: fsspec; extra == "sftp"
 Provides-Extra: slack
 Requires-Dist: slack_sdk[optional]; extra == "slack"
 Provides-Extra: snowflake
-Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Requires-Dist: psycopg2-binary; extra == "snowflake"
+Requires-Dist: snowflake-connector-python; extra == "snowflake"
 Provides-Extra: wikipedia
 Requires-Dist: wikipedia; extra == "wikipedia"
 Provides-Extra: weaviate
@@ -176,6 +178,8 @@ Provides-Extra: singlestore
 Requires-Dist: singlestoredb; extra == "singlestore"
 Provides-Extra: vectara
 Requires-Dist: requests; extra == "vectara"
+Requires-Dist: aiofiles; extra == "vectara"
+Requires-Dist: httpx; extra == "vectara"
 Provides-Extra: embed-huggingface
 Requires-Dist: sentence-transformers; extra == "embed-huggingface"
 Provides-Extra: embed-octoai

{unstructured_ingest-0.3.11 → unstructured_ingest-0.3.12}/pyproject.toml

@@ -74,3 +74,6 @@ omit=[
     "unstructured_ingest/ingest_backoff/*",
     "unstructured_ingest/enhanced_dataclass/*"
 ]
+
+[tool.pytest.ini_options]
+asyncio_default_fixture_loop_scope="function"

{unstructured_ingest-0.3.11 → unstructured_ingest-0.3.12}/setup.py

@@ -117,6 +117,7 @@ connectors_reqs = {
     "postgres": load_requirements("requirements/connectors/postgres.in"),
     "qdrant": load_requirements("requirements/connectors/qdrant.in"),
     "reddit": load_requirements("requirements/connectors/reddit.in"),
+    "redis": load_requirements("requirements/connectors/redis.in"),
     "s3": load_requirements("requirements/connectors/s3.in"),
     "sharepoint": load_requirements("requirements/connectors/sharepoint.in"),
     "salesforce": load_requirements("requirements/connectors/salesforce.in"),

{unstructured_ingest-0.3.11 → unstructured_ingest-0.3.12}/test/integration/connectors/test_milvus.py

@@ -174,6 +174,19 @@ def test_precheck_fails_on_nonexistent_collection(collection: str):
         uploader.precheck()
 
 
+@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
+def test_precheck_fails_on_nonexisting_db(collection: str):
+    uploader = MilvusUploader(
+        connection_config=MilvusConnectionConfig(uri=DB_URI),
+        upload_config=MilvusUploaderConfig(db_name="nonexisting_db", collection_name=collection),
+    )
+    with pytest.raises(
+        DestinationConnectionError,
+        match="database not found",
+    ):
+        uploader.precheck()
+
+
 @pytest.mark.parametrize("upload_file_str", ["upload_file_ndjson", "upload_file"])
 def test_milvus_stager(
     request: TopRequest,

{unstructured_ingest-0.3.11 → unstructured_ingest-0.3.12}/test/integration/connectors/test_onedrive.py

@@ -20,6 +20,9 @@ from unstructured_ingest.v2.processes.connectors.onedrive import (
 
 
 @pytest.fixture
+@pytest.mark.xfail(
+    reason="Issues with test setup on the provider side."
+)  # TODO: remove line when issues are addressed
 def onedrive_test_folder() -> str:
     """
     Pytest fixture that creates a test folder in OneDrive and deletes it after test run.
@@ -66,6 +69,9 @@ def get_connection_config():
 
 @pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG)
 @requires_env("MS_CLIENT_CRED", "MS_CLIENT_ID", "MS_TENANT_ID", "MS_USER_PNAME")
+@pytest.mark.xfail(
+    reason="Issues with test setup on the provider side."
+)  # TODO: remove line when issues are addressed
 def test_onedrive_destination(upload_file: Path, onedrive_test_folder: str):
     """
     Integration test for the OneDrive destination connector.

unstructured_ingest-0.3.12/test/integration/connectors/test_redis.py

@@ -0,0 +1,119 @@
+import asyncio
+import json
+import os
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+import pytest
+from redis import exceptions as redis_exceptions
+from redis.asyncio import Redis, from_url
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.v2.processes.connectors.redisdb import (
+    CONNECTOR_TYPE as REDIS_CONNECTOR_TYPE,
+)
+from unstructured_ingest.v2.processes.connectors.redisdb import (
+    RedisAccessConfig,
+    RedisConnectionConfig,
+    RedisUploader,
+    RedisUploaderConfig,
+)
+
+
+async def delete_record(client: Redis, element_id: str) -> None:
+    await client.delete(element_id)
+
+
+async def validate_upload(client: Redis, first_element: dict):
+    element_id = first_element["element_id"]
+    expected_text = first_element["text"]
+    expected_embeddings = first_element["embeddings"]
+    async with client.pipeline(transaction=True) as pipe:
+        try:
+            response = await pipe.json().get(element_id, "$").execute()
+            response = response[0][0]
+        except redis_exceptions.ResponseError:
+            response = await pipe.get(element_id).execute()
+            response = json.loads(response[0])
+
+    embedding_similarity = np.linalg.norm(
+        np.array(response["embeddings"]) - np.array(expected_embeddings)
+    )
+
+    assert response is not None
+    assert response["element_id"] == element_id
+    assert response["text"] == expected_text
+    assert embedding_similarity < 1e-10
+
+
+async def redis_destination_test(
+    upload_file: Path,
+    tmp_path: Path,
+    connection_kwargs: dict,
+    uri: Optional[str] = None,
+    password: Optional[str] = None,
+):
+    uploader = RedisUploader(
+        connection_config=RedisConnectionConfig(
+            **connection_kwargs, access_config=RedisAccessConfig(uri=uri, password=password)
+        ),
+        upload_config=RedisUploaderConfig(batch_size=10),
+    )
+
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=REDIS_CONNECTOR_TYPE,
+        identifier="mock-file-data",
+    )
+    with upload_file.open() as upload_fp:
+        elements = json.load(upload_fp)
+    first_element = elements[0]
+
+    try:
+        if uploader.is_async():
+            await uploader.run_data_async(data=elements, file_data=file_data)
+
+        if uri:
+            async with from_url(uri) as client:
+                await validate_upload(client=client, first_element=first_element)
+        else:
+            async with Redis(**connection_kwargs, password=password) as client:
+                await validate_upload(client=client, first_element=first_element)
+    except Exception as e:
+        raise e
+    finally:
+        if uri:
+            async with from_url(uri) as client:
+                tasks = [delete_record(client, element["element_id"]) for element in elements]
+                await asyncio.gather(*tasks)
+        else:
+            async with Redis(**connection_kwargs, password=password) as client:
+                tasks = [delete_record(client, element["element_id"]) for element in elements]
+                await asyncio.gather(*tasks)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(REDIS_CONNECTOR_TYPE, DESTINATION_TAG)
+@requires_env("AZURE_REDIS_INGEST_TEST_PASSWORD")
+async def test_redis_destination_azure_with_password(upload_file: Path, tmp_path: Path):
+    connection_kwargs = {
+        "host": "utic-dashboard-dev.redis.cache.windows.net",
+        "port": 6380,
+        "db": 0,
+        "ssl": True,
+    }
+    redis_pw = os.environ["AZURE_REDIS_INGEST_TEST_PASSWORD"]
+    await redis_destination_test(upload_file, tmp_path, connection_kwargs, password=redis_pw)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(REDIS_CONNECTOR_TYPE, DESTINATION_TAG, "redis")
+@requires_env("AZURE_REDIS_INGEST_TEST_PASSWORD")
+async def test_redis_destination_azure_with_uri(upload_file: Path, tmp_path: Path):
+    connection_kwargs = {}
+    redis_pw = os.environ["AZURE_REDIS_INGEST_TEST_PASSWORD"]
+    uri = f"rediss://:{redis_pw}@utic-dashboard-dev.redis.cache.windows.net:6380/0"
+    await redis_destination_test(upload_file, tmp_path, connection_kwargs, uri=uri)

unstructured_ingest-0.3.12/test/integration/connectors/test_vectara.py

@@ -0,0 +1,270 @@
+import json
+import os
+import time
+from pathlib import Path
+from typing import Generator
+from uuid import uuid4
+
+import pytest
+import requests
+
+from test.integration.connectors.utils.constants import DESTINATION_TAG
+from test.integration.utils import requires_env
+from unstructured_ingest.v2.interfaces.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.v2.logger import logger
+from unstructured_ingest.v2.processes.connectors.vectara import (
+    CONNECTOR_TYPE as VECTARA_CONNECTOR_TYPE,
+)
+from unstructured_ingest.v2.processes.connectors.vectara import (
+    VectaraAccessConfig,
+    VectaraConnectionConfig,
+    VectaraUploader,
+    VectaraUploaderConfig,
+    VectaraUploadStager,
+    VectaraUploadStagerConfig,
+)
+
+
+def validate_upload(response: dict, expected_data: dict):
+    element_id = expected_data["element_id"]
+    expected_text = expected_data["text"]
+    filename = expected_data["metadata"]["filename"]
+    filetype = expected_data["metadata"]["filetype"]
+    page_number = expected_data["metadata"]["page_number"]
+
+    response = response["search_results"][0]
+
+    assert response is not None
+    assert response["text"] == expected_text
+    assert response["part_metadata"]["element_id"] == element_id
+    assert response["part_metadata"]["filename"] == filename
+    assert response["part_metadata"]["filetype"] == filetype
+    assert response["part_metadata"]["page_number"] == page_number
+
+
+@requires_env("VECTARA_OAUTH_CLIENT_ID", "VECTARA_OAUTH_SECRET", "VECTARA_CUSTOMER_ID")
+def _get_jwt_token():
+    """Connect to the server and get a JWT token."""
+    customer_id = os.environ["VECTARA_CUSTOMER_ID"]
+    token_endpoint = (
+        f"https://vectara-prod-{customer_id}.auth.us-west-2.amazoncognito.com/oauth2/token"
+    )
+    headers = {
+        "Content-Type": "application/x-www-form-urlencoded",
+    }
+    data = {
+        "grant_type": "client_credentials",
+        "client_id": os.environ["VECTARA_OAUTH_CLIENT_ID"],
+        "client_secret": os.environ["VECTARA_OAUTH_SECRET"],
+    }
+
+    response = requests.post(token_endpoint, headers=headers, data=data)
+    response.raise_for_status()
+    response_json = response.json()
+
+    return response_json.get("access_token")
+
+
+def query_data(corpus_key: str, element_id: str) -> dict:
+
+    url = f"https://api.vectara.io/v2/corpora/{corpus_key}/query"
+
+    # the query below requires the corpus to have filter attributes for element_id
+
+    data = json.dumps(
+        {
+            "query": "string",
+            "search": {
+                "metadata_filter": f"part.element_id = '{element_id}'",
+                "lexical_interpolation": 1,
+                "limit": 10,
+            },
+        }
+    )
+
+    jwt_token = _get_jwt_token()
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        "Authorization": f"Bearer {jwt_token}",
+        "X-source": "unstructured",
+    }
+
+    response = requests.post(url, headers=headers, data=data)
+    response.raise_for_status()
+    response_json = response.json()
+
+    return response_json
+
+
+def create_corpora(corpus_key: str, corpus_name: str) -> None:
+    url = "https://api.vectara.io/v2/corpora"
+    data = json.dumps({"key": corpus_key, "name": corpus_name, "description": "integration test"})
+    jwt_token = _get_jwt_token()
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        "Authorization": f"Bearer {jwt_token}",
+        "X-source": "unstructured",
+    }
+
+    response = requests.post(url, headers=headers, data=data)
+    response.raise_for_status()
+
+
+def replace_filter_attributes(corpus_key: str) -> None:
+    url = f"https://api.vectara.io/v2/corpora/{corpus_key}/replace_filter_attributes"
+    data = json.dumps(
+        {
+            "filter_attributes": [
+                {"name": "element_id", "level": "part", "indexed": True, "type": "text"}
+            ]
+        }
+    )
+    jwt_token = _get_jwt_token()
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        "Authorization": f"Bearer {jwt_token}",
+        "X-source": "unstructured",
+    }
+
+    response = requests.post(url, headers=headers, data=data)
+    response.raise_for_status()
+
+
+def delete_corpora(corpus_key: str) -> None:
+    url = f"https://api.vectara.io/v2/corpora/{corpus_key}"
+
+    jwt_token = _get_jwt_token()
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        "Authorization": f"Bearer {jwt_token}",
+        "X-source": "unstructured",
+    }
+
+    response = requests.delete(url, headers=headers)
+    response.raise_for_status()
+
+
+def list_corpora() -> list:
+    url = "https://api.vectara.io/v2/corpora?limit=100"
+    jwt_token = _get_jwt_token()
+    headers = {
+        "Content-Type": "application/json",
+        "Accept": "application/json",
+        "Authorization": f"Bearer {jwt_token}",
+        "X-source": "unstructured",
+    }
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+    response_json = response.json()
+    if response_json.get("corpora"):
+        return [item["key"] for item in response_json.get("corpora")]
+    else:
+        return []
+
+
+def wait_for_ready(corpus_key: str, timeout=60, interval=2) -> None:
+    def is_ready_status():
+        corpora_list = list_corpora()
+        return corpus_key in corpora_list
+
+    start = time.time()
+    is_ready = is_ready_status()
+    while not is_ready and time.time() - start < timeout:
+        time.sleep(interval)
+        is_ready = is_ready_status()
+    if not is_ready:
+        raise TimeoutError("time out waiting for corpus to be ready")
+
+
+def wait_for_delete(corpus_key: str, timeout=60, interval=2) -> None:
+    start = time.time()
+    while time.time() - start < timeout:
+        corpora_list = list_corpora()
+        if corpus_key not in corpora_list:
+            return
+        time.sleep(interval)
+
+    raise TimeoutError("time out waiting for corpus to delete")
+
+
+@pytest.fixture
+def corpora_util() -> Generator[str, None, None]:
+    random_id = str(uuid4()).split("-")[0]
+    corpus_key = f"ingest-test-{random_id}"
+    corpus_name = "ingest-test"
+    logger.info(f"Creating corpus with key: {corpus_key}")
+    try:
+        create_corpora(corpus_key, corpus_name)
+        replace_filter_attributes(corpus_key)
+        wait_for_ready(corpus_key=corpus_key)
+        yield corpus_key
+    except Exception as e:
+        logger.error(f"failed to create corpus {corpus_key}: {e}")
+    finally:
+        logger.info(f"deleting corpus: {corpus_key}")
+        delete_corpora(corpus_key)
+        wait_for_delete(corpus_key=corpus_key)
+
+
+@pytest.mark.asyncio
+@pytest.mark.tags(VECTARA_CONNECTOR_TYPE, DESTINATION_TAG, "vectara")
+@requires_env("VECTARA_OAUTH_CLIENT_ID", "VECTARA_OAUTH_SECRET", "VECTARA_CUSTOMER_ID")
+async def test_vectara_destination(
+    upload_file: Path, tmp_path: Path, corpora_util: str, retries=30, interval=10
+):
+    corpus_key = corpora_util
+    connection_kwargs = {
+        "customer_id": os.environ["VECTARA_CUSTOMER_ID"],
+        "corpus_key": corpus_key,
+    }
+
+    oauth_client_id = os.environ["VECTARA_OAUTH_CLIENT_ID"]
+    oauth_secret = os.environ["VECTARA_OAUTH_SECRET"]
+
+    file_data = FileData(
+        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
+        connector_type=VECTARA_CONNECTOR_TYPE,
+        identifier="mock-file-data",
+    )
+
+    stager_config = VectaraUploadStagerConfig(batch_size=10)
+    stager = VectaraUploadStager(upload_stager_config=stager_config)
+    new_upload_file = stager.run(
+        elements_filepath=upload_file,
+        output_dir=tmp_path,
+        output_filename=upload_file.name,
+        file_data=file_data,
+    )
+
+    uploader = VectaraUploader(
+        connection_config=VectaraConnectionConfig(
+            **connection_kwargs,
+            access_config=VectaraAccessConfig(
+                oauth_client_id=oauth_client_id, oauth_secret=oauth_secret
+            ),
+        ),
+        upload_config=VectaraUploaderConfig(),
+    )
+
+    with new_upload_file.open() as new_upload_fp:
+        elements_stager = json.load(new_upload_fp)
+
+    if uploader.is_async():
+        await uploader.run_data_async(data=elements_stager, file_data=file_data)
+
+    with upload_file.open() as upload_fp:
+        elements = json.load(upload_fp)
+    first_element = elements[0]
+
+    for i in range(retries):
+        response = query_data(corpus_key, first_element["element_id"])
+        if not response["search_results"]:
+            time.sleep(interval)
+        else:
+            break
+
+    validate_upload(response=response, expected_data=first_element)

{unstructured_ingest-0.3.11 → unstructured_ingest-0.3.12}/test/integration/embedders/test_bedrock.py

@@ -2,9 +2,12 @@ import json
 import os
 from pathlib import Path
 
+import pytest
+
 from test.integration.embedders.utils import validate_embedding_output, validate_raw_embedder
 from test.integration.utils import requires_env
 from unstructured_ingest.embed.bedrock import BedrockEmbeddingConfig, BedrockEmbeddingEncoder
+from unstructured_ingest.v2.errors import UserAuthError, UserError
 from unstructured_ingest.v2.processes.embedder import Embedder, EmbedderConfig
 
 
@@ -47,3 +50,28 @@ def test_raw_bedrock_embedder(embedder_file: Path):
         expected_dimensions=(1536,),
         expected_is_unit_vector=False,
     )
+
+
+def test_raw_bedrock_embedder_invalid_credentials(embedder_file: Path):
+    embedder = BedrockEmbeddingEncoder(
+        config=BedrockEmbeddingConfig(
+            aws_access_key_id="no_key",
+            aws_secret_access_key="no_secret",
+        )
+    )
+    with pytest.raises(UserAuthError):
+        embedder.get_exemplary_embedding()
+
+
+@requires_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY")
+def test_raw_bedrock_embedder_invalid_model(embedder_file: Path):
+    aws_credentials = get_aws_credentials()
+    embedder = BedrockEmbeddingEncoder(
+        config=BedrockEmbeddingConfig(
+            aws_access_key_id=aws_credentials["aws_access_key_id"],
+            aws_secret_access_key=aws_credentials["aws_secret_access_key"],
+            model_name="invalid_model",
+        )
+    )
+    with pytest.raises(UserError):
+        embedder.get_exemplary_embedding()

{unstructured_ingest-0.3.11 → unstructured_ingest-0.3.12}/test/integration/embedders/test_octoai.py

@@ -2,9 +2,12 @@ import json
 import os
 from pathlib import Path
 
+import pytest
+
 from test.integration.embedders.utils import validate_embedding_output, validate_raw_embedder
 from test.integration.utils import requires_env
 from unstructured_ingest.embed.octoai import OctoAiEmbeddingConfig, OctoAIEmbeddingEncoder
+from unstructured_ingest.v2.errors import UserAuthError
 from unstructured_ingest.v2.processes.embedder import Embedder, EmbedderConfig
 
 API_KEY = "OCTOAI_API_KEY"
@@ -39,3 +42,14 @@ def test_raw_octoai_embedder(embedder_file: Path):
     validate_raw_embedder(
         embedder=embedder, embedder_file=embedder_file, expected_dimensions=(1024,)
     )
+
+
+@pytest.mark.skip(reason="Unexpected connection error at the moment")
+def test_raw_octoai_embedder_invalid_credentials():
+    embedder = OctoAIEmbeddingEncoder(
+        config=OctoAiEmbeddingConfig(
+            api_key="fake_api_key",
+        )
+    )
+    with pytest.raises(UserAuthError):
+        embedder.get_exemplary_embedding()

{unstructured_ingest-0.3.11 → unstructured_ingest-0.3.12}/test/integration/embedders/test_openai.py

@@ -2,9 +2,12 @@ import json
 import os
 from pathlib import Path
 
+import pytest
+
 from test.integration.embedders.utils import validate_embedding_output, validate_raw_embedder
 from test.integration.utils import requires_env
 from unstructured_ingest.embed.openai import OpenAIEmbeddingConfig, OpenAIEmbeddingEncoder
+from unstructured_ingest.v2.errors import UserAuthError
 from unstructured_ingest.v2.processes.embedder import Embedder, EmbedderConfig
 
 API_KEY = "OPENAI_API_KEY"
@@ -39,3 +42,13 @@ def test_raw_openai_embedder(embedder_file: Path):
     validate_raw_embedder(
         embedder=embedder, embedder_file=embedder_file, expected_dimensions=(1536,)
     )
+
+
+def test_raw_openai_embedder_invalid_credentials():
+    embedder = OpenAIEmbeddingEncoder(
+        config=OpenAIEmbeddingConfig(
+            api_key="fake_api_key",
+        )
+    )
+    with pytest.raises(UserAuthError):
+        embedder.get_exemplary_embedding()