unstructured-ingest 0.7.2__py3-none-any.whl → 1.0.2__py3-none-any.whl

test/integration/connectors/weaviate/test_local.py

@@ -1,152 +0,0 @@
-import json
-import time
-from pathlib import Path
-
-import pytest
-import requests
-import weaviate
-from weaviate.client import WeaviateClient
-
-from test.integration.connectors.utils.constants import DESTINATION_TAG, VECTOR_DB_TAG
-from test.integration.connectors.utils.docker import container_context
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.processes.connectors.weaviate.local import (
-    CONNECTOR_TYPE,
-    LocalWeaviateConnectionConfig,
-    LocalWeaviateUploader,
-    LocalWeaviateUploaderConfig,
-    LocalWeaviateUploadStager,
-)
-
-COLLECTION_NAME = "elements"
-
-
-def wait_for_container(timeout: int = 10, interval: int = 1) -> None:
-    start_time = time.time()
-    while time.time() - start_time < timeout:
-        try:
-            requests.get("http://localhost:8080/v1/.well-known/read", timeout=1)
-            return
-        except Exception as e:
-            print(f"Failed to validate container healthy, sleeping for {interval} seconds: {e}")
-            time.sleep(interval)
-    raise TimeoutError("Docker container never came up healthy")
-
-
-@pytest.fixture
-def weaviate_instance():
-    with container_context(
-        image="semitechnologies/weaviate:1.27.3",
-        ports={8080: 8080, 50051: 50051},
-    ) as ctx:
-        wait_for_container()
-        yield ctx
-
-
-@pytest.fixture
-def collection(weaviate_instance, collections_schema_config: dict) -> str:
-    with weaviate.connect_to_local() as weaviate_client:
-        weaviate_client.collections.create_from_dict(config=collections_schema_config)
-    return COLLECTION_NAME
-
-
-def get_count(client: WeaviateClient) -> int:
-    collection = client.collections.get(COLLECTION_NAME)
-    resp = collection.aggregate.over_all(total_count=True)
-    return resp.total_count
-
-
-def validate_count(expected_count: int, retries: int = 10, interval: int = 1) -> None:
-    with weaviate.connect_to_local() as weaviate_client:
-        current_count = get_count(client=weaviate_client)
-        retry_count = 0
-        while current_count != expected_count and retry_count < retries:
-            retry_count += 1
-            time.sleep(interval)
-            current_count = get_count(client=weaviate_client)
-        assert current_count == expected_count, (
-            f"Expected count ({expected_count}) doesn't match how "
-            f"much came back from collection: {current_count}"
-        )
-
-
-def run_uploader_and_validate(
-    uploader: LocalWeaviateUploader, path: Path, file_data: FileData, expected_count: int
-):
-    uploader.precheck()
-    uploader.run(path=path, file_data=file_data)
-    validate_count(expected_count=expected_count)
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
-def test_weaviate_local_destination(upload_file: Path, collection: str, tmp_path: Path):
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(fullpath=upload_file.name, filename=upload_file.name),
-        connector_type=CONNECTOR_TYPE,
-        identifier="mock file data",
-    )
-    stager = LocalWeaviateUploadStager()
-
-    staged_filepath = stager.run(
-        elements_filepath=upload_file,
-        file_data=file_data,
-        output_dir=tmp_path,
-        output_filename=upload_file.name,
-    )
-    dynamic_uploader = LocalWeaviateUploader(
-        upload_config=LocalWeaviateUploaderConfig(
-            collection=COLLECTION_NAME,
-        ),
-        connection_config=LocalWeaviateConnectionConfig(),
-    )
-    fixed_size_uploader = LocalWeaviateUploader(
-        upload_config=LocalWeaviateUploaderConfig(
-            collection=COLLECTION_NAME, batch_size=10, dynamic_batch=False
-        ),
-        connection_config=LocalWeaviateConnectionConfig(),
-    )
-    rate_limited_uploader = LocalWeaviateUploader(
-        upload_config=LocalWeaviateUploaderConfig(
-            collection=COLLECTION_NAME, requests_per_minute=50, dynamic_batch=False
-        ),
-        connection_config=LocalWeaviateConnectionConfig(),
-    )
-    with staged_filepath.open() as f:
-        staged_elements = json.load(f)
-    expected_count = len(staged_elements)
-
-    run_uploader_and_validate(
-        uploader=dynamic_uploader,
-        path=staged_filepath,
-        file_data=file_data,
-        expected_count=expected_count,
-    )
-    run_uploader_and_validate(
-        uploader=fixed_size_uploader,
-        path=staged_filepath,
-        file_data=file_data,
-        expected_count=expected_count,
-    )
-    run_uploader_and_validate(
-        uploader=rate_limited_uploader,
-        path=staged_filepath,
-        file_data=file_data,
-        expected_count=expected_count,
-    )
-
-
-@pytest.mark.tags(CONNECTOR_TYPE, DESTINATION_TAG, VECTOR_DB_TAG)
-def test_weaviate_local_create_destination(weaviate_instance):
-    uploader = LocalWeaviateUploader(
-        upload_config=LocalWeaviateUploaderConfig(),
-        connection_config=LocalWeaviateConnectionConfig(),
-    )
-    collection_name = "system_created-123"
-    formatted_collection_name = "System_created_123"
-    created = uploader.create_destination(destination_name=collection_name)
-    assert created
-    with uploader.connection_config.get_client() as weaviate_client:
-        assert weaviate_client.collections.exists(name=formatted_collection_name)
-
-    created = uploader.create_destination(destination_name=collection_name)
-    assert not created

test/integration/embedders/conftest.py

@@ -1,13 +0,0 @@
-from pathlib import Path
-
-import pytest
-
-
-@pytest.fixture
-def embedder_file() -> Path:
-    int_test_dir = Path(__file__).parent
-    assets_dir = int_test_dir / "assets"
-    embedder_file = assets_dir / "DA-1p-with-duplicate-pages.pdf.json"
-    assert embedder_file.exists()
-    assert embedder_file.is_file()
-    return embedder_file

test/integration/embedders/test_azure_openai.py

@@ -1,57 +0,0 @@
-import json
-import os
-from dataclasses import dataclass
-from pathlib import Path
-
-from test.integration.embedders.utils import validate_embedding_output, validate_raw_embedder
-from test.integration.utils import requires_env
-from unstructured_ingest.embed.azure_openai import (
-    AzureOpenAIEmbeddingConfig,
-    AzureOpenAIEmbeddingEncoder,
-)
-from unstructured_ingest.processes.embedder import Embedder, EmbedderConfig
-
-API_KEY = "AZURE_OPENAI_API_KEY"
-ENDPOINT = "AZURE_OPENAI_ENDPOINT"
-
-
-@dataclass(frozen=True)
-class AzureData:
-    api_key: str
-    endpoint: str
-
-
-def get_azure_data() -> AzureData:
-    api_key = os.getenv(API_KEY, None)
-    assert api_key
-    endpoint = os.getenv(ENDPOINT, None)
-    assert endpoint
-    return AzureData(api_key, endpoint)
-
-
-@requires_env(API_KEY, ENDPOINT)
-def test_azure_openai_embedder(embedder_file: Path):
-    azure_data = get_azure_data()
-    embedder_config = EmbedderConfig(
-        embedding_provider="azure-openai",
-        embedding_api_key=azure_data.api_key,
-        embedding_azure_endpoint=azure_data.endpoint,
-    )
-    embedder = Embedder(config=embedder_config)
-    results = embedder.run(elements_filepath=embedder_file)
-    assert results
-    with embedder_file.open("r") as f:
-        original_elements = json.load(f)
-    validate_embedding_output(original_elements=original_elements, output_elements=results)
-
-
-@requires_env(API_KEY, ENDPOINT)
-def test_raw_azure_openai_embedder(embedder_file: Path):
-    azure_data = get_azure_data()
-    embedder = AzureOpenAIEmbeddingEncoder(
-        config=AzureOpenAIEmbeddingConfig(
-            api_key=azure_data.api_key,
-            azure_endpoint=azure_data.endpoint,
-        )
-    )
-    validate_raw_embedder(embedder=embedder, embedder_file=embedder_file, expected_dimension=1536)

test/integration/embedders/test_bedrock.py

@@ -1,103 +0,0 @@
-import json
-import os
-from pathlib import Path
-
-import pytest
-
-from test.integration.embedders.utils import (
-    validate_embedding_output,
-    validate_raw_embedder,
-    validate_raw_embedder_async,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.embed.bedrock import (
-    AsyncBedrockEmbeddingEncoder,
-    BedrockEmbeddingConfig,
-    BedrockEmbeddingEncoder,
-)
-from unstructured_ingest.errors_v2 import UserAuthError, UserError
-from unstructured_ingest.processes.embedder import Embedder, EmbedderConfig
-
-
-def get_aws_credentials() -> dict:
-    access_key = os.getenv("AWS_ACCESS_KEY_ID", None)
-    assert access_key
-    secret_key = os.getenv("AWS_SECRET_ACCESS_KEY", None)
-    assert secret_key
-    return {"aws_access_key_id": access_key, "aws_secret_access_key": secret_key}
-
-
-@requires_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY")
-def test_bedrock_embedder(embedder_file: Path):
-    aws_credentials = get_aws_credentials()
-    embedder_config = EmbedderConfig(
-        embedding_provider="bedrock",
-        embedding_aws_access_key_id=aws_credentials["aws_access_key_id"],
-        embedding_aws_secret_access_key=aws_credentials["aws_secret_access_key"],
-    )
-    embedder = Embedder(config=embedder_config)
-    results = embedder.run(elements_filepath=embedder_file)
-    assert results
-    with embedder_file.open("r") as f:
-        original_elements = json.load(f)
-    validate_embedding_output(original_elements=original_elements, output_elements=results)
-
-
-@requires_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY")
-def test_raw_bedrock_embedder(embedder_file: Path):
-    aws_credentials = get_aws_credentials()
-    embedder = BedrockEmbeddingEncoder(
-        config=BedrockEmbeddingConfig(
-            aws_access_key_id=aws_credentials["aws_access_key_id"],
-            aws_secret_access_key=aws_credentials["aws_secret_access_key"],
-        )
-    )
-    validate_raw_embedder(
-        embedder=embedder,
-        embedder_file=embedder_file,
-        expected_dimension=1536,
-        expected_is_unit_vector=False,
-    )
-
-
-def test_raw_bedrock_embedder_invalid_credentials(embedder_file: Path):
-    embedder = BedrockEmbeddingEncoder(
-        config=BedrockEmbeddingConfig(
-            aws_access_key_id="no_key",
-            aws_secret_access_key="no_secret",
-        )
-    )
-    with pytest.raises(UserAuthError):
-        embedder.get_exemplary_embedding()
-
-
-@requires_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY")
-def test_raw_bedrock_embedder_invalid_model(embedder_file: Path):
-    aws_credentials = get_aws_credentials()
-    embedder = BedrockEmbeddingEncoder(
-        config=BedrockEmbeddingConfig(
-            aws_access_key_id=aws_credentials["aws_access_key_id"],
-            aws_secret_access_key=aws_credentials["aws_secret_access_key"],
-            model_name="invalid_model",
-        )
-    )
-    with pytest.raises(UserError):
-        embedder.get_exemplary_embedding()
-
-
-@requires_env("AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY")
-@pytest.mark.asyncio
-async def test_raw_async_bedrock_embedder(embedder_file: Path):
-    aws_credentials = get_aws_credentials()
-    embedder = AsyncBedrockEmbeddingEncoder(
-        config=BedrockEmbeddingConfig(
-            aws_access_key_id=aws_credentials["aws_access_key_id"],
-            aws_secret_access_key=aws_credentials["aws_secret_access_key"],
-        )
-    )
-    await validate_raw_embedder_async(
-        embedder=embedder,
-        embedder_file=embedder_file,
-        expected_dimension=1536,
-        expected_is_unit_vector=False,
-    )

test/integration/embedders/test_huggingface.py

@@ -1,24 +0,0 @@
-import json
-from pathlib import Path
-
-from test.integration.embedders.utils import validate_embedding_output, validate_raw_embedder
-from unstructured_ingest.embed.huggingface import (
-    HuggingFaceEmbeddingConfig,
-    HuggingFaceEmbeddingEncoder,
-)
-from unstructured_ingest.processes.embedder import Embedder, EmbedderConfig
-
-
-def test_huggingface_embedder(embedder_file: Path):
-    embedder_config = EmbedderConfig(embedding_provider="huggingface")
-    embedder = Embedder(config=embedder_config)
-    results = embedder.run(elements_filepath=embedder_file)
-    assert results
-    with embedder_file.open("r") as f:
-        original_elements = json.load(f)
-    validate_embedding_output(original_elements=original_elements, output_elements=results)
-
-
-def test_raw_hugginface_embedder(embedder_file: Path):
-    embedder = HuggingFaceEmbeddingEncoder(config=HuggingFaceEmbeddingConfig())
-    validate_raw_embedder(embedder=embedder, embedder_file=embedder_file, expected_dimension=384)

test/integration/embedders/test_mixedbread.py

@@ -1,71 +0,0 @@
-import json
-import os
-from pathlib import Path
-
-import pytest
-
-from test.integration.embedders.utils import (
-    validate_embedding_output,
-    validate_raw_embedder,
-    validate_raw_embedder_async,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.embed.mixedbreadai import (
-    AsyncMixedbreadAIEmbeddingEncoder,
-    MixedbreadAIEmbeddingConfig,
-    MixedbreadAIEmbeddingEncoder,
-)
-from unstructured_ingest.processes.embedder import Embedder, EmbedderConfig
-
-API_KEY = "MXBAI_API_KEY"
-
-
-def get_api_key() -> str:
-    api_key = os.getenv(API_KEY, None)
-    assert api_key
-    return api_key
-
-
-@requires_env(API_KEY)
-def test_mixedbread_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder_config = EmbedderConfig(embedding_provider="mixedbread-ai", embedding_api_key=api_key)
-    embedder = Embedder(config=embedder_config)
-    results = embedder.run(elements_filepath=embedder_file)
-    assert results
-    with embedder_file.open("r") as f:
-        original_elements = json.load(f)
-    validate_embedding_output(original_elements=original_elements, output_elements=results)
-
-
-@requires_env(API_KEY)
-def test_raw_mixedbread_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder = MixedbreadAIEmbeddingEncoder(
-        config=MixedbreadAIEmbeddingConfig(
-            api_key=api_key,
-        )
-    )
-    validate_raw_embedder(
-        embedder=embedder,
-        embedder_file=embedder_file,
-        expected_dimension=1024,
-        expected_is_unit_vector=True,
-    )
-
-
-@requires_env(API_KEY)
-@pytest.mark.asyncio
-async def test_raw_async_mixedbread_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder = AsyncMixedbreadAIEmbeddingEncoder(
-        config=MixedbreadAIEmbeddingConfig(
-            api_key=api_key,
-        )
-    )
-    await validate_raw_embedder_async(
-        embedder=embedder,
-        embedder_file=embedder_file,
-        expected_dimension=1024,
-        expected_is_unit_vector=True,
-    )

test/integration/embedders/test_octoai.py

@@ -1,75 +0,0 @@
-import json
-import os
-from pathlib import Path
-
-import pytest
-
-from test.integration.embedders.utils import (
-    validate_embedding_output,
-    validate_raw_embedder,
-    validate_raw_embedder_async,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.embed.octoai import (
-    AsyncOctoAIEmbeddingEncoder,
-    OctoAiEmbeddingConfig,
-    OctoAIEmbeddingEncoder,
-)
-from unstructured_ingest.errors_v2 import UserAuthError
-from unstructured_ingest.processes.embedder import Embedder, EmbedderConfig
-
-API_KEY = "OCTOAI_API_KEY"
-
-
-def get_api_key() -> str:
-    api_key = os.getenv(API_KEY, None)
-    assert api_key
-    return api_key
-
-
-@requires_env(API_KEY)
-def test_octoai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder_config = EmbedderConfig(embedding_provider="octoai", embedding_api_key=api_key)
-    embedder = Embedder(config=embedder_config)
-    results = embedder.run(elements_filepath=embedder_file)
-    assert results
-    with embedder_file.open("r") as f:
-        original_elements = json.load(f)
-    validate_embedding_output(original_elements=original_elements, output_elements=results)
-
-
-@requires_env(API_KEY)
-def test_raw_octoai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder = OctoAIEmbeddingEncoder(
-        config=OctoAiEmbeddingConfig(
-            api_key=api_key,
-        )
-    )
-    validate_raw_embedder(embedder=embedder, embedder_file=embedder_file, expected_dimension=1024)
-
-
-@pytest.mark.skip(reason="Unexpected connection error at the moment")
-def test_raw_octoai_embedder_invalid_credentials():
-    embedder = OctoAIEmbeddingEncoder(
-        config=OctoAiEmbeddingConfig(
-            api_key="fake_api_key",
-        )
-    )
-    with pytest.raises(UserAuthError):
-        embedder.get_exemplary_embedding()
-
-
-@requires_env(API_KEY)
-@pytest.mark.asyncio
-async def test_raw_async_octoai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder = AsyncOctoAIEmbeddingEncoder(
-        config=OctoAiEmbeddingConfig(
-            api_key=api_key,
-        )
-    )
-    await validate_raw_embedder_async(
-        embedder=embedder, embedder_file=embedder_file, expected_dimension=1024
-    )

test/integration/embedders/test_openai.py

@@ -1,74 +0,0 @@
-import json
-import os
-from pathlib import Path
-
-import pytest
-
-from test.integration.embedders.utils import (
-    validate_embedding_output,
-    validate_raw_embedder,
-    validate_raw_embedder_async,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.embed.openai import (
-    AsyncOpenAIEmbeddingEncoder,
-    OpenAIEmbeddingConfig,
-    OpenAIEmbeddingEncoder,
-)
-from unstructured_ingest.errors_v2 import UserAuthError
-from unstructured_ingest.processes.embedder import Embedder, EmbedderConfig
-
-API_KEY = "OPENAI_API_KEY"
-
-
-def get_api_key() -> str:
-    api_key = os.getenv(API_KEY, None)
-    assert api_key
-    return api_key
-
-
-@requires_env(API_KEY)
-def test_openai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder_config = EmbedderConfig(embedding_provider="openai", embedding_api_key=api_key)
-    embedder = Embedder(config=embedder_config)
-    results = embedder.run(elements_filepath=embedder_file)
-    assert results
-    with embedder_file.open("r") as f:
-        original_elements = json.load(f)
-    validate_embedding_output(original_elements=original_elements, output_elements=results)
-
-
-@requires_env(API_KEY)
-def test_raw_openai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder = OpenAIEmbeddingEncoder(
-        config=OpenAIEmbeddingConfig(
-            api_key=api_key,
-        )
-    )
-    validate_raw_embedder(embedder=embedder, embedder_file=embedder_file, expected_dimension=1536)
-
-
-def test_raw_openai_embedder_invalid_credentials():
-    embedder = OpenAIEmbeddingEncoder(
-        config=OpenAIEmbeddingConfig(
-            api_key="fake_api_key",
-        )
-    )
-    with pytest.raises(UserAuthError):
-        embedder.get_exemplary_embedding()
-
-
-@requires_env(API_KEY)
-@pytest.mark.asyncio
-async def test_raw_async_openai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder = AsyncOpenAIEmbeddingEncoder(
-        config=OpenAIEmbeddingConfig(
-            api_key=api_key,
-        )
-    )
-    await validate_raw_embedder_async(
-        embedder=embedder, embedder_file=embedder_file, expected_dimension=1536
-    )

test/integration/embedders/test_togetherai.py

@@ -1,71 +0,0 @@
-import json
-import os
-from pathlib import Path
-
-import pytest
-
-from test.integration.embedders.utils import (
-    validate_embedding_output,
-    validate_raw_embedder,
-    validate_raw_embedder_async,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.embed.togetherai import (
-    AsyncTogetherAIEmbeddingEncoder,
-    TogetherAIEmbeddingConfig,
-    TogetherAIEmbeddingEncoder,
-)
-from unstructured_ingest.errors_v2 import UserAuthError
-from unstructured_ingest.processes.embedder import Embedder, EmbedderConfig
-
-API_KEY = "TOGETHERAI_API_KEY"
-
-
-def get_api_key() -> str:
-    api_key = os.getenv(API_KEY, None)
-    assert api_key
-    return api_key
-
-
-@requires_env(API_KEY)
-def test_togetherai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder_config = EmbedderConfig(embedding_provider="togetherai", embedding_api_key=api_key)
-    embedder = Embedder(config=embedder_config)
-    results = embedder.run(elements_filepath=embedder_file)
-    assert results
-    with embedder_file.open("r") as f:
-        original_elements = json.load(f)
-    validate_embedding_output(original_elements=original_elements, output_elements=results)
-
-
-@requires_env(API_KEY)
-def test_raw_togetherai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder = TogetherAIEmbeddingEncoder(config=TogetherAIEmbeddingConfig(api_key=api_key))
-    validate_raw_embedder(
-        embedder=embedder,
-        embedder_file=embedder_file,
-        expected_dimension=768,
-        expected_is_unit_vector=False,
-    )
-
-
-def test_raw_togetherai_embedder_invalid_credentials():
-    embedder = TogetherAIEmbeddingEncoder(config=TogetherAIEmbeddingConfig(api_key="fake_api_key"))
-
-    with pytest.raises(UserAuthError):
-        embedder.get_exemplary_embedding()
-
-
-@requires_env(API_KEY)
-@pytest.mark.asyncio
-async def test_raw_async_togetherai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder = AsyncTogetherAIEmbeddingEncoder(config=TogetherAIEmbeddingConfig(api_key=api_key))
-    await validate_raw_embedder_async(
-        embedder=embedder,
-        embedder_file=embedder_file,
-        expected_dimension=768,
-        expected_is_unit_vector=False,
-    )