unstructured-ingest 0.7.2__py3-none-any.whl → 1.0.2__py3-none-any.whl

test/integration/embedders/test_vertexai.py

@@ -1,63 +0,0 @@
-import json
-import os
-from pathlib import Path
-
-import pytest
-
-from test.integration.embedders.utils import (
-    validate_embedding_output,
-    validate_raw_embedder,
-    validate_raw_embedder_async,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.embed.vertexai import (
-    AsyncVertexAIEmbeddingEncoder,
-    VertexAIEmbeddingConfig,
-    VertexAIEmbeddingEncoder,
-)
-from unstructured_ingest.processes.embedder import Embedder, EmbedderConfig
-
-API_KEY = "VERTEXAI_API_KEY"
-
-
-def get_api_key() -> str:
-    api_key = os.getenv(API_KEY, None)
-    assert api_key
-    return api_key
-
-
-@requires_env(API_KEY)
-def test_vertexai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder_config = EmbedderConfig(embedding_provider="vertexai", embedding_api_key=api_key)
-    embedder = Embedder(config=embedder_config)
-    results = embedder.run(elements_filepath=embedder_file)
-    assert results
-    with embedder_file.open("r") as f:
-        original_elements = json.load(f)
-    validate_embedding_output(original_elements=original_elements, output_elements=results)
-
-
-@requires_env(API_KEY)
-def test_raw_vertexai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder = VertexAIEmbeddingEncoder(
-        config=VertexAIEmbeddingConfig(
-            api_key=api_key,
-        )
-    )
-    validate_raw_embedder(embedder=embedder, embedder_file=embedder_file, expected_dimension=768)
-
-
-@requires_env(API_KEY)
-@pytest.mark.asyncio
-async def test_raw_async_vertexai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder = AsyncVertexAIEmbeddingEncoder(
-        config=VertexAIEmbeddingConfig(
-            api_key=api_key,
-        )
-    )
-    await validate_raw_embedder_async(
-        embedder=embedder, embedder_file=embedder_file, expected_dimension=768
-    )

test/integration/embedders/test_voyageai.py

@@ -1,79 +0,0 @@
-import json
-import os
-from pathlib import Path
-
-import pytest
-
-from test.integration.embedders.utils import (
-    validate_embedding_output,
-    validate_raw_embedder,
-    validate_raw_embedder_async,
-)
-from test.integration.utils import requires_env
-from unstructured_ingest.embed.voyageai import (
-    AsyncVoyageAIEmbeddingEncoder,
-    VoyageAIEmbeddingConfig,
-    VoyageAIEmbeddingEncoder,
-)
-from unstructured_ingest.processes.embedder import Embedder, EmbedderConfig
-
-API_KEY = "VOYAGEAI_API_KEY"
-
-
-def get_api_key() -> str:
-    api_key = os.getenv(API_KEY, None)
-    assert api_key
-    return api_key
-
-
-@requires_env(API_KEY)
-def test_voyageai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder_config = EmbedderConfig(embedding_provider="voyageai", embedding_api_key=api_key)
-    embedder = Embedder(config=embedder_config)
-    results = embedder.run(elements_filepath=embedder_file)
-    assert results
-    with embedder_file.open("r") as f:
-        original_elements = json.load(f)
-    validate_embedding_output(original_elements=original_elements, output_elements=results)
-
-
-@requires_env(API_KEY)
-def test_raw_voyageai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder = VoyageAIEmbeddingEncoder(
-        config=VoyageAIEmbeddingConfig(
-            api_key=api_key,
-        )
-    )
-    validate_raw_embedder(embedder=embedder, embedder_file=embedder_file, expected_dimension=1024)
-
-
-@requires_env(API_KEY)
-@pytest.mark.asyncio
-async def test_raw_async_voyageai_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder = AsyncVoyageAIEmbeddingEncoder(
-        config=VoyageAIEmbeddingConfig(
-            api_key=api_key,
-        )
-    )
-    await validate_raw_embedder_async(
-        embedder=embedder, embedder_file=embedder_file, expected_dimension=1024
-    )
-
-
-@requires_env(API_KEY)
-def test_voyageai_multimodal_embedder(embedder_file: Path):
-    api_key = get_api_key()
-    embedder_config = EmbedderConfig(
-        embedding_provider="voyageai",
-        embedding_api_key=api_key,
-        embedding_model_name="voyage-multimodal-3",
-    )
-    embedder = Embedder(config=embedder_config)
-    results = embedder.run(elements_filepath=embedder_file)
-    assert results
-    with embedder_file.open("r") as f:
-        original_elements = json.load(f)
-    validate_embedding_output(original_elements=original_elements, output_elements=results)

test/integration/embedders/utils.py

@@ -1,66 +0,0 @@
-import json
-from pathlib import Path
-from typing import Optional
-
-from unstructured_ingest.embed.interfaces import AsyncBaseEmbeddingEncoder, BaseEmbeddingEncoder
-
-
-def validate_embedding_output(original_elements: list[dict], output_elements: list[dict]):
-    """
-    Make sure the following characteristics are met:
-    * The same number of elements are returned
-    * For each element that had text, an embeddings entry was added in the output
-    * Other than the embedding, nothing about the element was changed
-    """
-    assert len(original_elements) == len(output_elements)
-    for original_element, output_element in zip(original_elements, output_elements):
-        if original_element.get("text"):
-            assert output_element.get("embeddings", None)
-        output_element.pop("embeddings", None)
-        assert original_element == output_element
-
-
-def validate_raw_embedder(
-    embedder: BaseEmbeddingEncoder,
-    embedder_file: Path,
-    expected_dimension: Optional[int] = None,
-    expected_is_unit_vector: bool = True,
-):
-    with open(embedder_file) as f:
-        elements = json.load(f)
-    all_text = [element["text"] for element in elements]
-    single_text = all_text[0]
-    dimension = embedder.dimension
-    if expected_dimension:
-        assert (
-            dimension == expected_dimension
-        ), f"dimensions {dimension} didn't match expected: {expected_dimension}"
-    is_unit_vector = embedder.is_unit_vector
-    assert is_unit_vector == expected_is_unit_vector
-    single_embedding = embedder.embed_query(query=single_text)
-    assert len(single_embedding) == dimension
-    embedded_elements = embedder.embed_documents(elements=elements)
-    validate_embedding_output(original_elements=elements, output_elements=embedded_elements)
-
-
-async def validate_raw_embedder_async(
-    embedder: AsyncBaseEmbeddingEncoder,
-    embedder_file: Path,
-    expected_dimension: Optional[int] = None,
-    expected_is_unit_vector: bool = True,
-):
-    with open(embedder_file) as f:
-        elements = json.load(f)
-    all_text = [element["text"] for element in elements]
-    single_text = all_text[0]
-    dimension = await embedder.dimension
-    if expected_dimension:
-        assert (
-            dimension == expected_dimension
-        ), f"dimension {dimension} didn't match expected: {expected_dimension}"
-    is_unit_vector = await embedder.is_unit_vector
-    assert is_unit_vector == expected_is_unit_vector
-    single_embedding = await embedder.embed_query(query=single_text)
-    assert len(single_embedding) == dimension
-    embedded_elements = await embedder.embed_documents(elements=elements)
-    validate_embedding_output(original_elements=elements, output_elements=embedded_elements)

test/integration/partitioners/test_partitioner.py

@@ -1,76 +0,0 @@
-import os
-from pathlib import Path
-
-import pytest
-
-from test.integration.utils import requires_env
-from unstructured_ingest.errors_v2 import UserError
-from unstructured_ingest.processes.partitioner import Partitioner, PartitionerConfig
-
-int_test_dir = Path(__file__).parent
-assets_dir = int_test_dir / "assets"
-
-all_partition_files = [path for path in assets_dir.iterdir() if path.is_file()]
-non_image_partition_files = [
-    path for path in all_partition_files if path.suffix not in [".jpg", ".png", ".tif"]
-]
-supported_fast_partition_files = [
-    path for path in non_image_partition_files if path.suffix != ".eml"
-]
-image_partition_files = [
-    path for path in all_partition_files if path not in non_image_partition_files
-]
-
-
-@pytest.mark.parametrize(
-    "partition_file", all_partition_files, ids=[path.name for path in all_partition_files]
-)
-@requires_env("UNSTRUCTURED_API_KEY", "UNSTRUCTURED_API_URL")
-@pytest.mark.asyncio
-async def test_partitioner_api_hi_res(partition_file: Path):
-    api_key = os.getenv("UNSTRUCTURED_API_KEY")
-    api_url = os.getenv("UNSTRUCTURED_API_URL")
-    partitioner_config = PartitionerConfig(
-        strategy="hi_res", partition_by_api=True, api_key=api_key, partition_endpoint=api_url
-    )
-    partitioner = Partitioner(config=partitioner_config)
-    results = await partitioner.run_async(filename=partition_file)
-    assert results
-
-
-@pytest.mark.parametrize(
-    "partition_file",
-    supported_fast_partition_files,
-    ids=[path.name for path in supported_fast_partition_files],
-)
-@requires_env("UNSTRUCTURED_API_KEY", "UNSTRUCTURED_API_URL")
-@pytest.mark.asyncio
-async def test_partitioner_api_fast(partition_file: Path):
-    api_key = os.getenv("UNSTRUCTURED_API_KEY")
-    api_url = os.getenv("UNSTRUCTURED_API_URL")
-    partitioner_config = PartitionerConfig(
-        strategy="fast", partition_by_api=True, api_key=api_key, partition_endpoint=api_url
-    )
-    partitioner = Partitioner(config=partitioner_config)
-    results = await partitioner.run_async(filename=partition_file)
-    assert results
-
-
-@pytest.mark.parametrize(
-    "partition_file", image_partition_files, ids=[path.name for path in image_partition_files]
-)
-@requires_env("UNSTRUCTURED_API_KEY", "UNSTRUCTURED_API_URL")
-@pytest.mark.asyncio
-async def test_partitioner_api_fast_error(partition_file: Path):
-    api_key = os.getenv("UNSTRUCTURED_API_KEY")
-    api_url = os.getenv("UNSTRUCTURED_API_URL")
-    partitioner_config = PartitionerConfig(
-        strategy="fast",
-        partition_by_api=True,
-        api_key=api_key,
-        partition_endpoint=api_url,
-        raise_unsupported_filetype=True,
-    )
-    partitioner = Partitioner(config=partitioner_config)
-    with pytest.raises(UserError):
-        await partitioner.run_async(filename=partition_file)

test/integration/utils.py

@@ -1,15 +0,0 @@
-import os
-
-import pytest
-
-
-def requires_env(*envs):
-    if len(envs) == 1:
-        env = envs[0]
-        return pytest.mark.skipif(
-            env not in os.environ, reason=f"Environment variable not set: {env}"
-        )
-    return pytest.mark.skipif(
-        not all(env in os.environ for env in envs),
-        reason="All required environment variables not set: {}".format(", ".join(envs)),
-    )

test/unit/chunkers/test_chunkers.py

@@ -1,49 +0,0 @@
-import random
-
-import faker
-import pytest
-
-from unstructured_ingest.processes.chunker import Chunker, ChunkerConfig
-
-fake = faker.Faker()
-
-
-def generate_chunker_config_params() -> dict:
-    params = {}
-    random_val = random.random()
-    if random_val < 0.5:
-        params["chunking_strategy"] = fake.word() if random.random() < 0.5 else None
-        params["chunk_combine_text_under_n_chars"] = (
-            fake.random_int() if random.random() < 0.5 else None
-        )
-        params["chunk_include_orig_elements"] = fake.boolean() if random.random() < 0.5 else None
-        params["chunk_max_characters"] = fake.random_int()
-        params["chunk_multipage_sections"] = fake.boolean()
-        params["chunk_new_after_n_chars"] = fake.random_int() if random.random() < 0.5 else None
-        params["chunk_overlap"] = fake.random_int() if random.random() < 0.5 else None
-        params["chunk_overlap_all"] = fake.boolean() if random.random() < 0.5 else None
-    if random_val < 0.5:
-        params["chunk_by_api"] = True
-        params["chunking_endpoint"] = fake.url()
-        params["chunk_api_key"] = fake.password()
-    else:
-        params["chunk_by_api"] = False
-
-    return params
-
-
-@pytest.mark.parametrize(
-    "partition_config_params", [generate_chunker_config_params() for i in range(10)]
-)
-def test_chunker_config(partition_config_params: dict):
-    chunker_config = ChunkerConfig.model_validate(partition_config_params)
-    assert chunker_config
-
-
-@pytest.mark.parametrize(
-    "partition_config_params", [generate_chunker_config_params() for i in range(10)]
-)
-def test_chunker(partition_config_params: dict):
-    chunker_config = ChunkerConfig.model_validate(partition_config_params)
-    chunker = Chunker(config=chunker_config)
-    assert chunker