unstructured-ingest 0.7.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

test/unit/embedders/test_huggingface.py

@@ -1,48 +0,0 @@
-import random
-from typing import Any
-
-import faker
-import pytest
-
-from test.unit.utils.data_generator import generate_random_dictionary
-from unstructured_ingest.embed.huggingface import (
-    HuggingFaceEmbeddingConfig,
-    HuggingFaceEmbeddingEncoder,
-)
-
-fake = faker.Faker()
-
-
-def generate_embedder_config_params() -> dict:
-    params = {}
-    if random.random() < 0.5:
-        params["embedder_model_name"] = fake.word() if random.random() < 0.5 else None
-        params["embedder_model_kwargs"] = (
-            generate_random_dictionary(key_type=str, value_type=Any)
-            if random.random() < 0.5
-            else None
-        )
-        params["encode_kwargs"] = (
-            generate_random_dictionary(key_type=str, value_type=Any)
-            if random.random() < 0.5
-            else None
-        )
-        params["cache_folder"] = fake.file_path() if random.random() < 0.5 else None
-    return params
-
-
-@pytest.mark.parametrize(
-    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
-)
-def test_embedder_config(embedder_config_params: dict):
-    embedder_config = HuggingFaceEmbeddingConfig.model_validate(embedder_config_params)
-    assert embedder_config
-
-
-@pytest.mark.parametrize(
-    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
-)
-def test_embedder(embedder_config_params: dict):
-    embedder_config = HuggingFaceEmbeddingConfig.model_validate(embedder_config_params)
-    embedder = HuggingFaceEmbeddingEncoder(config=embedder_config)
-    assert embedder

test/unit/embedders/test_mixedbread.py

@@ -1,37 +0,0 @@
-import random
-
-import faker
-import pytest
-
-from unstructured_ingest.embed.mixedbreadai import (
-    MixedbreadAIEmbeddingConfig,
-    MixedbreadAIEmbeddingEncoder,
-)
-
-fake = faker.Faker()
-
-
-def generate_embedder_config_params() -> dict:
-    params = {
-        "api_key": fake.password(),
-    }
-    if random.random() < 0.5:
-        params["embedder_model_name"] = fake.word()
-    return params
-
-
-@pytest.mark.parametrize(
-    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
-)
-def test_embedder_config(embedder_config_params: dict):
-    embedder_config = MixedbreadAIEmbeddingConfig.model_validate(embedder_config_params)
-    assert embedder_config
-
-
-@pytest.mark.parametrize(
-    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
-)
-def test_embedder(embedder_config_params: dict):
-    embedder_config = MixedbreadAIEmbeddingConfig.model_validate(embedder_config_params)
-    embedder = MixedbreadAIEmbeddingEncoder(config=embedder_config)
-    assert embedder

test/unit/embedders/test_octoai.py

@@ -1,35 +0,0 @@
-import random
-
-import faker
-import pytest
-
-from unstructured_ingest.embed.octoai import OctoAiEmbeddingConfig, OctoAIEmbeddingEncoder
-
-fake = faker.Faker()
-
-
-def generate_embedder_config_params() -> dict:
-    params = {
-        "api_key": fake.password(),
-    }
-    if random.random() < 0.5:
-        params["embedder_model_name"] = fake.word()
-        params["base_url"] = fake.url()
-    return params
-
-
-@pytest.mark.parametrize(
-    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
-)
-def test_embedder_config(embedder_config_params: dict):
-    embedder_config = OctoAiEmbeddingConfig.model_validate(embedder_config_params)
-    assert embedder_config
-
-
-@pytest.mark.parametrize(
-    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
-)
-def test_embedder(embedder_config_params: dict):
-    embedder_config = OctoAiEmbeddingConfig.model_validate(embedder_config_params)
-    embedder = OctoAIEmbeddingEncoder(config=embedder_config)
-    assert embedder

test/unit/embedders/test_openai.py

@@ -1,35 +0,0 @@
-import random
-
-import faker
-import pytest
-
-from unstructured_ingest.embed.openai import OpenAIEmbeddingConfig, OpenAIEmbeddingEncoder
-
-fake = faker.Faker()
-
-
-def generate_embedder_config_params() -> dict:
-    params = {
-        "api_key": fake.password(),
-    }
-    if random.random() < 0.5:
-        params["embedder_model_name"] = fake.word()
-        params["base_url"] = fake.url()
-    return params
-
-
-@pytest.mark.parametrize(
-    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
-)
-def test_embedder_config(embedder_config_params: dict):
-    embedder_config = OpenAIEmbeddingConfig.model_validate(embedder_config_params)
-    assert embedder_config
-
-
-@pytest.mark.parametrize(
-    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
-)
-def test_embedder(embedder_config_params: dict):
-    embedder_config = OpenAIEmbeddingConfig.model_validate(embedder_config_params)
-    embedder = OpenAIEmbeddingEncoder(config=embedder_config)
-    assert embedder

test/unit/embedders/test_togetherai.py

@@ -1,37 +0,0 @@
-import random
-
-import faker
-import pytest
-
-from unstructured_ingest.embed.togetherai import (
-    TogetherAIEmbeddingConfig,
-    TogetherAIEmbeddingEncoder,
-)
-
-fake = faker.Faker()
-
-
-def generate_embedder_config_params() -> dict:
-    params = {
-        "api_key": fake.password(),
-    }
-    if random.random() < 0.5:
-        params["embedder_model_name"] = fake.word()
-    return params
-
-
-@pytest.mark.parametrize(
-    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
-)
-def test_embedder_config(embedder_config_params: dict):
-    embedder_config = TogetherAIEmbeddingConfig.model_validate(embedder_config_params)
-    assert embedder_config
-
-
-@pytest.mark.parametrize(
-    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
-)
-def test_embedder(embedder_config_params: dict):
-    embedder_config = TogetherAIEmbeddingConfig.model_validate(embedder_config_params)
-    embedder = TogetherAIEmbeddingEncoder(config=embedder_config)
-    assert embedder

test/unit/embedders/test_vertexai.py

@@ -1,37 +0,0 @@
-import json
-import random
-from typing import Any
-
-import faker
-import pytest
-
-from test.unit.utils.data_generator import generate_random_dictionary
-from unstructured_ingest.embed.vertexai import VertexAIEmbeddingConfig, VertexAIEmbeddingEncoder
-
-fake = faker.Faker()
-
-
-def generate_embedder_config_params() -> dict:
-    params = {
-        "api_key": json.dumps(generate_random_dictionary(key_type=str, value_type=Any)),
-    }
-    if random.random() < 0.5:
-        params["embedder_model_name"] = fake.word()
-    return params
-
-
-@pytest.mark.parametrize(
-    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
-)
-def test_embedder_config(embedder_config_params: dict):
-    embedder_config = VertexAIEmbeddingConfig.model_validate(embedder_config_params)
-    assert embedder_config
-
-
-@pytest.mark.parametrize(
-    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
-)
-def test_embedder(embedder_config_params: dict):
-    embedder_config = VertexAIEmbeddingConfig.model_validate(embedder_config_params)
-    embedder = VertexAIEmbeddingEncoder(config=embedder_config)
-    assert embedder

test/unit/embedders/test_voyageai.py

@@ -1,38 +0,0 @@
-import random
-
-import faker
-import pytest
-
-from unstructured_ingest.embed.voyageai import VoyageAIEmbeddingConfig, VoyageAIEmbeddingEncoder
-
-fake = faker.Faker()
-
-
-def generate_embedder_config_params() -> dict:
-    params = {
-        "api_key": fake.password(),
-    }
-    if random.random() < 0.5:
-        params["embedder_model_name"] = fake.word()
-        params["batch_size"] = fake.random_int(max=100)
-        params["truncation"] = fake.boolean()
-        params["max_retries"] = fake.random_int()
-        params["timeout_in_seconds"] = fake.random_int()
-    return params
-
-
-@pytest.mark.parametrize(
-    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
-)
-def test_embedder_config(embedder_config_params: dict):
-    embedder_config = VoyageAIEmbeddingConfig.model_validate(embedder_config_params)
-    assert embedder_config
-
-
-@pytest.mark.parametrize(
-    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
-)
-def test_embedder(embedder_config_params: dict):
-    embedder_config = VoyageAIEmbeddingConfig.model_validate(embedder_config_params)
-    embedder = VoyageAIEmbeddingEncoder(config=embedder_config)
-    assert embedder

test/unit/partitioners/test_partitioner.py

@@ -1,63 +0,0 @@
-import random
-from typing import Any
-
-import faker
-import pytest
-
-from test.unit.utils.data_generator import generate_random_dictionary
-from unstructured_ingest.processes.partitioner import Partitioner, PartitionerConfig
-
-fake = faker.Faker()
-
-
-def generate_partitioner_config_params() -> dict:
-    params = {
-        "strategy": random.choice(["fast", "hi_res", "auto"]),
-        "ocr_languages": fake.words() if random.random() < 0.5 else None,
-        "encoding": fake.word() if random.random() < 0.5 else None,
-        "additional_partition_args": (
-            generate_random_dictionary(key_type=str, value_type=Any)
-            if random.random() < 0.5
-            else None
-        ),
-        "skip_infer_table_types": fake.words() if random.random() < 0.5 else None,
-        "flatten_metadata": fake.boolean(),
-        "hi_res_model_name": fake.word() if random.random() < 0.5 else None,
-    }
-    random_val = random.random()
-    # Randomly set the fields_include to a random list[str]
-    if random_val < 0.5:
-        params["fields_include"] = fake.words()
-
-    # Randomly set the metadata_exclude or metadata_include to a valid
-    # list[str] or don't set it at all
-    if random.random() < (1 / 3):
-        params["metadata_exclude"] = fake.words()
-    elif random_val < (2 / 3):
-        params["metadata_include"] = fake.words()
-
-    # Randomly set the values associated with calling the api, or not at all
-    if random.random() < 0.5:
-        params["partition_by_api"]: True
-        params["partition_endpoint"] = fake.url()
-        params["api_key"] = fake.password()
-    else:
-        params["partition_by_api"]: False
-    return params
-
-
-@pytest.mark.parametrize(
-    "partition_config_params", [generate_partitioner_config_params() for i in range(10)]
-)
-def test_partition_config(partition_config_params: dict):
-    partition_config = PartitionerConfig.model_validate(partition_config_params)
-    assert partition_config
-
-
-@pytest.mark.parametrize(
-    "partition_config_params", [generate_partitioner_config_params() for i in range(10)]
-)
-def test_partitioner(partition_config_params: dict):
-    partition_config = PartitionerConfig.model_validate(partition_config_params)
-    partitioner = Partitioner(config=partition_config)
-    assert partitioner

test/unit/test_error.py

@@ -1,27 +0,0 @@
-import pytest
-
-from unstructured_ingest.error import (
-    DestinationConnectionError,
-    PartitionError,
-    SourceConnectionError,
-)
-
-
-@pytest.mark.parametrize(
-    ("error_class", "exception_type", "error_message"),
-    [
-        (SourceConnectionError, ValueError, "Simulated connection error"),
-        (DestinationConnectionError, RuntimeError, "Simulated connection error"),
-        (PartitionError, FileNotFoundError, "Simulated partition error"),
-    ],
-)
-def test_custom_error_decorator(error_class, exception_type, error_message):
-    @error_class.wrap
-    def simulate_error():
-        raise exception_type(error_message)
-
-    with pytest.raises(error_class) as context:
-        simulate_error()
-
-    expected_error_string = error_class.error_string.format(error_message)
-    assert str(context.value) == expected_error_string

test/unit/test_html.py

@@ -1,112 +0,0 @@
-import base64
-from pathlib import Path
-
-from bs4 import BeautifulSoup
-from pytest_mock import MockerFixture
-
-from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
-from unstructured_ingest.utils.html import HtmlMixin
-
-
-def test_extract_images(mocker: MockerFixture):
-    mixin = HtmlMixin(extract_images=True)
-    mock_download_response = b"DOWNLOADED"
-    expected_image_src = base64.b64encode(mock_download_response).decode()
-    mocked_download_response = mocker.patch(
-        "unstructured_ingest.utils.html.HtmlMixin.download_content",
-        return_value=mock_download_response,
-    )
-    url = "http://mywebsite.com/path/to/page"
-    html = """
-    <img src="http://mywebsite.com/img1.jpg"/>
-    <img src="http://notmywebsite.com/img2.jpg"/>
-    <img src="img3.jpg"/>
-    <img src="data:image/png;base64,24689654..."/>
-    """
-    expected_html = f"""
-    <img src="data:image/png;base64,{expected_image_src}"/>
-    <img src="http://notmywebsite.com/img2.jpg"/>
-    <img src="data:image/png;base64,{expected_image_src}"/>
-    <img src="data:image/png;base64,24689654..."/>
-    """
-    expected_soup = BeautifulSoup(expected_html, "html.parser")
-    result = mixin.extract_html_images(url=url, html=html)
-    result_soup = BeautifulSoup(result, "html.parser")
-    assert expected_soup == result_soup
-    assert mocked_download_response.call_count == 2
-    urls_to_download = [
-        call_args_list.kwargs["url"] for call_args_list in mocked_download_response.call_args_list
-    ]
-    assert urls_to_download == ["http://mywebsite.com/img1.jpg", "http://mywebsite.com/img3.jpg"]
-
-
-def test_extract_images_allow_list(mocker: MockerFixture):
-    mixin = HtmlMixin(
-        extract_images=True, allow_list=["http://allowedwebsite1.com", "http://allowedwebsite2.com"]
-    )
-    mock_download_response = b"DOWNLOADED"
-    expected_image_src = base64.b64encode(mock_download_response).decode()
-    mocked_download_response = mocker.patch(
-        "unstructured_ingest.utils.html.HtmlMixin.download_content",
-        return_value=mock_download_response,
-    )
-    url = "http://mywebsite.com/path/to/page"
-    html = """
-    <img src="http://mywebsite.com/img1.jpg"/>
-    <img src="http://notmywebsite.com/img2.jpg"/>
-    <img src="http://allowedwebsite1.com/img2.jpg"/>
-    <img src="http://allowedwebsite2.com/img2.jpg"/>
-    """
-
-    expected_html = f"""
-    <img src="http://mywebsite.com/img1.jpg"/>
-    <img src="http://notmywebsite.com/img2.jpg"/>
-    <img src="data:image/png;base64,{expected_image_src}"/>
-    <img src="data:image/png;base64,{expected_image_src}"/>
-    """
-    expected_soup = BeautifulSoup(expected_html, "html.parser")
-    result = mixin.extract_html_images(url=url, html=html)
-    result_soup = BeautifulSoup(result, "html.parser")
-    assert expected_soup == result_soup
-    assert mocked_download_response.call_count == 2
-    urls_to_download = [
-        call_args_list.kwargs["url"] for call_args_list in mocked_download_response.call_args_list
-    ]
-    assert urls_to_download == [
-        "http://allowedwebsite1.com/img2.jpg",
-        "http://allowedwebsite2.com/img2.jpg",
-    ]
-
-
-def test_extract_embedded_docs(mocker: MockerFixture):
-    mixin = HtmlMixin(extract_files=True)
-    mock_download_response = b"DOWNLOADED"
-    mocked_download_response = mocker.patch(
-        "unstructured_ingest.utils.html.HtmlMixin.download_content",
-        return_value=mock_download_response,
-    )
-    mocked_write_content = mocker.patch("unstructured_ingest.utils.html.HtmlMixin.write_content")
-    url = "http://mywebsite.com/path/to/page"
-    html = """
-    <a href="http://mywebsite.com/file.pdf"/>
-    <a href="http://notmywebsite.com/file.pdf"/>
-    <a href="http://mywebsite.com/another/link"/>
-    <a href="another/link/2"/>
-    <a href="file.doc"/>
-    """
-    file_data = FileData(
-        source_identifiers=SourceIdentifiers(
-            fullpath="file.txt",
-            filename="file.txt",
-        ),
-        connector_type="my_connector",
-        identifier="mock_file_data",
-    )
-    results = mixin.extract_embedded_files(
-        url=url, html=html, download_dir=Path("/tmp/download/location"), original_filedata=file_data
-    )
-    assert len(results) == 2
-    downloaded_urls = [r["file_data"].metadata.url for r in results]
-    assert downloaded_urls == ["http://mywebsite.com/file.pdf", "http://mywebsite.com/file.doc"]
-    assert mocked_download_response.call_count == 2
-    assert mocked_write_content.call_count == 2

test/unit/test_interfaces.py

@@ -1,26 +0,0 @@
-import pytest
-from pydantic import Secret, ValidationError
-
-from unstructured_ingest.interfaces import AccessConfig, ConnectionConfig
-
-
-def test_failing_connection_config():
-    class MyAccessConfig(AccessConfig):
-        sensitive_value: str
-
-    class MyConnectionConfig(ConnectionConfig):
-        access_config: MyAccessConfig
-
-    with pytest.raises(ValidationError):
-        MyConnectionConfig(access_config=MyAccessConfig(sensitive_value="this"))
-
-
-def test_happy_path_connection_config():
-    class MyAccessConfig(AccessConfig):
-        sensitive_value: str
-
-    class MyConnectionConfig(ConnectionConfig):
-        access_config: Secret[MyAccessConfig]
-
-    connection_config = MyConnectionConfig(access_config=MyAccessConfig(sensitive_value="this"))
-    assert connection_config

test/unit/test_logger.py

@@ -1,78 +0,0 @@
-import json
-
-import pytest
-
-from unstructured_ingest.logger import (
-    default_is_data_sensitive,
-    hide_sensitive_fields,
-    redact_jsons,
-)
-
-
-@pytest.mark.parametrize(
-    ("key", "value", "is_sensitive"),
-    [
-        ("username", "john_smith", False),
-        ("password", "13?H%", True),
-        ("token", "123", True),
-        ("AWS_CREDENTIAL", "aws_credential", True),
-        ("AWS_KEY", None, False),
-    ],
-)
-def test_default_is_sensitive(key, value, is_sensitive):
-    assert default_is_data_sensitive(key, value) == is_sensitive
-
-
-def test_hide_sensitive_fields():
-    d = {
-        "username": "john_smith",
-        "password": "13?H%",
-        "inner": {
-            "token": "123",
-            "AWS_KEY": None,
-            "inner_j_string": json.dumps(
-                {"account_name": "secret name", "client_id": 123, "timestamp": 123}
-            ),
-        },
-    }
-    redacted_d = hide_sensitive_fields(d)
-    expected_d = {
-        "password": "*******",
-        "username": "john_smith",
-        "inner": {
-            "token": "*******",
-            "AWS_KEY": None,
-            "inner_j_string": json.dumps(
-                {"account_name": "*******", "client_id": "*******", "timestamp": 123}
-            ),
-        },
-    }
-    assert redacted_d == expected_d
-
-
-def test_redact_jsons():
-    d1 = {
-        "username": "john_smith",
-        "password": "13?H%",
-        "inner": {
-            "token": "123",
-            "AWS_KEY": None,
-            "inner_j_string": json.dumps(
-                {"account_name": "secret name", "client_id": 123, "timestamp": 123}
-            ),
-        },
-    }
-
-    d2 = {"username": "tim67", "update_time": 456}
-    d3 = {"account_name": "top secret", "host": "http://localhost:8888"}
-
-    sensitive_string = f"Some topic secret info ({json.dumps(d1)} regarding {d2} and {d3})"
-    expected_string = (
-        'Some topic secret info ({"username": "john_smith", "password": "*******", '
-        '"inner": {"token": "*******", "AWS_KEY": null, "inner_j_string": '
-        '"{\\"account_name\\": \\"*******\\", \\"client_id\\": \\"*******\\", '
-        '\\"timestamp\\": 123}"}} regarding {"username": "tim67", "update_time": 456} '
-        'and {"account_name": "*******", "host": "http://localhost:8888"})'
-    )
-    redacted_string = redact_jsons(sensitive_string)
-    assert redacted_string == expected_string