unstructured-ingest 0.3.13__py3-none-any.whl

test/unit/v2/embedders/test_huggingface.py

@@ -0,0 +1,48 @@
+import random
+from typing import Any
+
+import faker
+import pytest
+
+from test.unit.v2.utils.data_generator import generate_random_dictionary
+from unstructured_ingest.embed.huggingface import (
+    HuggingFaceEmbeddingConfig,
+    HuggingFaceEmbeddingEncoder,
+)
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {}
+    if random.random() < 0.5:
+        params["embed_model_name"] = fake.word() if random.random() < 0.5 else None
+        params["embedder_model_kwargs"] = (
+            generate_random_dictionary(key_type=str, value_type=Any)
+            if random.random() < 0.5
+            else None
+        )
+        params["encode_kwargs"] = (
+            generate_random_dictionary(key_type=str, value_type=Any)
+            if random.random() < 0.5
+            else None
+        )
+        params["cache_folder"] = fake.file_path() if random.random() < 0.5 else None
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = HuggingFaceEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = HuggingFaceEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = HuggingFaceEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/embedders/test_mixedbread.py

@@ -0,0 +1,37 @@
+import random
+
+import faker
+import pytest
+
+from unstructured_ingest.embed.mixedbreadai import (
+    MixedbreadAIEmbeddingConfig,
+    MixedbreadAIEmbeddingEncoder,
+)
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {
+        "api_key": fake.password(),
+    }
+    if random.random() < 0.5:
+        params["embedder_model_name"] = fake.word()
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = MixedbreadAIEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = MixedbreadAIEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = MixedbreadAIEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/embedders/test_octoai.py

@@ -0,0 +1,35 @@
+import random
+
+import faker
+import pytest
+
+from unstructured_ingest.embed.octoai import OctoAiEmbeddingConfig, OctoAIEmbeddingEncoder
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {
+        "api_key": fake.password(),
+    }
+    if random.random() < 0.5:
+        params["embedder_model_name"] = fake.word()
+        params["base_url"] = fake.url()
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = OctoAiEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = OctoAiEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = OctoAIEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/embedders/test_openai.py

@@ -0,0 +1,35 @@
+import random
+
+import faker
+import pytest
+
+from unstructured_ingest.embed.openai import OpenAIEmbeddingConfig, OpenAIEmbeddingEncoder
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {
+        "api_key": fake.password(),
+    }
+    if random.random() < 0.5:
+        params["embedder_model_name"] = fake.word()
+        params["base_url"] = fake.url()
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = OpenAIEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = OpenAIEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = OpenAIEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/embedders/test_togetherai.py

@@ -0,0 +1,37 @@
+import random
+
+import faker
+import pytest
+
+from unstructured_ingest.embed.togetherai import (
+    TogetherAIEmbeddingConfig,
+    TogetherAIEmbeddingEncoder,
+)
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {
+        "api_key": fake.password(),
+    }
+    if random.random() < 0.5:
+        params["embedder_model_name"] = fake.word()
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = TogetherAIEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = TogetherAIEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = TogetherAIEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/embedders/test_vertexai.py

@@ -0,0 +1,37 @@
+import json
+import random
+from typing import Any
+
+import faker
+import pytest
+
+from test.unit.v2.utils.data_generator import generate_random_dictionary
+from unstructured_ingest.embed.vertexai import VertexAIEmbeddingConfig, VertexAIEmbeddingEncoder
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {
+        "api_key": json.dumps(generate_random_dictionary(key_type=str, value_type=Any)),
+    }
+    if random.random() < 0.5:
+        params["embedder_model_name"] = fake.word()
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = VertexAIEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = VertexAIEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = VertexAIEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/embedders/test_voyageai.py

@@ -0,0 +1,38 @@
+import random
+
+import faker
+import pytest
+
+from unstructured_ingest.embed.voyageai import VoyageAIEmbeddingConfig, VoyageAIEmbeddingEncoder
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {
+        "api_key": fake.password(),
+    }
+    if random.random() < 0.5:
+        params["embedder_model_name"] = fake.word()
+        params["batch_size"] = fake.random_int()
+        params["truncation"] = fake.boolean()
+        params["max_retries"] = fake.random_int()
+        params["timeout_in_seconds"] = fake.random_int()
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = VoyageAIEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = VoyageAIEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = VoyageAIEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/partitioners/test_partitioner.py

@@ -0,0 +1,63 @@
+import random
+from typing import Any
+
+import faker
+import pytest
+
+from test.unit.v2.utils.data_generator import generate_random_dictionary
+from unstructured_ingest.v2.processes.partitioner import Partitioner, PartitionerConfig
+
+fake = faker.Faker()
+
+
+def generate_partitioner_config_params() -> dict:
+    params = {
+        "strategy": random.choice(["fast", "hi_res", "auto"]),
+        "ocr_languages": fake.words() if random.random() < 0.5 else None,
+        "encoding": fake.word() if random.random() < 0.5 else None,
+        "additional_partition_args": (
+            generate_random_dictionary(key_type=str, value_type=Any)
+            if random.random() < 0.5
+            else None
+        ),
+        "skip_infer_table_types": fake.words() if random.random() < 0.5 else None,
+        "flatten_metadata": fake.boolean(),
+        "hi_res_model_name": fake.word() if random.random() < 0.5 else None,
+    }
+    random_val = random.random()
+    # Randomly set the fields_include to a random list[str]
+    if random_val < 0.5:
+        params["fields_include"] = fake.words()
+
+    # Randomly set the metadata_exclude or metadata_include to a valid
+    # list[str] or don't set it at all
+    if random.random() < (1 / 3):
+        params["metadata_exclude"] = fake.words()
+    elif random_val < (2 / 3):
+        params["metadata_include"] = fake.words()
+
+    # Randomly set the values associated with calling the api, or not at all
+    if random.random() < 0.5:
+        params["partition_by_api"]: True
+        params["partition_endpoint"] = fake.url()
+        params["api_key"] = fake.password()
+    else:
+        params["partition_by_api"]: False
+    return params
+
+
+@pytest.mark.parametrize(
+    "partition_config_params", [generate_partitioner_config_params() for i in range(10)]
+)
+def test_partition_config(partition_config_params: dict):
+    partition_config = PartitionerConfig.model_validate(partition_config_params)
+    assert partition_config
+
+
+@pytest.mark.parametrize(
+    "partition_config_params", [generate_partitioner_config_params() for i in range(10)]
+)
+def test_partitioner(partition_config_params: dict):
+    partition_config = PartitionerConfig.model_validate(partition_config_params)
+    partitioner = Partitioner(config=partition_config)
+    assert partitioner

test/unit/v2/test_interfaces.py

@@ -0,0 +1,26 @@
+import pytest
+from pydantic import Secret, ValidationError
+
+from unstructured_ingest.v2.interfaces import AccessConfig, ConnectionConfig
+
+
+def test_failing_connection_config():
+    class MyAccessConfig(AccessConfig):
+        sensitive_value: str
+
+    class MyConnectionConfig(ConnectionConfig):
+        access_config: MyAccessConfig
+
+    with pytest.raises(ValidationError):
+        MyConnectionConfig(access_config=MyAccessConfig(sensitive_value="this"))
+
+
+def test_happy_path_connection_config():
+    class MyAccessConfig(AccessConfig):
+        sensitive_value: str
+
+    class MyConnectionConfig(ConnectionConfig):
+        access_config: Secret[MyAccessConfig]
+
+    connection_config = MyConnectionConfig(access_config=MyAccessConfig(sensitive_value="this"))
+    assert connection_config

test/unit/v2/test_utils.py

@@ -0,0 +1,82 @@
+import json
+from typing import Any
+
+from pydantic import BaseModel, Field, Secret, SecretStr
+from pydantic.types import _SecretBase
+
+from unstructured_ingest.v2.utils import serialize_base_model, serialize_base_model_json
+
+
+class MockChildBaseModel(BaseModel):
+    child_secret_str: SecretStr
+    child_secret_float: Secret[float]
+    child_not_secret_dict: dict[str, Any] = Field(default_factory=dict)
+
+
+class MockBaseModel(BaseModel):
+    secret_str: SecretStr
+    not_secret_bool: bool
+    secret_child_base: Secret[MockChildBaseModel]
+    not_secret_list: list[int] = Field(default_factory=list)
+
+
+model = MockBaseModel(
+    secret_str="secret string",
+    not_secret_bool=False,
+    secret_child_base=MockChildBaseModel(
+        child_secret_str="child secret string",
+        child_secret_float=3.14,
+        child_not_secret_dict={"key": "value"},
+    ),
+    not_secret_list=[1, 2, 3],
+)
+
+
+def test_serialize_base_model():
+
+    serialized_dict = model.model_dump()
+    assert isinstance(serialized_dict["secret_str"], _SecretBase)
+    assert isinstance(serialized_dict["secret_child_base"], _SecretBase)
+
+    serialized_dict_w_secrets = serialize_base_model(model=model)
+    assert not isinstance(serialized_dict_w_secrets["secret_str"], _SecretBase)
+    assert not isinstance(serialized_dict_w_secrets["secret_child_base"], _SecretBase)
+
+    expected_dict = {
+        "secret_str": "secret string",
+        "not_secret_bool": False,
+        "secret_child_base": {
+            "child_secret_str": "child secret string",
+            "child_secret_float": 3.14,
+            "child_not_secret_dict": {"key": "value"},
+        },
+        "not_secret_list": [1, 2, 3],
+    }
+
+    assert serialized_dict_w_secrets == expected_dict
+
+
+def test_serialize_base_model_json():
+    serialized_json = model.model_dump_json()
+    serialized_dict = json.loads(serialized_json)
+    expected_dict = {
+        "secret_str": "**********",
+        "not_secret_bool": False,
+        "secret_child_base": "**********",
+        "not_secret_list": [1, 2, 3],
+    }
+    assert expected_dict == serialized_dict
+
+    serialized_json_w_secrets = serialize_base_model_json(model=model)
+    serialized_dict_w_secrets = json.loads(serialized_json_w_secrets)
+    expected_dict_w_secrets = {
+        "secret_str": "secret string",
+        "not_secret_bool": False,
+        "secret_child_base": {
+            "child_secret_str": "child secret string",
+            "child_secret_float": 3.14,
+            "child_not_secret_dict": {"key": "value"},
+        },
+        "not_secret_list": [1, 2, 3],
+    }
+    assert expected_dict_w_secrets == serialized_dict_w_secrets

test/unit/v2/utils/data_generator.py

@@ -0,0 +1,32 @@
+import random
+from typing import Any, Type
+
+from faker import Faker
+
+fake = Faker()
+
+type_to_random_value_map = {
+    str: fake.sentence,
+    int: fake.random_int,
+    float: fake.random_digit,
+    bool: fake.boolean,
+}
+type_to_random_value_map_key = type_to_random_value_map.copy()
+type_to_random_value_map_key[str] = fake.word
+
+
+def generate_random_dictionary(key_type: Type = str, value_type: Type = str) -> dict:
+    d = {}
+    num_keys = random.randint(1, 3)
+    for i in range(num_keys):
+        key = type_to_random_value_map_key[key_type]()
+        current_value_type = value_type
+        if current_value_type == Any:
+            current_value_type = random.choice(list(type_to_random_value_map.keys()) + [dict])
+        value = (
+            generate_random_dictionary(key_type=key_type, value_type=value_type)
+            if current_value_type is dict
+            else type_to_random_value_map[current_value_type]()
+        )
+        d[key] = value
+    return d

unstructured_ingest/__init__.py

@@ -0,0 +1 @@
+from __future__ import annotations

unstructured_ingest/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.3.13"  # pragma: no cover

unstructured_ingest/cli/__init__.py

@@ -0,0 +1,14 @@
+import typing as t
+
+import click
+
+from unstructured_ingest.cli.cmds import base_dest_cmd_fns, base_src_cmd_fns
+
+src: t.List[click.Group] = [v().get_src_cmd() for v in base_src_cmd_fns]
+
+dest: t.List[click.Command] = [v().get_dest_cmd() for v in base_dest_cmd_fns]
+
+__all__ = [
+    "src",
+    "dest",
+]

unstructured_ingest/cli/base/cmd.py

@@ -0,0 +1,19 @@
+import typing as t
+from abc import ABC
+from dataclasses import dataclass, field
+
+from unstructured_ingest.cli.interfaces import CliConfig
+from unstructured_ingest.interfaces import BaseConfig
+
+
+@dataclass
+class BaseCmd(ABC):
+    cmd_name: str
+    cli_config: t.Optional[t.Type[BaseConfig]] = None
+    additional_cli_options: t.List[t.Type[CliConfig]] = field(default_factory=list)
+    addition_configs: t.Dict[str, t.Type[BaseConfig]] = field(default_factory=dict)
+    is_fsspec: bool = False
+
+    @property
+    def cmd_name_key(self):
+        return self.cmd_name.replace("-", "_")

unstructured_ingest/cli/base/dest.py

@@ -0,0 +1,87 @@
+import logging
+import typing as t
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.cmd import BaseCmd
+from unstructured_ingest.cli.cmd_factory import get_src_cmd
+from unstructured_ingest.cli.common import (
+    log_options,
+)
+from unstructured_ingest.cli.interfaces import BaseConfig, CliFilesStorageConfig
+from unstructured_ingest.cli.utils import (
+    add_options,
+    conform_click_options,
+    extract_config,
+    extract_configs,
+)
+from unstructured_ingest.logger import ingest_log_streaming_init, logger
+from unstructured_ingest.runner.writers import writer_map
+
+
+@dataclass
+class BaseDestCmd(BaseCmd):
+    write_config: t.Optional[t.Type[BaseConfig]] = None
+
+    def get_dest_runner(self, source_cmd: str, options: dict, parent_options: dict):
+        src_cmd_fn = get_src_cmd(cmd_name=source_cmd)
+        src_cmd = src_cmd_fn()
+        runner = src_cmd.get_source_runner(options=parent_options)
+        addition_configs = self.addition_configs
+        if "connector_config" not in addition_configs:
+            addition_configs["connector_config"] = self.cli_config
+        if self.write_config:
+            addition_configs["write_config"] = self.write_config
+        configs = extract_configs(
+            options,
+            validate=[self.cli_config] if self.cli_config else None,
+            extras=addition_configs,
+            add_defaults=False,
+        )
+        writer_cls = writer_map[self.cmd_name_key]
+        writer = writer_cls(**configs)  # type: ignore
+        runner.writer = writer
+        runner.writer_kwargs = options
+        return runner
+
+    def check_dest_options(self, options: dict):
+        extract_config(flat_data=options, config=self.cli_config)
+
+    def dest(self, ctx: click.Context, **options):
+        if not ctx.parent:
+            raise click.ClickException("destination command called without a parent")
+        if not ctx.parent.info_name:
+            raise click.ClickException("parent command missing info name")
+        source_cmd = ctx.parent.info_name.replace("-", "_")
+        parent_options: dict = ctx.parent.params if ctx.parent else {}
+        conform_click_options(options)
+        verbose = parent_options.get("verbose", False)
+        ingest_log_streaming_init(logging.DEBUG if verbose else logging.INFO)
+        log_options(parent_options, verbose=verbose)
+        log_options(options, verbose=verbose)
+        try:
+            self.check_dest_options(options=options)
+            runner = self.get_dest_runner(
+                source_cmd=source_cmd,
+                options=options,
+                parent_options=parent_options,
+            )
+            runner.run(**parent_options)
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise click.ClickException(str(e)) from e
+
+    def get_dest_cmd(self) -> click.Command:
+        # Dynamically create the command without the use of click decorators
+        fn = self.dest
+        fn = click.pass_context(fn)
+        cmd: click.Group = click.command(fn)
+        cmd.name = self.cmd_name
+        cmd.invoke_without_command = True
+        options = [self.cli_config] if self.cli_config else []
+        options += self.additional_cli_options
+        if self.is_fsspec and CliFilesStorageConfig not in options:
+            options.append(CliFilesStorageConfig)
+        add_options(cmd, extras=options, is_src=False)
+        return cmd

unstructured_ingest/cli/base/src.py

@@ -0,0 +1,57 @@
+import logging
+from dataclasses import dataclass
+
+import click
+
+from unstructured_ingest.cli.base.cmd import BaseCmd
+from unstructured_ingest.cli.common import (
+    log_options,
+)
+from unstructured_ingest.cli.interfaces import CliFilesStorageConfig
+from unstructured_ingest.cli.utils import Group, add_options, conform_click_options, extract_configs
+from unstructured_ingest.logger import ingest_log_streaming_init, logger
+from unstructured_ingest.runner import runner_map
+
+
+@dataclass
+class BaseSrcCmd(BaseCmd):
+    def get_source_runner(self, options: dict):
+        addition_configs = self.addition_configs
+        if "connector_config" not in addition_configs:
+            addition_configs["connector_config"] = self.cli_config
+        configs = extract_configs(
+            options,
+            validate=[self.cli_config] if self.cli_config else None,
+            extras=addition_configs,
+        )
+        runner = runner_map[self.cmd_name_key]
+        return runner(**configs)  # type: ignore
+
+    def src(self, ctx: click.Context, **options):
+        if ctx.invoked_subcommand:
+            return
+
+        conform_click_options(options)
+        verbose = options.get("verbose", False)
+        ingest_log_streaming_init(logging.DEBUG if verbose else logging.INFO)
+        log_options(options, verbose=verbose)
+        try:
+            runner = self.get_source_runner(options=options)
+            runner.run(**options)
+        except Exception as e:
+            logger.error(e, exc_info=True)
+            raise click.ClickException(str(e)) from e
+
+    def get_src_cmd(self) -> click.Group:
+        # Dynamically create the command without the use of click decorators
+        fn = self.src
+        fn = click.pass_context(fn)
+        cmd: click.Group = click.group(fn, cls=Group)
+        cmd.name = self.cmd_name
+        cmd.invoke_without_command = True
+        extra_options = [self.cli_config] if self.cli_config else []
+        extra_options += self.additional_cli_options
+        if self.is_fsspec and CliFilesStorageConfig not in extra_options:
+            extra_options.append(CliFilesStorageConfig)
+        add_options(cmd, extras=extra_options)
+        return cmd