unstructured-ingest 0.2.2__py3-none-any.whl → 0.3.0__py3-none-any.whl

test/integration/connectors/test_s3.py

@@ -71,6 +71,29 @@ async def test_s3_source(anon_connection_config: S3ConnectionConfig):
         )
 
 
+@pytest.mark.asyncio
+@pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
+async def test_s3_source_special_char(anon_connection_config: S3ConnectionConfig):
+    indexer_config = S3IndexerConfig(remote_url="s3://utic-dev-tech-fixtures/special-characters/")
+    with tempfile.TemporaryDirectory() as tempdir:
+        tempdir_path = Path(tempdir)
+        download_config = S3DownloaderConfig(download_dir=tempdir_path)
+        indexer = S3Indexer(connection_config=anon_connection_config, index_config=indexer_config)
+        downloader = S3Downloader(
+            connection_config=anon_connection_config, download_config=download_config
+        )
+        await source_connector_validation(
+            indexer=indexer,
+            downloader=downloader,
+            configs=ValidationConfigs(
+                test_id="s3-specialchar",
+                predownload_file_data_check=validate_predownload_file_data,
+                postdownload_file_data_check=validate_postdownload_file_data,
+                expected_num_files=1,
+            ),
+        )
+
+
 @pytest.mark.asyncio
 @pytest.mark.tags(CONNECTOR_TYPE, SOURCE_TAG)
 async def test_s3_source_no_access(anon_connection_config: S3ConnectionConfig):

test/unit/v2/chunkers/test_chunkers.py

@@ -0,0 +1,49 @@
+import random
+
+import faker
+import pytest
+
+from unstructured_ingest.v2.processes.chunker import Chunker, ChunkerConfig
+
+fake = faker.Faker()
+
+
+def generate_chunker_config_params() -> dict:
+    params = {}
+    random_val = random.random()
+    if random_val < 0.5:
+        params["chunking_strategy"] = fake.word() if random.random() < 0.5 else None
+        params["chunk_combine_text_under_n_chars"] = (
+            fake.random_int() if random.random() < 0.5 else None
+        )
+        params["chunk_include_orig_elements"] = fake.boolean() if random.random() < 0.5 else None
+        params["chunk_max_characters"] = fake.random_int()
+        params["chunk_multipage_sections"] = fake.boolean()
+        params["chunk_new_after_n_chars"] = fake.random_int() if random.random() < 0.5 else None
+        params["chunk_overlap"] = fake.random_int() if random.random() < 0.5 else None
+        params["chunk_overlap_all"] = fake.boolean() if random.random() < 0.5 else None
+    if random_val < 0.5:
+        params["chunk_by_api"] = True
+        params["chunking_endpoint"] = fake.url()
+        params["chunk_api_key"] = fake.password()
+    else:
+        params["chunk_by_api"] = False
+
+    return params
+
+
+@pytest.mark.parametrize(
+    "partition_config_params", [generate_chunker_config_params() for i in range(10)]
+)
+def test_chunker_config(partition_config_params: dict):
+    chunker_config = ChunkerConfig.model_validate(partition_config_params)
+    assert chunker_config
+
+
+@pytest.mark.parametrize(
+    "partition_config_params", [generate_chunker_config_params() for i in range(10)]
+)
+def test_chunker(partition_config_params: dict):
+    chunker_config = ChunkerConfig.model_validate(partition_config_params)
+    chunker = Chunker(config=chunker_config)
+    assert chunker

test/unit/v2/embedders/test_bedrock.py

@@ -0,0 +1,36 @@
+import random
+
+import faker
+import pytest
+
+from unstructured_ingest.embed.bedrock import BedrockEmbeddingConfig, BedrockEmbeddingEncoder
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {
+        "aws_access_key_id": fake.password(),
+        "aws_secret_access_key": fake.password(),
+        "region_name": fake.city(),
+    }
+    if random.random() < 0.5:
+        params["embed_model_name"] = fake.word()
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = BedrockEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = BedrockEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = BedrockEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/embedders/test_huggingface.py

@@ -0,0 +1,48 @@
+import random
+from typing import Any
+
+import faker
+import pytest
+
+from test.unit.v2.utils.data_generator import generate_random_dictionary
+from unstructured_ingest.embed.huggingface import (
+    HuggingFaceEmbeddingConfig,
+    HuggingFaceEmbeddingEncoder,
+)
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {}
+    if random.random() < 0.5:
+        params["embed_model_name"] = fake.word() if random.random() < 0.5 else None
+        params["embedder_model_kwargs"] = (
+            generate_random_dictionary(key_type=str, value_type=Any)
+            if random.random() < 0.5
+            else None
+        )
+        params["encode_kwargs"] = (
+            generate_random_dictionary(key_type=str, value_type=Any)
+            if random.random() < 0.5
+            else None
+        )
+        params["cache_folder"] = fake.file_path() if random.random() < 0.5 else None
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = HuggingFaceEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = HuggingFaceEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = HuggingFaceEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/embedders/test_mixedbread.py

@@ -0,0 +1,37 @@
+import random
+
+import faker
+import pytest
+
+from unstructured_ingest.embed.mixedbreadai import (
+    MixedbreadAIEmbeddingConfig,
+    MixedbreadAIEmbeddingEncoder,
+)
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {
+        "api_key": fake.password(),
+    }
+    if random.random() < 0.5:
+        params["embedder_model_name"] = fake.word()
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = MixedbreadAIEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = MixedbreadAIEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = MixedbreadAIEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/embedders/test_octoai.py

@@ -0,0 +1,35 @@
+import random
+
+import faker
+import pytest
+
+from unstructured_ingest.embed.octoai import OctoAiEmbeddingConfig, OctoAIEmbeddingEncoder
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {
+        "api_key": fake.password(),
+    }
+    if random.random() < 0.5:
+        params["embedder_model_name"] = fake.word()
+        params["base_url"] = fake.url()
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = OctoAiEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = OctoAiEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = OctoAIEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/embedders/test_openai.py

@@ -0,0 +1,35 @@
+import random
+
+import faker
+import pytest
+
+from unstructured_ingest.embed.openai import OpenAIEmbeddingConfig, OpenAIEmbeddingEncoder
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {
+        "api_key": fake.password(),
+    }
+    if random.random() < 0.5:
+        params["embedder_model_name"] = fake.word()
+        params["base_url"] = fake.url()
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = OpenAIEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = OpenAIEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = OpenAIEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/embedders/test_togetherai.py

@@ -0,0 +1,37 @@
+import random
+
+import faker
+import pytest
+
+from unstructured_ingest.embed.togetherai import (
+    TogetherAIEmbeddingConfig,
+    TogetherAIEmbeddingEncoder,
+)
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {
+        "api_key": fake.password(),
+    }
+    if random.random() < 0.5:
+        params["embedder_model_name"] = fake.word()
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = TogetherAIEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = TogetherAIEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = TogetherAIEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/embedders/test_vertexai.py

@@ -0,0 +1,37 @@
+import json
+import random
+from typing import Any
+
+import faker
+import pytest
+
+from test.unit.v2.utils.data_generator import generate_random_dictionary
+from unstructured_ingest.embed.vertexai import VertexAIEmbeddingConfig, VertexAIEmbeddingEncoder
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {
+        "api_key": json.dumps(generate_random_dictionary(key_type=str, value_type=Any)),
+    }
+    if random.random() < 0.5:
+        params["embedder_model_name"] = fake.word()
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = VertexAIEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = VertexAIEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = VertexAIEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/embedders/test_voyageai.py

@@ -0,0 +1,38 @@
+import random
+
+import faker
+import pytest
+
+from unstructured_ingest.embed.voyageai import VoyageAIEmbeddingConfig, VoyageAIEmbeddingEncoder
+
+fake = faker.Faker()
+
+
+def generate_embedder_config_params() -> dict:
+    params = {
+        "api_key": fake.password(),
+    }
+    if random.random() < 0.5:
+        params["embedder_model_name"] = fake.word()
+        params["batch_size"] = fake.random_int()
+        params["truncation"] = fake.boolean()
+        params["max_retries"] = fake.random_int()
+        params["timeout_in_seconds"] = fake.random_int()
+    return params
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder_config(embedder_config_params: dict):
+    embedder_config = VoyageAIEmbeddingConfig.model_validate(embedder_config_params)
+    assert embedder_config
+
+
+@pytest.mark.parametrize(
+    "embedder_config_params", [generate_embedder_config_params() for i in range(10)]
+)
+def test_embedder(embedder_config_params: dict):
+    embedder_config = VoyageAIEmbeddingConfig.model_validate(embedder_config_params)
+    embedder = VoyageAIEmbeddingEncoder(config=embedder_config)
+    assert embedder

test/unit/v2/partitioners/test_partitioner.py

@@ -0,0 +1,63 @@
+import random
+from typing import Any
+
+import faker
+import pytest
+
+from test.unit.v2.utils.data_generator import generate_random_dictionary
+from unstructured_ingest.v2.processes.partitioner import Partitioner, PartitionerConfig
+
+fake = faker.Faker()
+
+
+def generate_partitioner_config_params() -> dict:
+    params = {
+        "strategy": random.choice(["fast", "hi_res", "auto"]),
+        "ocr_languages": fake.words() if random.random() < 0.5 else None,
+        "encoding": fake.word() if random.random() < 0.5 else None,
+        "additional_partition_args": (
+            generate_random_dictionary(key_type=str, value_type=Any)
+            if random.random() < 0.5
+            else None
+        ),
+        "skip_infer_table_types": fake.words() if random.random() < 0.5 else None,
+        "flatten_metadata": fake.boolean(),
+        "hi_res_model_name": fake.word() if random.random() < 0.5 else None,
+    }
+    random_val = random.random()
+    # Randomly set the fields_include to a random list[str]
+    if random_val < 0.5:
+        params["fields_include"] = fake.words()
+
+    # Randomly set the metadata_exclude or metadata_include to a valid
+    # list[str] or don't set it at all
+    if random.random() < (1 / 3):
+        params["metadata_exclude"] = fake.words()
+    elif random_val < (2 / 3):
+        params["metadata_include"] = fake.words()
+
+    # Randomly set the values associated with calling the api, or not at all
+    if random.random() < 0.5:
+        params["partition_by_api"]: True
+        params["partition_endpoint"] = fake.url()
+        params["api_key"] = fake.password()
+    else:
+        params["partition_by_api"]: False
+    return params
+
+
+@pytest.mark.parametrize(
+    "partition_config_params", [generate_partitioner_config_params() for i in range(10)]
+)
+def test_partition_config(partition_config_params: dict):
+    partition_config = PartitionerConfig.model_validate(partition_config_params)
+    assert partition_config
+
+
+@pytest.mark.parametrize(
+    "partition_config_params", [generate_partitioner_config_params() for i in range(10)]
+)
+def test_partitioner(partition_config_params: dict):
+    partition_config = PartitionerConfig.model_validate(partition_config_params)
+    partitioner = Partitioner(config=partition_config)
+    assert partitioner

test/unit/v2/utils/data_generator.py

@@ -0,0 +1,32 @@
+import random
+from typing import Any, Type
+
+from faker import Faker
+
+fake = Faker()
+
+type_to_random_value_map = {
+    str: fake.sentence,
+    int: fake.random_int,
+    float: fake.random_digit,
+    bool: fake.boolean,
+}
+type_to_random_value_map_key = type_to_random_value_map.copy()
+type_to_random_value_map_key[str] = fake.word
+
+
+def generate_random_dictionary(key_type: Type = str, value_type: Type = str) -> dict:
+    d = {}
+    num_keys = random.randint(1, 3)
+    for i in range(num_keys):
+        key = type_to_random_value_map_key[key_type]()
+        current_value_type = value_type
+        if current_value_type == Any:
+            current_value_type = random.choice(list(type_to_random_value_map.keys()) + [dict])
+        value = (
+            generate_random_dictionary(key_type=key_type, value_type=value_type)
+            if current_value_type is dict
+            else type_to_random_value_map[current_value_type]()
+        )
+        d[key] = value
+    return d

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.2.2"  # pragma: no cover
+__version__ = "0.3.0"  # pragma: no cover

unstructured_ingest/cli/cmds/__init__.py

@@ -9,7 +9,7 @@ from unstructured_ingest.cli.cmds.fsspec.sftp import get_base_src_cmd as sftp_ba
 from .airtable import get_base_src_cmd as airtable_base_src_cmd
 from .astradb import get_base_dest_cmd as astradb_base_dest_cmd
 from .astradb import get_base_src_cmd as astradb_base_src_cmd
-from .azure_cognitive_search import get_base_dest_cmd as azure_cognitive_search_base_dest_cmd
+from .azure_ai_search import get_base_dest_cmd as azure_ai_search_base_dest_cmd
 from .biomed import get_base_src_cmd as biomed_base_src_cmd
 from .chroma import get_base_dest_cmd as chroma_base_dest_cmd
 from .clarifai import get_base_dest_cmd as clarifai_base_dest_cmd
@@ -118,7 +118,7 @@ base_dest_cmd_fns: t.List[t.Callable[[], "BaseDestCmd"]] = [
     gcs_base_dest_cmd,
     kafka_base_dest_cmd,
     s3_base_dest_cmd,
-    azure_cognitive_search_base_dest_cmd,
+    azure_ai_search_base_dest_cmd,
     delta_table_dest_cmd,
     sql_base_dest_cmd,
     weaviate_dest_cmd,

unstructured_ingest/cli/cmds/{azure_cognitive_search.py → azure_ai_search.py}

@@ -6,14 +6,14 @@ import click
 from unstructured_ingest.cli.interfaces import (
     CliConfig,
 )
-from unstructured_ingest.connector.azure_cognitive_search import (
-    AzureCognitiveSearchWriteConfig,
-    SimpleAzureCognitiveSearchStorageConfig,
+from unstructured_ingest.connector.azure_ai_search import (
+    AzureAISearchWriteConfig,
+    SimpleAzureAISearchStorageConfig,
 )
 
 
 @dataclass
-class AzureCognitiveSearchCliConfig(SimpleAzureCognitiveSearchStorageConfig, CliConfig):
+class AzureAISearchCliConfig(SimpleAzureAISearchStorageConfig, CliConfig):
     @staticmethod
     def get_cli_options() -> t.List[click.Option]:
         options = [
@@ -39,7 +39,7 @@ class AzureCognitiveSearchCliConfig(SimpleAzureCognitiveSearchStorageConfig, Cli
 
 
 @dataclass
-class AzureCognitiveSearchCliWriteConfig(AzureCognitiveSearchWriteConfig, CliConfig):
+class AzureAISearchCliWriteConfig(AzureAISearchWriteConfig, CliConfig):
     @staticmethod
     def get_cli_options() -> t.List[click.Option]:
         options = [
@@ -57,9 +57,9 @@ def get_base_dest_cmd():
     from unstructured_ingest.cli.base.dest import BaseDestCmd
 
     cmd_cls = BaseDestCmd(
-        cmd_name="azure-cognitive-search",
-        cli_config=AzureCognitiveSearchCliConfig,
-        additional_cli_options=[AzureCognitiveSearchCliWriteConfig],
-        write_config=AzureCognitiveSearchCliWriteConfig,
+        cmd_name="azure-ai-search",
+        cli_config=AzureAISearchCliConfig,
+        additional_cli_options=[AzureAISearchCliWriteConfig],
+        write_config=AzureAISearchCliWriteConfig,
     )
     return cmd_cls

unstructured_ingest/connector/{azure_cognitive_search.py → azure_ai_search.py}

@@ -19,28 +19,28 @@ if t.TYPE_CHECKING:
 
 
 @dataclass
-class AzureCognitiveSearchAccessConfig(AccessConfig):
+class AzureAiSearchAccessConfig(AccessConfig):
     key: str = enhanced_field(sensitive=True)
 
 
 @dataclass
-class SimpleAzureCognitiveSearchStorageConfig(BaseConnectorConfig):
+class SimpleAzureAISearchStorageConfig(BaseConnectorConfig):
     endpoint: str
-    access_config: AzureCognitiveSearchAccessConfig
+    access_config: AzureAiSearchAccessConfig
 
 
 @dataclass
-class AzureCognitiveSearchWriteConfig(WriteConfig):
+class AzureAISearchWriteConfig(WriteConfig):
     index: str
 
 
 @dataclass
-class AzureCognitiveSearchDestinationConnector(BaseDestinationConnector):
-    write_config: AzureCognitiveSearchWriteConfig
-    connector_config: SimpleAzureCognitiveSearchStorageConfig
+class AzureAISearchDestinationConnector(BaseDestinationConnector):
+    write_config: AzureAISearchWriteConfig
+    connector_config: SimpleAzureAISearchStorageConfig
     _client: t.Optional["SearchClient"] = field(init=False, default=None)
 
-    @requires_dependencies(["azure.search"], extras="azure-cognitive-search")
+    @requires_dependencies(["azure.search"], extras="azure-ai-search")
     def generate_client(self) -> "SearchClient":
         from azure.core.credentials import AzureKeyCredential
         from azure.search.documents import SearchClient
@@ -112,7 +112,7 @@ class AzureCognitiveSearchDestinationConnector(BaseDestinationConnector):
         if page_number := data.get("metadata", {}).get("page_number"):
             data["metadata"]["page_number"] = str(page_number)
 
-    @requires_dependencies(["azure"], extras="azure-cognitive-search")
+    @requires_dependencies(["azure"], extras="azure-ai-search")
     def write_dict(self, *args, elements_dict: t.List[t.Dict[str, t.Any]], **kwargs) -> None:
         import azure.core.exceptions
 

unstructured_ingest/runner/writers/__init__.py

@@ -1,7 +1,7 @@
 import typing as t
 
 from .astradb import AstraDBWriter
-from .azure_cognitive_search import AzureCognitiveSearchWriter
+from .azure_ai_search import AzureAiSearchWriter
 from .base_writer import Writer
 from .chroma import ChromaWriter
 from .clarifai import ClarifaiWriter
@@ -25,7 +25,7 @@ from .weaviate import WeaviateWriter
 writer_map: t.Dict[str, t.Type[Writer]] = {
     "astradb": AstraDBWriter,
     "azure": AzureWriter,
-    "azure_cognitive_search": AzureCognitiveSearchWriter,
+    "azure_ai_search": AzureAiSearchWriter,
     "box": BoxWriter,
     "chroma": ChromaWriter,
     "clarifai": ClarifaiWriter,

unstructured_ingest/runner/writers/azure_ai_search.py

@@ -0,0 +1,24 @@
+import typing as t
+from dataclasses import dataclass
+
+from unstructured_ingest.interfaces import BaseDestinationConnector
+from unstructured_ingest.runner.writers.base_writer import Writer
+
+if t.TYPE_CHECKING:
+    from unstructured_ingest.connector.azure_ai_search import (
+        AzureAISearchWriteConfig,
+        SimpleAzureAISearchStorageConfig,
+    )
+
+
+@dataclass
+class AzureAiSearchWriter(Writer):
+    connector_config: "SimpleAzureAISearchStorageConfig"
+    write_config: "AzureAISearchWriteConfig"
+
+    def get_connector_cls(self) -> t.Type[BaseDestinationConnector]:
+        from unstructured_ingest.connector.azure_ai_search import (
+            AzureAISearchDestinationConnector,
+        )
+
+        return AzureAISearchDestinationConnector

unstructured_ingest/v2/constants.py

@@ -0,0 +1,2 @@
+# Used to append to metadata for uploaders that store element-level data
+RECORD_ID_LABEL = "record_id"