unstructured-ingest 0.0.25__py3-none-any.whl → 0.1.1__py3-none-any.whl

test/unit/test_interfaces.py

@@ -0,0 +1,280 @@
+from __future__ import annotations
+
+import pathlib
+from dataclasses import dataclass
+from typing import Any, Dict
+
+import pytest
+from unstructured.documents.elements import DataSourceMetadata
+from unstructured.partition.auto import partition
+from unstructured.staging.base import elements_to_dicts
+
+from unstructured_ingest.interfaces import (
+    BaseConnectorConfig,
+    BaseSingleIngestDoc,
+    ChunkingConfig,
+    PartitionConfig,
+    ProcessorConfig,
+    ReadConfig,
+)
+
+DIRECTORY = pathlib.Path(__file__).parents[2].resolve()
+EXAMPLE_DOCS_DIRECTORY = DIRECTORY / "example-docs"
+TEST_DOWNLOAD_DIR = "/tmp"
+TEST_OUTPUT_DIR = "/tmp"
+TEST_ID = "test"
+TEST_FILE_PATH = str(EXAMPLE_DOCS_DIRECTORY / "book-war-and-peace-1p.txt")
+
+
+@dataclass
+class ExampleConfig(BaseConnectorConfig):
+    id: str
+    path: str
+
+
+TEST_CONFIG = ExampleConfig(id=TEST_ID, path=TEST_FILE_PATH)
+TEST_SOURCE_URL = "test-source-url"
+TEST_VERSION = "1.1.1"
+TEST_RECORD_LOCATOR = {"id": "data-source-id"}
+TEST_DATE_CREATED = "2021-01-01T00:00:00"
+TEST_DATE_MODIFIED = "2021-01-02T00:00:00"
+TEST_DATE_PROCESSED = "2022-12-13T15:44:08"
+
+
+@dataclass
+class ExampleIngestDoc(BaseSingleIngestDoc):
+    connector_config: ExampleConfig
+
+    @property
+    def filename(self):
+        return TEST_FILE_PATH
+
+    @property
+    def _output_filename(self):
+        return TEST_FILE_PATH + ".json"
+
+    @property
+    def source_url(self) -> str:
+        return TEST_SOURCE_URL
+
+    @property
+    def version(self) -> str:
+        return TEST_VERSION
+
+    @property
+    def record_locator(self) -> Dict[str, Any]:
+        return TEST_RECORD_LOCATOR
+
+    @property
+    def date_created(self) -> str:
+        return TEST_DATE_CREATED
+
+    @property
+    def date_modified(self) -> str:
+        return TEST_DATE_MODIFIED
+
+    @property
+    def exists(self) -> bool:
+        return True
+
+    def cleanup_file(self):
+        pass
+
+    def get_file(self):
+        pass
+
+    def has_output(self):
+        return True
+
+    def write_result(self, result):
+        pass
+
+
+@pytest.fixture
+def partition_test_results():
+    # Reusable partition test results, calculated only once
+    result = partition(
+        filename=str(TEST_FILE_PATH),
+        data_source_metadata=DataSourceMetadata(
+            url=TEST_SOURCE_URL,
+            version=TEST_VERSION,
+            record_locator=TEST_RECORD_LOCATOR,
+            date_created=TEST_DATE_CREATED,
+            date_modified=TEST_DATE_MODIFIED,
+            date_processed=TEST_DATE_PROCESSED,
+        ),
+    )
+    return result
+
+
+@pytest.fixture
+def partition_file_test_results(partition_test_results):
+    # Reusable partition_file test results, calculated only once
+    return elements_to_dicts(partition_test_results)
+
+
+def test_partition_file():
+    """Validate partition_file returns a list of dictionaries with the expected keys,
+    metadatakeys, and data source metadata values."""
+    test_ingest_doc = ExampleIngestDoc(
+        connector_config=TEST_CONFIG,
+        read_config=ReadConfig(download_dir=TEST_DOWNLOAD_DIR),
+        processor_config=ProcessorConfig(output_dir=TEST_OUTPUT_DIR),
+    )
+    test_ingest_doc._date_processed = TEST_DATE_PROCESSED
+    elements = test_ingest_doc.partition_file(partition_config=PartitionConfig())
+    element_dicts = elements_to_dicts(elements)
+    assert len(element_dicts)
+    expected_keys = {
+        "element_id",
+        "text",
+        "type",
+        "metadata",
+    }
+    # The document in TEST_FILE_PATH does not have elements with coordinates so
+    # partition is not expected to return coordinates metadata.
+    expected_metadata_keys = {
+        "data_source",
+        "filename",
+        "file_directory",
+        "filetype",
+        "languages",
+        "last_modified",
+    }
+    for elem in element_dicts:
+        # Parent IDs are non-deterministic - remove them from the test
+        elem["metadata"].pop("parent_id", None)
+
+        assert expected_keys == set(elem.keys())
+        assert expected_metadata_keys == set(elem["metadata"].keys())
+        data_source_metadata = elem["metadata"]["data_source"]
+        assert data_source_metadata["url"] == TEST_SOURCE_URL
+        assert data_source_metadata["version"] == TEST_VERSION
+        assert data_source_metadata["record_locator"] == TEST_RECORD_LOCATOR
+        assert data_source_metadata["date_created"] == TEST_DATE_CREATED
+        assert data_source_metadata["date_modified"] == TEST_DATE_MODIFIED
+        assert data_source_metadata["date_processed"] == TEST_DATE_PROCESSED
+
+
+def test_process_file_fields_include_default(mocker, partition_test_results):
+    """Validate when metadata_include and metadata_exclude are not set, all fields:
+    ("element_id", "text", "type", "metadata") are included"""
+    mock_partition = mocker.patch(
+        "unstructured.partition.auto.partition",
+        return_value=partition_test_results,
+    )
+    test_ingest_doc = ExampleIngestDoc(
+        connector_config=TEST_CONFIG,
+        read_config=ReadConfig(download_dir=TEST_DOWNLOAD_DIR),
+        processor_config=ProcessorConfig(output_dir=TEST_OUTPUT_DIR),
+    )
+    elements = test_ingest_doc.partition_file(partition_config=PartitionConfig())
+    element_dicts = elements_to_dicts(elements)
+    assert len(element_dicts)
+    assert mock_partition.call_count == 1
+    for elem in element_dicts:
+        # Parent IDs are non-deterministic - remove them from the test
+        elem["metadata"].pop("parent_id", None)
+
+        assert {"element_id", "text", "type", "metadata"} == set(elem.keys())
+        data_source_metadata = elem["metadata"]["data_source"]
+        assert data_source_metadata["url"] == TEST_SOURCE_URL
+        assert data_source_metadata["version"] == TEST_VERSION
+        assert data_source_metadata["record_locator"] == TEST_RECORD_LOCATOR
+        assert data_source_metadata["date_created"] == TEST_DATE_CREATED
+        assert data_source_metadata["date_modified"] == TEST_DATE_MODIFIED
+        assert data_source_metadata["date_processed"] == TEST_DATE_PROCESSED
+
+
+def test_process_file_metadata_includes_filename_and_filetype(
+    mocker,
+    partition_test_results,
+):
+    """Validate when metadata_include is set to "filename,filetype",
+    only filename is included in metadata"""
+    mocker.patch(
+        "unstructured.partition.auto.partition",
+        return_value=partition_test_results,
+    )
+    partition_config = PartitionConfig(
+        metadata_include=["filename", "filetype"],
+    )
+    test_ingest_doc = ExampleIngestDoc(
+        connector_config=TEST_CONFIG,
+        read_config=ReadConfig(download_dir=TEST_DOWNLOAD_DIR),
+        processor_config=ProcessorConfig(output_dir=TEST_OUTPUT_DIR),
+    )
+    isd_elems = test_ingest_doc.process_file(partition_config=partition_config)
+    assert len(isd_elems)
+    for elem in isd_elems:
+        # Parent IDs are non-deterministic - remove them from the test
+        elem["metadata"].pop("parent_id", None)
+
+        assert set(elem["metadata"].keys()) == {"filename", "filetype"}
+
+
+def test_process_file_metadata_exclude_filename_pagenum(mocker, partition_test_results):
+    """Validate when metadata_exclude is set to "filename,page_number",
+    neither filename nor page_number are included in metadata"""
+    mocker.patch(
+        "unstructured.partition.auto.partition",
+        return_value=partition_test_results,
+    )
+    partition_config = PartitionConfig(
+        metadata_exclude=["filename", "page_number"],
+    )
+    test_ingest_doc = ExampleIngestDoc(
+        connector_config=TEST_CONFIG,
+        read_config=ReadConfig(download_dir=TEST_DOWNLOAD_DIR),
+        processor_config=ProcessorConfig(
+            output_dir=TEST_OUTPUT_DIR,
+        ),
+    )
+    isd_elems = test_ingest_doc.process_file(partition_config=partition_config)
+    assert len(isd_elems)
+    for elem in isd_elems:
+        assert "filename" not in elem["metadata"]
+        assert "page_number" not in elem["metadata"]
+
+
+def test_process_file_flatten_metadata(mocker, partition_test_results):
+    mocker.patch(
+        "unstructured.partition.auto.partition",
+        return_value=partition_test_results,
+    )
+    partition_config = PartitionConfig(
+        metadata_include=["filename", "file_directory", "filetype"],
+        flatten_metadata=True,
+    )
+    test_ingest_doc = ExampleIngestDoc(
+        connector_config=TEST_CONFIG,
+        read_config=ReadConfig(download_dir=TEST_DOWNLOAD_DIR),
+        processor_config=ProcessorConfig(
+            output_dir=TEST_OUTPUT_DIR,
+        ),
+    )
+    isd_elems = test_ingest_doc.process_file(partition_config=partition_config)
+    expected_keys = {"element_id", "text", "type", "filename", "file_directory", "filetype"}
+    for elem in isd_elems:
+        assert expected_keys == set(elem.keys())
+
+
+class DescribeChunkingConfig:
+    """Unit tests for unstructured.ingest.interfaces.ChunkingConfig"""
+
+    def it_accepts_chunking_strategy_by_itself(self):
+        config = ChunkingConfig(chunking_strategy="basic")
+        assert config.chunking_strategy == "basic"
+
+    def it_defaults_to_chunk_by_title_if_only_chunk_elements_is_True(self):
+        config = ChunkingConfig(chunk_elements=True)
+        assert config.chunking_strategy == "by_title"
+
+    def but_it_defaults_to_chunking_strategy_over_chunk_elements(self):
+        config = ChunkingConfig(chunk_elements=True, chunking_strategy="basic")
+        assert config.chunking_strategy == "basic"
+
+    def it_silently_accepts_unrecognized_chunker(self, caplog: pytest.LogCaptureFixture):
+        config = ChunkingConfig(chunking_strategy="foobar")
+        assert config.chunking_strategy == "foobar"
+        assert caplog.text == ""

test/unit/test_interfaces_v2.py

@@ -0,0 +1,26 @@
+import pytest
+from pydantic import Secret, ValidationError
+
+from unstructured_ingest.v2.interfaces import AccessConfig, ConnectionConfig
+
+
+def test_failing_connection_config():
+    class MyAccessConfig(AccessConfig):
+        sensitive_value: str
+
+    class MyConnectionConfig(ConnectionConfig):
+        access_config: MyAccessConfig
+
+    with pytest.raises(ValidationError):
+        MyConnectionConfig(access_config=MyAccessConfig(sensitive_value="this"))
+
+
+def test_happy_path_connection_config():
+    class MyAccessConfig(AccessConfig):
+        sensitive_value: str
+
+    class MyConnectionConfig(ConnectionConfig):
+        access_config: Secret[MyAccessConfig]
+
+    connection_config = MyConnectionConfig(access_config=MyAccessConfig(sensitive_value="this"))
+    assert connection_config

test/unit/test_logger.py

@@ -0,0 +1,78 @@
+import json
+
+import pytest
+
+from unstructured_ingest.logger import (
+    default_is_data_sensitive,
+    hide_sensitive_fields,
+    redact_jsons,
+)
+
+
+@pytest.mark.parametrize(
+    ("key", "value", "is_sensitive"),
+    [
+        ("username", "john_smith", False),
+        ("password", "13?H%", True),
+        ("token", "123", True),
+        ("AWS_CREDENTIAL", "aws_credential", True),
+        ("AWS_KEY", None, False),
+    ],
+)
+def test_default_is_sensitive(key, value, is_sensitive):
+    assert default_is_data_sensitive(key, value) == is_sensitive
+
+
+def test_hide_sensitive_fields():
+    d = {
+        "username": "john_smith",
+        "password": "13?H%",
+        "inner": {
+            "token": "123",
+            "AWS_KEY": None,
+            "inner_j_string": json.dumps(
+                {"account_name": "secret name", "client_id": 123, "timestamp": 123}
+            ),
+        },
+    }
+    redacted_d = hide_sensitive_fields(d)
+    expected_d = {
+        "password": "*******",
+        "username": "john_smith",
+        "inner": {
+            "token": "*******",
+            "AWS_KEY": None,
+            "inner_j_string": json.dumps(
+                {"account_name": "*******", "client_id": "*******", "timestamp": 123}
+            ),
+        },
+    }
+    assert redacted_d == expected_d
+
+
+def test_redact_jsons():
+    d1 = {
+        "username": "john_smith",
+        "password": "13?H%",
+        "inner": {
+            "token": "123",
+            "AWS_KEY": None,
+            "inner_j_string": json.dumps(
+                {"account_name": "secret name", "client_id": 123, "timestamp": 123}
+            ),
+        },
+    }
+
+    d2 = {"username": "tim67", "update_time": 456}
+    d3 = {"account_name": "top secret", "host": "http://localhost:8888"}
+
+    sensitive_string = f"Some topic secret info ({json.dumps(d1)} regarding {d2} and {d3})"
+    expected_string = (
+        'Some topic secret info ({"username": "john_smith", "password": "*******", '
+        '"inner": {"token": "*******", "AWS_KEY": null, "inner_j_string": '
+        '"{\\"account_name\\": \\"*******\\", \\"client_id\\": \\"*******\\", '
+        '\\"timestamp\\": 123}"}} regarding {"username": "tim67", "update_time": 456} '
+        'and {"account_name": "*******", "host": "http://localhost:8888"})'
+    )
+    redacted_string = redact_jsons(sensitive_string)
+    assert redacted_string == expected_string

test/unit/test_utils.py

@@ -0,0 +1,164 @@
+import json
+import typing as t
+from dataclasses import dataclass, field
+from datetime import datetime
+
+import pytest
+import pytz
+
+from unstructured_ingest.cli.utils import extract_config
+from unstructured_ingest.interfaces import BaseConfig
+from unstructured_ingest.utils.string_and_date_utils import ensure_isoformat_datetime, json_to_dict
+
+
+@dataclass
+class A(BaseConfig):
+    a: str
+
+
+@dataclass
+class B(BaseConfig):
+    a: A
+    b: int
+
+
+flat_data = {"a": "test", "b": 4, "c": True}
+
+
+def test_extract_config_concrete():
+    @dataclass
+    class C(BaseConfig):
+        b: B
+        c: bool
+
+    c = extract_config(flat_data=flat_data, config=C)
+    expected_result = {"b": {"a": {"a": "test"}, "b": 4}, "c": True}
+    assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
+
+
+def test_extract_config_optional():
+    @dataclass
+    class C(BaseConfig):
+        c: bool
+        b: t.Optional[B] = None
+
+    c = extract_config(flat_data=flat_data, config=C)
+    expected_result = {"b": {"a": {"a": "test"}, "b": 4}, "c": True}
+    assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
+
+
+def test_extract_config_union():
+    @dataclass
+    class C(BaseConfig):
+        c: bool
+        b: t.Optional[t.Union[B, int]] = None
+
+    c = extract_config(flat_data=flat_data, config=C)
+    expected_result = {"b": 4, "c": True}
+    assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
+
+
+def test_extract_config_list():
+    @dataclass
+    class C(BaseConfig):
+        c: t.List[int]
+        b: B
+
+    flat_data = {"a": "test", "b": 4, "c": [1, 2, 3]}
+    c = extract_config(flat_data=flat_data, config=C)
+    expected_result = {"b": {"a": {"a": "test"}, "b": 4}, "c": [1, 2, 3]}
+    assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
+
+
+def test_extract_config_optional_list():
+    @dataclass
+    class C(BaseConfig):
+        b: B
+        c: t.Optional[t.List[int]] = None
+
+    flat_data = {"a": "test", "b": 4, "c": [1, 2, 3]}
+    c = extract_config(flat_data=flat_data, config=C)
+    expected_result = {"b": {"a": {"a": "test"}, "b": 4}, "c": [1, 2, 3]}
+    assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
+
+
+def test_extract_config_dataclass_list():
+    @dataclass
+    class C(BaseConfig):
+        c: bool
+        b: t.List[B] = field(default_factory=list)
+
+    flat_data = {"a": "test", "c": True}
+    c = extract_config(flat_data=flat_data, config=C)
+    expected_result = {"b": [], "c": True}
+    assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
+
+
+def test_extract_config_dict():
+    @dataclass
+    class C(BaseConfig):
+        c: bool
+        b: t.Dict[str, B] = field(default_factory=dict)
+
+    flat_data = {"c": True}
+    c = extract_config(flat_data=flat_data, config=C)
+    expected_result = {"c": True, "b": {}}
+    assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
+
+
+def test_json_to_dict_valid_json():
+    json_string = '{"key": "value"}'
+    expected_result = {"key": "value"}
+    assert json_to_dict(json_string) == expected_result
+    assert isinstance(json_to_dict(json_string), dict)
+
+
+def test_json_to_dict_malformed_json():
+    json_string = '{"key": "value"'
+    expected_result = '{"key": "value"'
+    assert json_to_dict(json_string) == expected_result
+    assert isinstance(json_to_dict(json_string), str)
+
+
+def test_json_to_dict_single_quotes():
+    json_string = "{'key': 'value'}"
+    expected_result = {"key": "value"}
+    assert json_to_dict(json_string) == expected_result
+    assert isinstance(json_to_dict(json_string), dict)
+
+
+def test_json_to_dict_path():
+    json_string = "/path/to/file.json"
+    expected_result = "/path/to/file.json"
+    assert json_to_dict(json_string) == expected_result
+    assert isinstance(json_to_dict(json_string), str)
+
+
+def test_ensure_isoformat_datetime_for_datetime():
+    dt = ensure_isoformat_datetime(datetime(2021, 1, 1, 12, 0, 0))
+    assert dt == "2021-01-01T12:00:00"
+
+
+def test_ensure_isoformat_datetime_for_datetime_with_tz():
+    dt = ensure_isoformat_datetime(datetime(2021, 1, 1, 12, 0, 0, tzinfo=pytz.UTC))
+    assert dt == "2021-01-01T12:00:00+00:00"
+
+
+def test_ensure_isoformat_datetime_for_string():
+    dt = ensure_isoformat_datetime("2021-01-01T12:00:00")
+    assert dt == "2021-01-01T12:00:00"
+
+
+def test_ensure_isoformat_datetime_for_string2():
+    dt = ensure_isoformat_datetime("2021-01-01T12:00:00+00:00")
+    assert dt == "2021-01-01T12:00:00+00:00"
+
+
+def test_ensure_isoformat_datetime_fails_on_string():
+    with pytest.raises(ValueError):
+        ensure_isoformat_datetime("bad timestamp")
+
+
+def test_ensure_isoformat_datetime_fails_on_int():
+    with pytest.raises(TypeError):
+        ensure_isoformat_datetime(1111)

test/unit/test_utils_v2.py

@@ -0,0 +1,82 @@
+import json
+from typing import Any
+
+from pydantic import BaseModel, Field, Secret, SecretStr
+from pydantic.types import _SecretBase
+
+from unstructured_ingest.v2.utils import serialize_base_model, serialize_base_model_json
+
+
+class MockChildBaseModel(BaseModel):
+    child_secret_str: SecretStr
+    child_secret_float: Secret[float]
+    child_not_secret_dict: dict[str, Any] = Field(default_factory=dict)
+
+
+class MockBaseModel(BaseModel):
+    secret_str: SecretStr
+    not_secret_bool: bool
+    secret_child_base: Secret[MockChildBaseModel]
+    not_secret_list: list[int] = Field(default_factory=list)
+
+
+model = MockBaseModel(
+    secret_str="secret string",
+    not_secret_bool=False,
+    secret_child_base=MockChildBaseModel(
+        child_secret_str="child secret string",
+        child_secret_float=3.14,
+        child_not_secret_dict={"key": "value"},
+    ),
+    not_secret_list=[1, 2, 3],
+)
+
+
+def test_serialize_base_model():
+
+    serialized_dict = model.model_dump()
+    assert isinstance(serialized_dict["secret_str"], _SecretBase)
+    assert isinstance(serialized_dict["secret_child_base"], _SecretBase)
+
+    serialized_dict_w_secrets = serialize_base_model(model=model)
+    assert not isinstance(serialized_dict_w_secrets["secret_str"], _SecretBase)
+    assert not isinstance(serialized_dict_w_secrets["secret_child_base"], _SecretBase)
+
+    expected_dict = {
+        "secret_str": "secret string",
+        "not_secret_bool": False,
+        "secret_child_base": {
+            "child_secret_str": "child secret string",
+            "child_secret_float": 3.14,
+            "child_not_secret_dict": {"key": "value"},
+        },
+        "not_secret_list": [1, 2, 3],
+    }
+
+    assert serialized_dict_w_secrets == expected_dict
+
+
+def test_serialize_base_model_json():
+    serialized_json = model.model_dump_json()
+    serialized_dict = json.loads(serialized_json)
+    expected_dict = {
+        "secret_str": "**********",
+        "not_secret_bool": False,
+        "secret_child_base": "**********",
+        "not_secret_list": [1, 2, 3],
+    }
+    assert expected_dict == serialized_dict
+
+    serialized_json_w_secrets = serialize_base_model_json(model=model)
+    serialized_dict_w_secrets = json.loads(serialized_json_w_secrets)
+    expected_dict_w_secrets = {
+        "secret_str": "secret string",
+        "not_secret_bool": False,
+        "secret_child_base": {
+            "child_secret_str": "child secret string",
+            "child_secret_float": 3.14,
+            "child_not_secret_dict": {"key": "value"},
+        },
+        "not_secret_list": [1, 2, 3],
+    }
+    assert expected_dict_w_secrets == serialized_dict_w_secrets

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.0.25"  # pragma: no cover
+__version__ = "0.1.1"  # pragma: no cover

unstructured_ingest/cli/interfaces.py

@@ -341,9 +341,9 @@ class CliPartitionConfig(PartitionConfig, CliMixin):
             ),
             click.Option(
                 ["--partition-endpoint"],
-                default="https://api.unstructured.io/general/v0/general",
+                default="https://api.unstructuredapp.io/general/v0/general",
                 help="If partitioning via api, use the following host. "
-                "Default: https://api.unstructured.io/general/v0/general",
+                "Default: https://api.unstructuredapp.io/general/v0/general",
             ),
             click.Option(
                 ["--api-key"],

unstructured_ingest/connector/notion/types/block.py

@@ -58,6 +58,7 @@ class Block(FromJSONMixin, GetHTMLMixin):
     last_edited_time: str
     last_edited_by: PartialUser
     archived: bool
+    in_trash: bool
     has_children: bool
     parent: Parent
     block: BlockBase

unstructured_ingest/connector/notion/types/database.py

@@ -26,6 +26,7 @@ class Database(FromJSONMixin, GetHTMLMixin):
     last_edited_time: str
     last_edited_by: PartialUser
     archived: bool
+    in_trash: bool
     parent: Parent
     url: str
     is_inline: bool

unstructured_ingest/connector/notion/types/page.py

@@ -16,6 +16,7 @@ class Page(FromJSONMixin):
     last_edited_time: str
     last_edited_by: PartialUser
     archived: bool
+    in_trash: bool
     properties: dict
     parent: Parent
     url: str