unstructured-ingest 0.6.4__py3-none-any.whl → 0.7.0__py3-none-any.whl

test/integration/embedders/test_togetherai.py

@@ -15,8 +15,8 @@ from unstructured_ingest.embed.togetherai import (
     TogetherAIEmbeddingConfig,
     TogetherAIEmbeddingEncoder,
 )
-from unstructured_ingest.v2.errors import UserAuthError
-from unstructured_ingest.v2.processes.embedder import Embedder, EmbedderConfig
+from unstructured_ingest.errors_v2 import UserAuthError
+from unstructured_ingest.processes.embedder import Embedder, EmbedderConfig
 
 API_KEY = "TOGETHERAI_API_KEY"
 

test/integration/embedders/test_vertexai.py

@@ -15,7 +15,7 @@ from unstructured_ingest.embed.vertexai import (
     VertexAIEmbeddingConfig,
     VertexAIEmbeddingEncoder,
 )
-from unstructured_ingest.v2.processes.embedder import Embedder, EmbedderConfig
+from unstructured_ingest.processes.embedder import Embedder, EmbedderConfig
 
 API_KEY = "VERTEXAI_API_KEY"
 

test/integration/embedders/test_voyageai.py

@@ -15,7 +15,7 @@ from unstructured_ingest.embed.voyageai import (
     VoyageAIEmbeddingConfig,
     VoyageAIEmbeddingEncoder,
 )
-from unstructured_ingest.v2.processes.embedder import Embedder, EmbedderConfig
+from unstructured_ingest.processes.embedder import Embedder, EmbedderConfig
 
 API_KEY = "VOYAGEAI_API_KEY"
 

test/integration/partitioners/test_partitioner.py

@@ -4,8 +4,8 @@ from pathlib import Path
 import pytest
 
 from test.integration.utils import requires_env
-from unstructured_ingest.v2.errors import UserError
-from unstructured_ingest.v2.processes.partitioner import Partitioner, PartitionerConfig
+from unstructured_ingest.errors_v2 import UserError
+from unstructured_ingest.processes.partitioner import Partitioner, PartitionerConfig
 
 int_test_dir = Path(__file__).parent
 assets_dir = int_test_dir / "assets"

test/unit/{v2/chunkers → chunkers}/test_chunkers.py

@@ -3,7 +3,7 @@ import random
 import faker
 import pytest
 
-from unstructured_ingest.v2.processes.chunker import Chunker, ChunkerConfig
+from unstructured_ingest.processes.chunker import Chunker, ChunkerConfig
 
 fake = faker.Faker()
 

test/unit/{v2/connectors → connectors}/ibm_watsonx/test_ibm_watsonx_s3.py

@@ -7,15 +7,15 @@ from pydantic import Secret
 from pyiceberg.exceptions import CommitFailedException
 from pytest_mock import MockerFixture
 
-from unstructured_ingest.v2.errors import ProviderError, UserError
-from unstructured_ingest.v2.processes.connectors.ibm_watsonx import IBM_WATSONX_S3_CONNECTOR_TYPE
-from unstructured_ingest.v2.processes.connectors.ibm_watsonx.ibm_watsonx_s3 import (
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.errors_v2 import ProviderError, UserError
+from unstructured_ingest.processes.connectors.ibm_watsonx import IBM_WATSONX_S3_CONNECTOR_TYPE
+from unstructured_ingest.processes.connectors.ibm_watsonx.ibm_watsonx_s3 import (
     IbmWatsonxAccessConfig,
     IbmWatsonxConnectionConfig,
     IbmWatsonxUploader,
     IbmWatsonxUploaderConfig,
 )
-from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture
@@ -138,7 +138,7 @@ def test_ibm_watsonx_connection_config_iceberg_url(
     connection_config: IbmWatsonxConnectionConfig,
 ):
     mocker.patch(
-        "unstructured_ingest.v2.processes.connectors.ibm_watsonx.ibm_watsonx_s3.DEFAULT_ICEBERG_URI_PATH",  # noqa: E501
+        "unstructured_ingest.processes.connectors.ibm_watsonx.ibm_watsonx_s3.DEFAULT_ICEBERG_URI_PATH",  # noqa: E501
         new="/mds/iceberg",
     )
     expected_url = "https://test_iceberg_endpoint/mds/iceberg"
@@ -218,7 +218,7 @@ def test_ibm_watsonx_connection_config_get_catalog_success(
     mocker: MockerFixture, connection_config: IbmWatsonxConnectionConfig
 ):
     mocker.patch(
-        "unstructured_ingest.v2.processes.connectors.ibm_watsonx.ibm_watsonx_s3.DEFAULT_ICEBERG_URI_PATH",  # noqa: E501
+        "unstructured_ingest.processes.connectors.ibm_watsonx.ibm_watsonx_s3.DEFAULT_ICEBERG_URI_PATH",  # noqa: E501
         new="/mds/iceberg",
     )
     mocker.patch.object(

test/unit/{v2/connectors → connectors}/motherduck/test_base.py

@@ -3,9 +3,9 @@ from pathlib import Path
 import pytest
 from pytest_mock import MockerFixture
 
-from unstructured_ingest.v2.interfaces.upload_stager import UploadStagerConfig
-from unstructured_ingest.v2.processes.connectors.duckdb.base import BaseDuckDBUploadStager
-from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.interfaces import UploadStagerConfig
+from unstructured_ingest.processes.connectors.duckdb.base import BaseDuckDBUploadStager
 
 
 @pytest.fixture
@@ -36,7 +36,7 @@ def test_run_output_filename_suffix(
 
     # Mocks
     mock_get_data = mocker.patch(
-        "unstructured_ingest.v2.processes.connectors.duckdb.base.get_data",
+        "unstructured_ingest.processes.connectors.duckdb.base.get_data",
         return_value=[{"key": "value"}, {"key": "value2"}],
     )
     mock_conform_dict = mocker.patch.object(
@@ -48,7 +48,7 @@ def test_run_output_filename_suffix(
         BaseDuckDBUploadStager, "get_output_path", return_value=output_dir / expected
     )
     mock_write_output = mocker.patch(
-        "unstructured_ingest.v2.processes.connectors.duckdb.base.write_data", return_value=None
+        "unstructured_ingest.processes.connectors.duckdb.base.write_data", return_value=None
     )
 
     # Act

test/unit/{v2/connectors → connectors}/sql/test_sql.py

@@ -4,13 +4,13 @@ import pandas as pd
 import pytest
 from pytest_mock import MockerFixture
 
-from unstructured_ingest.v2.processes.connectors.sql.sql import (
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
+from unstructured_ingest.processes.connectors.sql.sql import (
     SQLConnectionConfig,
     SQLUploader,
     SQLUploaderConfig,
     SQLUploadStager,
 )
-from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 @pytest.fixture
@@ -52,7 +52,7 @@ def test_run_output_filename_suffix(
 
     # Mocks
     mock_get_data = mocker.patch(
-        "unstructured_ingest.v2.processes.connectors.sql.sql.get_data",
+        "unstructured_ingest.processes.connectors.sql.sql.get_data",
         return_value=[{"key": "value"}, {"key": "value2"}],
     )
     mock_conform_dict = mocker.patch.object(
@@ -65,7 +65,7 @@ def test_run_output_filename_suffix(
         SQLUploadStager, "get_output_path", return_value=output_dir / expected
     )
     mock_write_output = mocker.patch(
-        "unstructured_ingest.v2.processes.connectors.sql.sql.write_data", return_value=None
+        "unstructured_ingest.processes.connectors.sql.sql.write_data", return_value=None
     )
 
     # Act

test/unit/{v2/connectors → connectors}/test_confluence.py

@@ -1,7 +1,7 @@
 import pytest
 from pydantic import ValidationError
 
-from unstructured_ingest.v2.processes.connectors.confluence import (
+from unstructured_ingest.processes.connectors.confluence import (
     ConfluenceAccessConfig,
     ConfluenceConnectionConfig,
 )

test/unit/{v2/connectors → connectors}/test_jira.py

@@ -4,7 +4,7 @@ import pytest
 from pydantic import ValidationError
 from pytest_mock import MockerFixture
 
-from unstructured_ingest.v2.processes.connectors.jira import (
+from unstructured_ingest.processes.connectors.jira import (
     FieldGetter,
     JiraAccessConfig,
     JiraConnectionConfig,

test/unit/{v2/embedders → embedders}/test_huggingface.py

@@ -4,7 +4,7 @@ from typing import Any
 import faker
 import pytest
 
-from test.unit.v2.utils.data_generator import generate_random_dictionary
+from test.unit.utils.data_generator import generate_random_dictionary
 from unstructured_ingest.embed.huggingface import (
     HuggingFaceEmbeddingConfig,
     HuggingFaceEmbeddingEncoder,

test/unit/{v2/embedders → embedders}/test_vertexai.py

@@ -5,7 +5,7 @@ from typing import Any
 import faker
 import pytest
 
-from test.unit.v2.utils.data_generator import generate_random_dictionary
+from test.unit.utils.data_generator import generate_random_dictionary
 from unstructured_ingest.embed.vertexai import VertexAIEmbeddingConfig, VertexAIEmbeddingEncoder
 
 fake = faker.Faker()

test/unit/{v2/partitioners → partitioners}/test_partitioner.py

@@ -4,8 +4,8 @@ from typing import Any
 import faker
 import pytest
 
-from test.unit.v2.utils.data_generator import generate_random_dictionary
-from unstructured_ingest.v2.processes.partitioner import Partitioner, PartitionerConfig
+from test.unit.utils.data_generator import generate_random_dictionary
+from unstructured_ingest.processes.partitioner import Partitioner, PartitionerConfig
 
 fake = faker.Faker()
 

test/unit/test_html.py

@@ -4,8 +4,8 @@ from pathlib import Path
 from bs4 import BeautifulSoup
 from pytest_mock import MockerFixture
 
+from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.utils.html import HtmlMixin
-from unstructured_ingest.v2.types.file_data import FileData, SourceIdentifiers
 
 
 def test_extract_images(mocker: MockerFixture):

test/unit/{v2/test_interfaces.py → test_interfaces.py}

@@ -1,7 +1,7 @@
 import pytest
 from pydantic import Secret, ValidationError
 
-from unstructured_ingest.v2.interfaces import AccessConfig, ConnectionConfig
+from unstructured_ingest.interfaces import AccessConfig, ConnectionConfig
 
 
 def test_failing_connection_config():

test/unit/test_utils.py

@@ -1,13 +1,19 @@
+import base64
 import json
-import typing as t
-from dataclasses import dataclass, field
+import zlib
 from datetime import datetime
+from typing import Any
 
 import pytest
 import pytz
+from pydantic import BaseModel, Field, Secret, SecretStr
+from pydantic.types import _SecretBase
 
-from unstructured_ingest.cli.utils import extract_config
-from unstructured_ingest.interfaces import BaseConfig
+from unstructured_ingest.processes.connectors.utils import format_and_truncate_orig_elements
+from unstructured_ingest.utils.pydantic_models import (
+    serialize_base_model,
+    serialize_base_model_json,
+)
 from unstructured_ingest.utils.string_and_date_utils import (
     ensure_isoformat_datetime,
     fix_unescaped_unicode,
@@ -15,102 +21,9 @@ from unstructured_ingest.utils.string_and_date_utils import (
     truncate_string_bytes,
 )
 
-
-@dataclass
-class A(BaseConfig):
-    a: str
-
-
-@dataclass
-class B(BaseConfig):
-    a: A
-    b: int
-
-
 flat_data = {"a": "test", "b": 4, "c": True}
 
 
-def test_extract_config_concrete():
-    @dataclass
-    class C(BaseConfig):
-        b: B
-        c: bool
-
-    c = extract_config(flat_data=flat_data, config=C)
-    expected_result = {"b": {"a": {"a": "test"}, "b": 4}, "c": True}
-    assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
-
-
-def test_extract_config_optional():
-    @dataclass
-    class C(BaseConfig):
-        c: bool
-        b: t.Optional[B] = None
-
-    c = extract_config(flat_data=flat_data, config=C)
-    expected_result = {"b": {"a": {"a": "test"}, "b": 4}, "c": True}
-    assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
-
-
-def test_extract_config_union():
-    @dataclass
-    class C(BaseConfig):
-        c: bool
-        b: t.Optional[t.Union[B, int]] = None
-
-    c = extract_config(flat_data=flat_data, config=C)
-    expected_result = {"b": 4, "c": True}
-    assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
-
-
-def test_extract_config_list():
-    @dataclass
-    class C(BaseConfig):
-        c: t.List[int]
-        b: B
-
-    flat_data = {"a": "test", "b": 4, "c": [1, 2, 3]}
-    c = extract_config(flat_data=flat_data, config=C)
-    expected_result = {"b": {"a": {"a": "test"}, "b": 4}, "c": [1, 2, 3]}
-    assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
-
-
-def test_extract_config_optional_list():
-    @dataclass
-    class C(BaseConfig):
-        b: B
-        c: t.Optional[t.List[int]] = None
-
-    flat_data = {"a": "test", "b": 4, "c": [1, 2, 3]}
-    c = extract_config(flat_data=flat_data, config=C)
-    expected_result = {"b": {"a": {"a": "test"}, "b": 4}, "c": [1, 2, 3]}
-    assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
-
-
-def test_extract_config_dataclass_list():
-    @dataclass
-    class C(BaseConfig):
-        c: bool
-        b: t.List[B] = field(default_factory=list)
-
-    flat_data = {"a": "test", "c": True}
-    c = extract_config(flat_data=flat_data, config=C)
-    expected_result = {"b": [], "c": True}
-    assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
-
-
-def test_extract_config_dict():
-    @dataclass
-    class C(BaseConfig):
-        c: bool
-        b: t.Dict[str, B] = field(default_factory=dict)
-
-    flat_data = {"c": True}
-    c = extract_config(flat_data=flat_data, config=C)
-    expected_result = {"c": True, "b": {}}
-    assert c.to_json(sort_keys=True) == json.dumps(expected_result, sort_keys=True)
-
-
 def test_json_to_dict_valid_json():
     json_string = '{"key": "value"}'
     expected_result = {"key": "value"}
@@ -209,3 +122,99 @@ def test_fix_unescaped_unicode_encoding_error(caplog: pytest.LogCaptureFixture):
     with caplog.at_level("WARNING"):
         fix_unescaped_unicode(text)
         assert "Failed to fix unescaped Unicode sequences" in caplog.text
+
+
+class MockChildBaseModel(BaseModel):
+    child_secret_str: SecretStr
+    child_secret_float: Secret[float]
+    child_not_secret_dict: dict[str, Any] = Field(default_factory=dict)
+
+
+class MockBaseModel(BaseModel):
+    secret_str: SecretStr
+    not_secret_bool: bool
+    secret_child_base: Secret[MockChildBaseModel]
+    not_secret_list: list[int] = Field(default_factory=list)
+
+
+model = MockBaseModel(
+    secret_str="secret string",
+    not_secret_bool=False,
+    secret_child_base=MockChildBaseModel(
+        child_secret_str="child secret string",
+        child_secret_float=3.14,
+        child_not_secret_dict={"key": "value"},
+    ),
+    not_secret_list=[1, 2, 3],
+)
+
+
+def test_serialize_base_model():
+
+    serialized_dict = model.model_dump()
+    assert isinstance(serialized_dict["secret_str"], _SecretBase)
+    assert isinstance(serialized_dict["secret_child_base"], _SecretBase)
+
+    serialized_dict_w_secrets = serialize_base_model(model=model)
+    assert not isinstance(serialized_dict_w_secrets["secret_str"], _SecretBase)
+    assert not isinstance(serialized_dict_w_secrets["secret_child_base"], _SecretBase)
+
+    expected_dict = {
+        "secret_str": "secret string",
+        "not_secret_bool": False,
+        "secret_child_base": {
+            "child_secret_str": "child secret string",
+            "child_secret_float": 3.14,
+            "child_not_secret_dict": {"key": "value"},
+        },
+        "not_secret_list": [1, 2, 3],
+    }
+
+    assert serialized_dict_w_secrets == expected_dict
+
+
+def test_serialize_base_model_json():
+    serialized_json = model.model_dump_json()
+    serialized_dict = json.loads(serialized_json)
+    expected_dict = {
+        "secret_str": "**********",
+        "not_secret_bool": False,
+        "secret_child_base": "**********",
+        "not_secret_list": [1, 2, 3],
+    }
+    assert expected_dict == serialized_dict
+
+    serialized_json_w_secrets = serialize_base_model_json(model=model)
+    serialized_dict_w_secrets = json.loads(serialized_json_w_secrets)
+    expected_dict_w_secrets = {
+        "secret_str": "secret string",
+        "not_secret_bool": False,
+        "secret_child_base": {
+            "child_secret_str": "child secret string",
+            "child_secret_float": 3.14,
+            "child_not_secret_dict": {"key": "value"},
+        },
+        "not_secret_list": [1, 2, 3],
+    }
+    assert expected_dict_w_secrets == serialized_dict_w_secrets
+
+
+def test_format_and_truncate_orig_elements():
+    original_elements = [
+        {
+            "text": "Hello, world!",
+            "metadata": {
+                "image_base64": "iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAABwUlEQVR42mNk",
+                "text_as_html": "<p>Hello, world!</p>",
+                "page": 1,
+            },
+        }
+    ]
+    json_bytes = json.dumps(original_elements, sort_keys=True).encode("utf-8")
+    deflated_bytes = zlib.compress(json_bytes)
+    b64_deflated_bytes = base64.b64encode(deflated_bytes)
+    b64_deflated_bytes.decode("utf-8")
+
+    assert format_and_truncate_orig_elements(
+        {"text": "Hello, world!", "metadata": {"orig_elements": b64_deflated_bytes.decode("utf-8")}}
+    ) == [{"metadata": {"page": 1}}]

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "0.6.4"  # pragma: no cover
+__version__ = "0.7.0"  # pragma: no cover

unstructured_ingest/cli/__init__.py

@@ -1,14 +0,0 @@
-import typing as t
-
-import click
-
-from unstructured_ingest.cli.cmds import base_dest_cmd_fns, base_src_cmd_fns
-
-src: t.List[click.Group] = [v().get_src_cmd() for v in base_src_cmd_fns]
-
-dest: t.List[click.Command] = [v().get_dest_cmd() for v in base_dest_cmd_fns]
-
-__all__ = [
-    "src",
-    "dest",
-]

unstructured_ingest/cli/base/__init__.py

@@ -0,0 +1,4 @@
+from .dest import DestCmd
+from .src import SrcCmd
+
+__all__ = ["SrcCmd", "DestCmd"]