deltacat 1.1.35__py3-none-any.whl → 2.0__py3-none-any.whl

deltacat/tests/storage/model/test_transaction.py

@@ -0,0 +1,308 @@
+import pytest
+
+from deltacat.storage import (
+    Transaction,
+    TransactionOperation,
+    TransactionType,
+    TransactionOperationType,
+)
+from deltacat.storage.model.metafile import (
+    Metafile,
+)
+
+
+class TestAbsToRelative:
+    @classmethod
+    def setup_method(cls):
+        cls.catalog_root = "/catalog/root/path"
+
+    # Test cases for the abs_to_relative function
+    def test_abs_to_relative_simple(self):
+        """
+        Tests the function which relativizes absolute paths (string) into relative paths (string)
+        """
+        catalog_root = TestAbsToRelative.catalog_root
+        absolute_path = "/catalog/root/path/namespace/table/table_version/stream_id/partition_id/00000000000000000001.mpk"
+        relative_path = Transaction._abs_txn_meta_path_to_relative(
+            catalog_root, absolute_path
+        )
+        assert (
+            relative_path
+            == "namespace/table/table_version/stream_id/partition_id/00000000000000000001.mpk"
+        )
+
+    def test_abs_to_relative_same_paths(self):
+        catalog_root = TestAbsToRelative.catalog_root
+        absolute_path = TestAbsToRelative.catalog_root
+        with pytest.raises(
+            ValueError,
+            match="Target and root are identical, but expected target to be a child of root.",
+        ):
+            Transaction._abs_txn_meta_path_to_relative(catalog_root, absolute_path)
+
+    def test_abs_to_relative_root_with_trailing_slash(self):
+        catalog_root = "/catalog/root/path/"
+        absolute_path = "/catalog/root/path/namespace/table/table_version/stream_id/partition_id/00000000000000000001.mpk"
+        relative_path = Transaction._abs_txn_meta_path_to_relative(
+            catalog_root, absolute_path
+        )
+        assert (
+            relative_path
+            == "namespace/table/table_version/stream_id/partition_id/00000000000000000001.mpk"
+        )
+
+    def test_abs_to_relative_bad_root(self):
+        catalog_root = TestAbsToRelative.catalog_root
+        absolute_path = "/cat/rt/pth/namespace/table/table_version/stream_id/partition_id/00000000000000000001.mpk"
+        with pytest.raises(ValueError, match="Expected target to be a child of root."):
+            Transaction._abs_txn_meta_path_to_relative(catalog_root, absolute_path)
+
+    def test_abs_to_relative_empty_path(self):
+        with pytest.raises(ValueError, match="Expected target to be a child of root."):
+            Transaction._abs_txn_meta_path_to_relative("", "/lorem/ipsum")
+        with pytest.raises(ValueError, match="Expected target to be a child of root."):
+            Transaction._abs_txn_meta_path_to_relative("/lorem/ipsum/", "")
+
+    # Test cases for the relativize_operation_paths function
+    def test_relativize_metafile_write_paths(self):
+        catalog_root = "/catalog/root"
+        absolute_paths = [
+            "/catalog/root/path/to/metafile1.mpk",
+            "/catalog/root/path/to/metafile2.mpk",
+            "/catalog/root/another/path/lore_ipsum.mpk",
+            "/catalog/root/another/path/meta/to/lorem_ipsum.mpk",
+            "/catalog/root/another/path/lorem_ipsum.mpk",
+            "/catalog/root/here.mpk",
+        ]
+        expected_relative_paths = [
+            "path/to/metafile1.mpk",
+            "path/to/metafile2.mpk",
+            "another/path/lore_ipsum.mpk",
+            "another/path/meta/to/lorem_ipsum.mpk",
+            "another/path/lorem_ipsum.mpk",
+            "here.mpk",
+        ]
+        # Create a dummy transaction operation with absolute paths
+        dest_metafile = Metafile({"id": "dummy_metafile_id"})
+        transaction_operation = TransactionOperation.of(
+            operation_type=TransactionOperationType.CREATE,
+            dest_metafile=dest_metafile,
+        )
+        # use replace method as setter
+        transaction_operation.metafile_write_paths = absolute_paths
+        # Create a transaction and relativize paths
+        transaction = Transaction.of(
+            txn_type=TransactionType.APPEND, txn_operations=[transaction_operation]
+        )
+        transaction.relativize_operation_paths(transaction_operation, catalog_root)
+        # Verify the paths have been correctly relativized
+        assert transaction_operation.metafile_write_paths == expected_relative_paths
+
+    def test_relativize_locator_write_paths(self):
+        catalog_root = "/catalog/root"
+        absolute_paths = [
+            "/catalog/root/path/to/loc1.mpk",
+            "/catalog/root/path/to/loc2.mpk",
+            "/catalog/root/another/path/lore_ipsum.mpk",
+            "/catalog/root/another/path/meta/to/lorem_ipsum.mpk",
+            "/catalog/root/another/path/lorem_ipsum.mpk",
+            "/catalog/root/here.mpk",
+        ]
+        expected_relative_paths = [
+            "path/to/loc1.mpk",
+            "path/to/loc2.mpk",
+            "another/path/lore_ipsum.mpk",
+            "another/path/meta/to/lorem_ipsum.mpk",
+            "another/path/lorem_ipsum.mpk",
+            "here.mpk",
+        ]
+        # Create a dummy transaction operation with absolute paths
+        dest_metafile = Metafile({"id": "dummy_metafile_id"})
+        transaction_operation = TransactionOperation.of(
+            operation_type=TransactionOperationType.CREATE,
+            dest_metafile=dest_metafile,
+        )
+        # use replace as setter
+        transaction_operation.locator_write_paths = absolute_paths
+        # Create a transaction and relativize paths
+        transaction = Transaction.of(
+            txn_type=TransactionType.APPEND, txn_operations=[transaction_operation]
+        )
+        transaction.relativize_operation_paths(transaction_operation, catalog_root)
+        # Verify the paths have been correctly relativized
+        assert transaction_operation.locator_write_paths == expected_relative_paths
+
+    def test_relativize_metafile_and_locator_paths(self):
+        catalog_root = "/meta_catalog/root_dir/a/b/c"
+        meta_absolute_paths = [
+            "/meta_catalog/root_dir/a/b/c/namespace/table/table_version/stream_id/partition_id/00000000000000000001.mpk",
+            "/meta_catalog/root_dir/a/b/c/namespace/table/table_version/stream_id/partition_id/00000000000000000002.mpk",
+            "/meta_catalog/root_dir/a/b/c/namespace/table/table_version/stream_id/partition_id/00000000000000000003.mpk",
+        ]
+        loc_absolute_paths = [
+            "/meta_catalog/root_dir/a/b/c/d/table/table_version/stream_id/partition_id/00000000000000000001.mpk",
+            "/meta_catalog/root_dir/a/b/c/e/table/table_version/stream_id/partition_id/00000000000000000002.mpk",
+            "/meta_catalog/root_dir/a/b/c/f/table/table_version/stream_id/partition_id/00000000000000000003.mpk",
+        ]
+        meta_relative_paths = [
+            "namespace/table/table_version/stream_id/partition_id/00000000000000000001.mpk",
+            "namespace/table/table_version/stream_id/partition_id/00000000000000000002.mpk",
+            "namespace/table/table_version/stream_id/partition_id/00000000000000000003.mpk",
+        ]
+        loc_relative_paths = [
+            "d/table/table_version/stream_id/partition_id/00000000000000000001.mpk",
+            "e/table/table_version/stream_id/partition_id/00000000000000000002.mpk",
+            "f/table/table_version/stream_id/partition_id/00000000000000000003.mpk",
+        ]
+        # Create a dummy transaction operation with absolute paths
+        dest_metafile = Metafile({"id": "dummy_metafile_id"})
+        transaction_operation = TransactionOperation.of(
+            operation_type=TransactionOperationType.CREATE,
+            dest_metafile=dest_metafile,
+        )
+        # use replace as setter
+        transaction_operation.metafile_write_paths = meta_absolute_paths
+        transaction_operation.locator_write_paths = loc_absolute_paths
+        # Create a transaction and relativize paths
+        transaction = Transaction.of(
+            txn_type=TransactionType.APPEND, txn_operations=[transaction_operation]
+        )
+        transaction.relativize_operation_paths(transaction_operation, catalog_root)
+        # Verify the paths have been correctly relativized
+        assert (
+            transaction_operation.metafile_write_paths == meta_relative_paths
+        ), f"Expected: {meta_relative_paths}, but got: {transaction_operation.metafile_write_paths}"
+        assert (
+            transaction_operation.locator_write_paths == loc_relative_paths
+        ), f"Expected: {loc_relative_paths}, but got: {transaction_operation.locator_write_paths}"
+
+    def test_multiple_operations_relativize_paths(self):
+        catalog_root = "/catalog/root"
+        meta_absolute_paths = [
+            "/catalog/root/path/to/metafile1.mpk",
+            "/catalog/root/path/to/metafile2.mpk",
+            "/catalog/root/another/path/lore_ipsum.mpk",
+            "/catalog/root/another/path/meta/to/lorem_ipsum.mpk",
+            "/catalog/root/another/path/lorem_ipsum.mpk",
+            "/catalog/root/here.mpk",
+        ]
+        loc_absolute_paths = [
+            "/catalog/root/path/to/loc1.mpk",
+            "/catalog/root/path/to/loc2.mpk",
+            "/catalog/root/another/path/lore_ipsum.mpk",
+            "/catalog/root/another/path/meta/to/lorem_ipsum.mpk",
+            "/catalog/root/another/path/lorem_ipsum.mpk",
+            "/catalog/root/here.mpk",
+        ]
+        meta_expected_relative_paths = [
+            "path/to/metafile1.mpk",
+            "path/to/metafile2.mpk",
+            "another/path/lore_ipsum.mpk",
+            "another/path/meta/to/lorem_ipsum.mpk",
+            "another/path/lorem_ipsum.mpk",
+            "here.mpk",
+        ]
+        loc_expected_relative_paths = [
+            "path/to/loc1.mpk",
+            "path/to/loc2.mpk",
+            "another/path/lore_ipsum.mpk",
+            "another/path/meta/to/lorem_ipsum.mpk",
+            "another/path/lorem_ipsum.mpk",
+            "here.mpk",
+        ]
+        # Create a dummy transaction operation with absolute paths
+        dest_metafile = Metafile({"id": "dummy_metafile_id"})
+        transaction_operations = []
+        for i in range(11):
+            transaction_operation = TransactionOperation.of(
+                operation_type=TransactionOperationType.CREATE,
+                dest_metafile=dest_metafile,
+            )
+            transaction_operation.metafile_write_paths = meta_absolute_paths
+            transaction_operation.locator_write_paths = loc_absolute_paths
+            transaction_operations.append(transaction_operation)
+        # Create a transaction and relativize paths
+        transaction = Transaction.of(
+            txn_type=TransactionType.APPEND, txn_operations=transaction_operations
+        )
+        for operation in transaction_operations:
+            transaction.relativize_operation_paths(operation, catalog_root)
+        # Verify the paths have been correctly relativized
+        for operation in transaction_operations:
+            assert operation.metafile_write_paths == meta_expected_relative_paths
+            assert operation.locator_write_paths == loc_expected_relative_paths
+
+    def test_empty_metafile_and_locator_write_paths(self):
+        catalog_root = "/catalog/root"
+        transaction_operation = TransactionOperation.of(
+            operation_type=TransactionOperationType.CREATE,
+            dest_metafile=Metafile({"id": "dummy_metafile_id"}),
+        )
+        # Empty paths
+        transaction_operation.metafile_write_paths = []
+        transaction_operation.locator_write_paths = []
+        transaction = Transaction.of(
+            txn_type=TransactionType.APPEND, txn_operations=[transaction_operation]
+        )
+        transaction.relativize_operation_paths(transaction_operation, catalog_root)
+        assert transaction_operation.metafile_write_paths == []
+        assert transaction_operation.locator_write_paths == []
+
+    def test_large_number_of_paths(self):
+        catalog_root = "/catalog/root"
+        absolute_paths = [f"/catalog/root/path/to/file{i}.mpk" for i in range(5000)]
+        expected_paths = [f"path/to/file{i}.mpk" for i in range(5000)]
+        transaction_operation = TransactionOperation.of(
+            operation_type=TransactionOperationType.CREATE,
+            dest_metafile=Metafile({"id": "dummy_metafile_id"}),
+        )
+        transaction_operation.metafile_write_paths = absolute_paths
+        transaction = Transaction.of(
+            txn_type=TransactionType.APPEND, txn_operations=[transaction_operation]
+        )
+        transaction.relativize_operation_paths(transaction_operation, catalog_root)
+        assert transaction_operation.metafile_write_paths == expected_paths
+
+    def test_large_number_of_paths_multi_ops(self):
+        catalog_root = "/catalog/root"
+        absolute_paths = [f"/catalog/root/path/to/file{i}.mpk" for i in range(1000)]
+        expected_paths = [f"path/to/file{i}.mpk" for i in range(1000)]
+
+        # Different operation types to test
+        operation_types = [
+            TransactionOperationType.CREATE,
+            # TransactionOperationType.UPDATE,
+            TransactionOperationType.DELETE,
+            TransactionOperationType.READ_EXISTS,
+            TransactionOperationType.READ_LATEST,
+            TransactionOperationType.READ_CHILDREN,
+            TransactionOperationType.READ_SIBLINGS,
+        ]
+
+        # Different transaction types to test
+        txn_types = [
+            TransactionType.APPEND,
+            TransactionType.ALTER,
+            TransactionType.DELETE,
+            TransactionType.OVERWRITE,
+            TransactionType.READ,
+            TransactionType.RESTATE,
+        ]
+
+        for txn_type in txn_types:
+            transaction_ops = []
+            for op_type in operation_types:
+                transaction_operation = TransactionOperation.of(
+                    operation_type=op_type,
+                    dest_metafile=Metafile({"id": "dummy_metafile_id"}),
+                )
+                transaction_operation.metafile_write_paths = absolute_paths
+                transaction_ops.append(transaction_operation)
+            transaction = Transaction.of(
+                txn_type=txn_type, txn_operations=[transaction_operation]
+            )
+            transaction.relativize_operation_paths(transaction_operation, catalog_root)
+            # Assert paths are relativized correctly
+            assert (
+                transaction_operation.metafile_write_paths == expected_paths
+            ), f"Failed for transaction type {txn_type} and operation type {op_type}"

deltacat/tests/storage/rivulet/conftest.py

@@ -0,0 +1,149 @@
+import io
+
+import pytest
+from faker import Faker
+
+from deltacat.storage.rivulet.schema.datatype import Datatype
+from deltacat.storage.rivulet.mvp.Table import MvpTable
+from deltacat.storage.rivulet.schema.schema import Schema
+import random
+import string
+from PIL import Image
+
+FIXTURE_ROW_COUNT = 10000
+
+
+@pytest.fixture
+def ds1_dataset() -> MvpTable:
+    """
+    dataset with one million rows
+    primary key is integer between 1 and 1,000,000
+
+    TODO change to user Faker instead of int ranges
+    """
+
+    # Function to generate random names
+    def generate_random_name():
+        return "".join(
+            random.choices(
+                string.ascii_uppercase + string.ascii_lowercase, k=random.randint(3, 10)
+            )
+        )
+
+    # Create a list of numbers from 1 to TEST_ROW_COUNT
+    ids = list(range(1, FIXTURE_ROW_COUNT + 1))
+    random.shuffle(ids)
+
+    # Generate one million rows
+    return MvpTable(
+        {
+            "id": ids,
+            "name": [generate_random_name() for _ in range(FIXTURE_ROW_COUNT)],
+            "age": [random.randint(18, 100) for _ in range(FIXTURE_ROW_COUNT)],
+        }
+    )
+
+
+@pytest.fixture
+def ds1_schema():
+    return Schema(
+        {
+            ("id", Datatype.int32()),
+            ("name", Datatype.string()),
+            ("age", Datatype.int32()),
+        },
+        "id",
+    )
+
+
+@pytest.fixture
+def ds2_dataset():
+    """
+    dataset2 with one million rows that can be joined to ds1
+    primary key is integer between 1 and 1,000,000
+    """
+    # Create a list of numbers from 1 to 1,000,000
+    ids = list(range(1, FIXTURE_ROW_COUNT + 1))
+    random.shuffle(ids)
+
+    fake = Faker()
+
+    # Generate one million rows
+    return MvpTable(
+        {
+            "id": ids,
+            "address": [fake.address() for _ in range(FIXTURE_ROW_COUNT)],
+            "zip": [fake.zipcode() for _ in range(FIXTURE_ROW_COUNT)],
+        }
+    )
+
+
+@pytest.fixture
+def ds2_schema():
+    return Schema(
+        {
+            ("id", Datatype.int32()),
+            ("address", Datatype.string()),
+            ("zip", Datatype.string()),
+        },
+        "id",
+    )
+
+
+@pytest.fixture
+def combined_schema(ds1_schema, ds2_schema):
+    return Schema(
+        {
+            ("id", Datatype.int32()),
+            ("address", Datatype.string()),
+            ("zip", Datatype.string()),
+            ("name", Datatype.string()),
+            ("age", Datatype.int32()),
+        },
+        "id",
+    )
+
+
+@pytest.fixture
+def dataset_images_with_label() -> (MvpTable, Schema):
+    """
+    dataset with one thousand images and labels, generated dynamically
+    primary key is integer between 1 and 1,000
+    """
+    ROW_COUNT = 1000
+    fake = Faker()
+    schema = Schema(
+        {
+            ("id", Datatype.int32()),
+            ("image", Datatype.image("jpg")),
+            ("label", Datatype.string()),
+        },
+        "id",
+    )
+
+    # Create a list of numbers from 1 to ROW_COUNT
+    ids = list(range(1, ROW_COUNT + 1))
+    random.shuffle(ids)
+
+    fake_image = Image.new(
+        "RGB",
+        (512, 512),
+        color=(random.randint(0, 255), random.randint(0, 255), random.randint(0, 255)),
+    )
+    # get bytes from image encoded as png
+    buffer = io.BytesIO()
+    fake_image.save(buffer, format="PNG")
+    # seek to start of buffer since we just wrote to it
+    buffer.seek(0)
+    image_bytes = buffer.read()
+    # Generate one million rows
+    return (
+        MvpTable(
+            {
+                "id": ids,
+                "image": [image_bytes for _ in range(ROW_COUNT)],
+                "label": [fake.name() for _ in range(ROW_COUNT)],
+            }
+        ),
+        schema,
+    )

deltacat/tests/storage/rivulet/fs/test_file_location_provider.py

@@ -0,0 +1,93 @@
+import pytest
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+from deltacat import Datatype, Dataset
+from deltacat.storage.rivulet import Schema, Field
+from deltacat.utils.metafile_locator import _find_partition_path
+
+
+@pytest.fixture
+def sample_schema():
+    return Schema(
+        fields=[
+            Field("id", Datatype.int32(), is_merge_key=True),
+            Field("name", Datatype.string()),
+            Field("age", Datatype.int32()),
+        ]
+    )
+
+
+@pytest.fixture
+def sample_pydict():
+    return {"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"], "age": [25, 30, 35]}
+
+
+@pytest.fixture
+def temp_storage_path(tmp_path):
+    return tmp_path
+
+
+@pytest.fixture
+def sample_parquet_data(temp_storage_path, sample_pydict):
+    parquet_path = temp_storage_path / "test.parquet"
+    table = pa.Table.from_pydict(sample_pydict)
+    pq.write_table(table, parquet_path)
+    return parquet_path
+
+
+@pytest.fixture
+def dataset(sample_parquet_data):
+    return Dataset.from_parquet(
+        file_uri=sample_parquet_data, name="dataset", merge_keys="id"
+    )
+
+
+@pytest.fixture
+def file_provider(dataset):
+    return dataset._file_provider
+
+
+def test_provide_data_file(file_provider):
+    output_file = file_provider.provide_data_file("parquet")
+    assert "data" in output_file.location
+    assert output_file.location.endswith(".parquet")
+
+    output_file2 = file_provider.provide_data_file("parquet")
+    assert "data" in output_file2.location
+    assert output_file2.location.endswith(".parquet")
+
+    assert (
+        output_file.location != output_file2.location
+    ), "Two output files should have different locations."
+
+
+def test_provide_manifest_file(file_provider):
+    output_file = file_provider.provide_manifest_file()
+    assert "metadata/manifests" in output_file.location
+    assert output_file.location.endswith(".json")
+
+
+def test_provide_l0_sst_file(file_provider):
+    output_file = file_provider.provide_l0_sst_file()
+    assert "metadata/ssts/0" in output_file.location
+    assert output_file.location.endswith(".json")
+
+
+def test_provide_input_file(file_provider, sample_parquet_data):
+    input_file = file_provider.provide_input_file(str(sample_parquet_data))
+    assert input_file.location == str(sample_parquet_data)
+
+
+def test_generate_sst_uris(file_provider):
+    generated_files = list(file_provider.generate_sst_uris())
+    for file in generated_files:
+        assert "metadata/ssts/0" in file.location
+        assert file.location.endswith(".json")
+
+
+def test_get_scan_directories(file_provider):
+    partition_path = _find_partition_path(file_provider.uri, file_provider._locator)
+    assert file_provider.get_sst_scan_directories() == [
+        f"{partition_path}/metadata/ssts/0/"
+    ]