deltacat 2.0.0b10__py3-none-any.whl → 2.0.0b12__py3-none-any.whl

deltacat/tests/storage/model/test_shard.py

@@ -7,7 +7,9 @@ def test_sharding_strategy_from_string_range():
     """
     Tests that from_string('range') returns an instance of RangeShardingStrategy.
     """
-    from deltacat.storage.rivulet.shard.range_shard import RangeShardingStrategy
+    from deltacat.experimental.storage.rivulet.shard.range_shard import (
+        RangeShardingStrategy,
+    )
 
     strategy = ShardingStrategy.from_string("range")
     assert isinstance(strategy, RangeShardingStrategy)

deltacat/tests/storage/model/test_sort_scheme.py

@@ -0,0 +1,90 @@
+import pytest
+
+from deltacat.storage import (
+    SortKey,
+    SortScheme,
+    SortOrder,
+    NullOrder,
+)
+
+
+def test_sort_scheme_validates_empty_keys():
+    # When creating a sort scheme with empty keys list
+    with pytest.raises(ValueError, match="Sort scheme cannot have empty keys list"):
+        SortScheme.of(
+            keys=[],
+            name="test_sort_scheme",
+            scheme_id="test_sort_scheme_id",
+        )
+
+
+def test_sort_scheme_validates_duplicate_keys():
+    # When creating a sort scheme with duplicate keys
+    with pytest.raises(ValueError, match="Duplicate sort key found: col1"):
+        SortScheme.of(
+            keys=[
+                SortKey.of(
+                    key=["col1"],
+                    sort_order=SortOrder.ASCENDING,
+                    null_order=NullOrder.AT_END,
+                ),
+                SortKey.of(
+                    key=["col1"],  # Duplicate key
+                    sort_order=SortOrder.DESCENDING,
+                    null_order=NullOrder.AT_START,
+                ),
+            ],
+            name="test_sort_scheme",
+            scheme_id="test_sort_scheme_id",
+        )
+
+
+def test_sort_scheme_allows_valid_keys():
+    # When creating a sort scheme with valid keys
+    sort_scheme = SortScheme.of(
+        keys=[
+            SortKey.of(
+                key=["col1"],
+                sort_order=SortOrder.ASCENDING,
+                null_order=NullOrder.AT_END,
+            ),
+            SortKey.of(
+                key=["col2"],
+                sort_order=SortOrder.DESCENDING,
+                null_order=NullOrder.AT_END,
+            ),
+        ],
+        name="test_sort_scheme",
+        scheme_id="test_sort_scheme_id",
+    )
+
+    # Then it should succeed
+    assert sort_scheme is not None
+    assert len(sort_scheme.keys) == 2
+    assert sort_scheme.name == "test_sort_scheme"
+    assert sort_scheme.id == "test_sort_scheme_id"
+
+
+def test_sort_scheme_validates_null_order_consistency():
+    # When creating a sort scheme with inconsistent null orders
+    with pytest.raises(
+        ValueError, match="All arrow sort keys must use the same null order"
+    ):
+        sort_scheme = SortScheme.of(
+            keys=[
+                SortKey.of(
+                    key=["col1"],
+                    sort_order=SortOrder.ASCENDING,
+                    null_order=NullOrder.AT_END,
+                ),
+                SortKey.of(
+                    key=["col2"],
+                    sort_order=SortOrder.DESCENDING,
+                    null_order=NullOrder.AT_START,  # Different null order
+                ),
+            ],
+            name="test_sort_scheme",
+            scheme_id="test_sort_scheme_id",
+        )
+        # Access arrow property to trigger validation
+        sort_scheme.arrow

deltacat/tests/storage/model/test_transaction.py

@@ -1,15 +1,25 @@
 import pytest
+import os
+import pyarrow
+import msgpack
+import posixpath
+
 
 from deltacat.storage import (
     Transaction,
     TransactionOperation,
-    TransactionType,
     TransactionOperationType,
-)
-from deltacat.storage.model.metafile import (
+    Namespace,
+    NamespaceLocator,
     Metafile,
 )
 
+from deltacat.constants import (
+    TXN_DIR_NAME,
+    RUNNING_TXN_DIR_NAME,
+    PAUSED_TXN_DIR_NAME,
+)
+
 
 class TestAbsToRelative:
     @classmethod
@@ -64,7 +74,7 @@ class TestAbsToRelative:
             Transaction._abs_txn_meta_path_to_relative("/lorem/ipsum/", "")
 
     # Test cases for the relativize_operation_paths function
-    def test_relativize_metafile_write_paths(self):
+    def test_relativizemetafile_write_paths(self):
         catalog_root = "/catalog/root"
         absolute_paths = [
             "/catalog/root/path/to/metafile1.mpk",
@@ -91,9 +101,7 @@ class TestAbsToRelative:
         # use replace method as setter
         transaction_operation.metafile_write_paths = absolute_paths
         # Create a transaction and relativize paths
-        transaction = Transaction.of(
-            txn_type=TransactionType.APPEND, txn_operations=[transaction_operation]
-        )
+        transaction = Transaction.of([transaction_operation])
         transaction.relativize_operation_paths(transaction_operation, catalog_root)
         # Verify the paths have been correctly relativized
         assert transaction_operation.metafile_write_paths == expected_relative_paths
@@ -125,9 +133,7 @@ class TestAbsToRelative:
         # use replace as setter
         transaction_operation.locator_write_paths = absolute_paths
         # Create a transaction and relativize paths
-        transaction = Transaction.of(
-            txn_type=TransactionType.APPEND, txn_operations=[transaction_operation]
-        )
+        transaction = Transaction.of(txn_operations=[transaction_operation])
         transaction.relativize_operation_paths(transaction_operation, catalog_root)
         # Verify the paths have been correctly relativized
         assert transaction_operation.locator_write_paths == expected_relative_paths
@@ -164,9 +170,7 @@ class TestAbsToRelative:
         transaction_operation.metafile_write_paths = meta_absolute_paths
         transaction_operation.locator_write_paths = loc_absolute_paths
         # Create a transaction and relativize paths
-        transaction = Transaction.of(
-            txn_type=TransactionType.APPEND, txn_operations=[transaction_operation]
-        )
+        transaction = Transaction.of([transaction_operation])
         transaction.relativize_operation_paths(transaction_operation, catalog_root)
         # Verify the paths have been correctly relativized
         assert (
@@ -222,9 +226,7 @@ class TestAbsToRelative:
             transaction_operation.locator_write_paths = loc_absolute_paths
             transaction_operations.append(transaction_operation)
         # Create a transaction and relativize paths
-        transaction = Transaction.of(
-            txn_type=TransactionType.APPEND, txn_operations=transaction_operations
-        )
+        transaction = Transaction.of(transaction_operations)
         for operation in transaction_operations:
             transaction.relativize_operation_paths(operation, catalog_root)
         # Verify the paths have been correctly relativized
@@ -241,9 +243,7 @@ class TestAbsToRelative:
         # Empty paths
         transaction_operation.metafile_write_paths = []
         transaction_operation.locator_write_paths = []
-        transaction = Transaction.of(
-            txn_type=TransactionType.APPEND, txn_operations=[transaction_operation]
-        )
+        transaction = Transaction.of([transaction_operation])
         transaction.relativize_operation_paths(transaction_operation, catalog_root)
         assert transaction_operation.metafile_write_paths == []
         assert transaction_operation.locator_write_paths == []
@@ -257,9 +257,7 @@ class TestAbsToRelative:
             dest_metafile=Metafile({"id": "dummy_metafile_id"}),
         )
         transaction_operation.metafile_write_paths = absolute_paths
-        transaction = Transaction.of(
-            txn_type=TransactionType.APPEND, txn_operations=[transaction_operation]
-        )
+        transaction = Transaction.of([transaction_operation])
         transaction.relativize_operation_paths(transaction_operation, catalog_root)
         assert transaction_operation.metafile_write_paths == expected_paths
 
@@ -279,30 +277,377 @@ class TestAbsToRelative:
             TransactionOperationType.READ_SIBLINGS,
         ]
 
-        # Different transaction types to test
-        txn_types = [
-            TransactionType.APPEND,
-            TransactionType.ALTER,
-            TransactionType.DELETE,
-            TransactionType.OVERWRITE,
-            TransactionType.READ,
-            TransactionType.RESTATE,
-        ]
-
-        for txn_type in txn_types:
-            transaction_ops = []
-            for op_type in operation_types:
-                transaction_operation = TransactionOperation.of(
-                    operation_type=op_type,
-                    dest_metafile=Metafile({"id": "dummy_metafile_id"}),
-                )
-                transaction_operation.metafile_write_paths = absolute_paths
-                transaction_ops.append(transaction_operation)
-            transaction = Transaction.of(
-                txn_type=txn_type, txn_operations=[transaction_operation]
+        transaction_ops = []
+        for op_type in operation_types:
+            transaction_operation = TransactionOperation.of(
+                operation_type=op_type,
+                dest_metafile=Metafile({"id": "dummy_metafile_id"}),
             )
-            transaction.relativize_operation_paths(transaction_operation, catalog_root)
-            # Assert paths are relativized correctly
-            assert (
-                transaction_operation.metafile_write_paths == expected_paths
-            ), f"Failed for transaction type {txn_type} and operation type {op_type}"
+            transaction_operation.metafile_write_paths = absolute_paths
+            transaction_ops.append(transaction_operation)
+        transaction = Transaction.of([transaction_operation])
+        transaction.relativize_operation_paths(transaction_operation, catalog_root)
+        # Assert paths are relativized correctly
+        assert (
+            transaction_operation.metafile_write_paths == expected_paths
+        ), f"Failed for operation type {op_type}"
+
+
+class TestTransactionPersistence:
+
+    # Verifies that transactions initialized with empty or None operations are marked interactive,
+    # while valid operations are not
+    def test_create_iterative_transaction(self):
+        txn_1 = Transaction.of(txn_operations=[])
+        txn_2 = Transaction.of(txn_operations=None)
+        op = TransactionOperation.of(
+            operation_type=TransactionOperationType.CREATE,
+            dest_metafile=Metafile({"id": "dummy_metafile_id"}),
+        )
+        txn_3 = Transaction.of(txn_operations=[op, op])
+        assert (
+            txn_1.interactive
+        )  # check if constructor detect empty list --> interactive transaction
+        assert (
+            txn_2.interactive
+        )  # check if we can initialize with no list --> interactive transaction
+        assert (
+            not txn_3.interactive
+        )  # check that valid operations_list --> not interactive transaction
+
+    # Builds and commits a transaction step-by-step, then validates the output files and transaction success log
+    def test_commit_iterative_transaction(self, temp_dir):
+        # Create two simple namespaces
+        namespace_locator1 = NamespaceLocator.of(namespace="test_ns_1")
+        namespace_locator2 = NamespaceLocator.of(namespace="test_ns_2")
+        ns1 = Namespace.of(locator=namespace_locator1)
+        ns2 = Namespace.of(locator=namespace_locator2)
+        # Start with an empty transaction (interactive)
+        transaction = Transaction.of()
+        txn = transaction.start(temp_dir)  # operate on deep-copy
+        # Build operations manually and step them in
+        op1 = TransactionOperation.of(
+            operation_type=TransactionOperationType.CREATE,
+            dest_metafile=ns1,
+        )
+        op2 = TransactionOperation.of(
+            operation_type=TransactionOperationType.CREATE,
+            dest_metafile=ns2,
+        )
+        # steps
+        txn.step(op1)
+        txn.step(op2)
+
+        # seal() for interactive transactions
+        write_paths, success_log_path = txn.seal()
+
+        # Check output files exist and are valid
+        deserialized_ns1 = Namespace.read(write_paths[0])
+        deserialized_ns2 = Namespace.read(write_paths[1])
+
+        assert ns1.equivalent_to(deserialized_ns1)
+        assert ns2.equivalent_to(deserialized_ns2)
+        assert success_log_path.endswith(str(txn.end_time))
+
+    # Ensures that stepping and committing a transaction writes non-empty output files and a valid success log
+    def test_commit_iterative_file_creation(self, temp_dir):
+        ns = Namespace.of(locator=NamespaceLocator.of(namespace="check_writes"))
+        txn = Transaction.of().start(temp_dir)
+        op = TransactionOperation.of(TransactionOperationType.CREATE, dest_metafile=ns)
+        txn.step(op)
+        write_paths, success_log_path = txn.seal()
+
+        # check the files were created
+        for path in write_paths:
+            abs_path = os.path.join(temp_dir, path)
+            assert os.path.exists(abs_path)
+            assert os.path.getsize(abs_path) > 0
+
+        # check the success log exists
+        assert os.path.exists(success_log_path)
+        assert os.path.getsize(success_log_path) > 0
+
+    # Confirms that a transaction can be paused, resumed, and successfully committed without data los
+    def test_transaction_pause_and_resume_roundtrip(self, temp_dir):
+        # Create a test namespace
+        ns = Namespace.of(locator=NamespaceLocator.of(namespace="paused_resume_ns"))
+
+        # Start interactive transaction
+        txn = Transaction.of().start(temp_dir)
+        op = TransactionOperation.of(TransactionOperationType.CREATE, dest_metafile=ns)
+
+        txn.step(op)
+
+        # Pause transaction (writes to paused/)
+        txn.pause()
+
+        # Resume transaction (reads from paused/)
+        txn.resume()
+
+        # Commit resumed transaction
+        write_paths, success_log_path = txn.seal()
+
+        # Validate outputs
+        deserialized = Namespace.read(write_paths[0])
+        assert ns.equivalent_to(deserialized)
+        assert os.path.exists(success_log_path)
+        assert success_log_path.endswith(str(txn.end_time))
+
+    # Validates that transaction state, including ID and write paths, is correctly preserved across pause/resume cycles
+    def test_resume_preserves_state_after_pause(self, temp_dir):
+        ns = Namespace.of(locator=NamespaceLocator.of(namespace="resume_state_check"))
+
+        txn = Transaction.of().start(temp_dir)
+        op = TransactionOperation.of(TransactionOperationType.CREATE, dest_metafile=ns)
+
+        txn.step(op)
+        txn_id_before = txn.id
+
+        txn.pause()
+        txn.resume()
+
+        # Ensure the ID and provider are still valid
+        assert txn.id == txn_id_before
+        assert txn._time_provider is not None
+        assert hasattr(txn, "metafile_write_paths")
+        assert len(txn.metafile_write_paths) == 1
+
+        # Check commit still works
+        _, success_log_path = txn.seal()
+        assert os.path.exists(success_log_path)
+
+    # Explicitly checks that fields are preserved
+    def test_resume_preserves_state_after_pause_deep(self, temp_dir):
+        ns = Namespace.of(locator=NamespaceLocator.of(namespace="resume_state_check"))
+
+        txn = Transaction.of().start(temp_dir)
+        op = TransactionOperation.of(TransactionOperationType.CREATE, dest_metafile=ns)
+
+        txn.step(op)
+
+        # Save values before pause
+        txn_id_before = txn.id
+        start_time_before = txn.start_time
+        root_before = txn.catalog_root_normalized
+        meta_paths_before = list(txn.metafile_write_paths)
+        locator_paths_before = list(txn.locator_write_paths)
+
+        txn.pause()
+        txn.resume()
+
+        # Field-by-field checks
+        assert txn.id == txn_id_before, "Transaction ID should be preserved"
+        assert txn._time_provider is not None, "Time provider should be reinitialized"
+        assert txn.start_time == start_time_before, "Start time should be preserved"
+        assert txn.catalog_root_normalized == root_before, "Catalog root should match"
+        assert (
+            txn.metafile_write_paths == meta_paths_before
+        ), "Metafile paths must match"
+        assert (
+            txn.locator_write_paths == locator_paths_before
+        ), "Locator paths must match"
+        assert (
+            isinstance(txn.operations, list) and len(txn.operations) == 1
+        ), "Operations must be restored"
+        assert txn.pause_time is not None, "Pause time should be restored"
+
+        # Final commit still works
+        write_paths, success_log_path = txn.seal()
+        assert os.path.exists(success_log_path)
+
+    # Checks that pausing a transaction moves its log from running/ to paused/ and preserves valid transaction state
+    def test_pause_moves_running_to_paused(self, temp_dir):
+        # Set up a transaction and a single operation
+        locator = NamespaceLocator.of(namespace="pause_test")
+        ns = Namespace.of(locator=locator)
+        txn = Transaction.of().start(temp_dir)
+
+        op = TransactionOperation.of(TransactionOperationType.CREATE, dest_metafile=ns)
+        txn.step(op)
+
+        fs = pyarrow.fs.LocalFileSystem()
+        txn_id = txn.id
+        txn_log_dir = posixpath.join(temp_dir, TXN_DIR_NAME)
+
+        running_path = posixpath.join(txn_log_dir, RUNNING_TXN_DIR_NAME, txn_id)
+        paused_path = posixpath.join(txn_log_dir, PAUSED_TXN_DIR_NAME, txn_id)
+
+        # Sanity check: file should be in running/
+        assert fs.get_file_info(running_path).type == pyarrow.fs.FileType.File
+
+        # Pause transaction
+        txn.pause()
+        # Ensure the running file is deleted
+        assert fs.get_file_info(running_path).type == pyarrow.fs.FileType.NotFound
+
+        # Ensure the paused file exists and contains valid msgpack
+        paused_info = fs.get_file_info(paused_path)
+        assert paused_info.type == pyarrow.fs.FileType.File
+        with fs.open_input_stream(paused_path) as f:
+            data = f.readall()
+            txn_loaded = msgpack.loads(data)
+            assert "operations" in txn_loaded
+
+    # Simulates a full multi-step transaction with multiple pause/resume cycles and verifies correctness of all outputs
+    def test_transaction_pause_and_resume_roundtrip_complex(self, temp_dir):
+        # Step 0: Create an empty interactive transaction
+        txn = Transaction.of().start(temp_dir)
+
+        # Step 1: Add first namespace, pause
+        ns1 = Namespace.of(locator=NamespaceLocator.of(namespace="roundtrip_ns_1"))
+        op1 = TransactionOperation.of(
+            TransactionOperationType.CREATE, dest_metafile=ns1
+        )
+        txn.step(op1)
+        txn.pause()
+
+        # Step 2: Resume, add second namespace, pause
+        txn.resume()
+        ns2 = Namespace.of(locator=NamespaceLocator.of(namespace="roundtrip_ns_2"))
+        op2 = TransactionOperation.of(
+            TransactionOperationType.CREATE, dest_metafile=ns2
+        )
+        txn.step(op2)
+        txn.pause()
+
+        # Step 3: Resume again, add third namespace, commit
+        txn.resume()
+        ns3 = Namespace.of(locator=NamespaceLocator.of(namespace="roundtrip_ns_3"))
+        op3 = TransactionOperation.of(
+            TransactionOperationType.CREATE, dest_metafile=ns3
+        )
+        txn.step(op3)
+
+        # Final commit
+        write_paths, success_log_path = txn.seal()
+
+        # Read and verify written namespaces
+        for i, ns in enumerate([ns1, ns2, ns3]):
+            written_path = write_paths[i]
+            deserialized_ns = Namespace.read(written_path)
+            assert ns.equivalent_to(
+                deserialized_ns
+            ), f"Mismatch in ns{i+1}: {ns} != {deserialized_ns}"
+            assert os.path.exists(written_path), f"Missing file: {written_path}"
+            assert os.path.getsize(written_path) > 0
+
+        # Check success log exists and is correct
+        assert os.path.exists(success_log_path)
+        assert success_log_path.endswith(str(txn.end_time))
+
+    # Repeats a complex pause/resume flow with additional assertions on namespace equality and time consistency
+    def test_transaction_pause_and_resume_roundtrip_complex_2(self, temp_dir):
+        # Step 0: Create an empty interactive transaction
+        txn = Transaction.of().start(temp_dir)
+
+        # Step 1: Add first namespace, pause
+        ns1 = Namespace.of(locator=NamespaceLocator.of(namespace="roundtrip_ns_1"))
+        op1 = TransactionOperation.of(
+            TransactionOperationType.CREATE, dest_metafile=ns1
+        )
+        txn.step(op1)
+        txn.pause()
+
+        # Step 2: Resume, add second namespace, pause
+        txn.resume()
+        ns2 = Namespace.of(locator=NamespaceLocator.of(namespace="roundtrip_ns_2"))
+        op2 = TransactionOperation.of(
+            TransactionOperationType.CREATE, dest_metafile=ns2
+        )
+        txn.step(op2)
+
+        txn.pause()
+
+        # Step 3: Resume again, add third namespace, commit
+        txn.resume()
+        ns3 = Namespace.of(locator=NamespaceLocator.of(namespace="roundtrip_ns_3"))
+        op3 = TransactionOperation.of(
+            TransactionOperationType.CREATE, dest_metafile=ns3
+        )
+        txn.step(op3)
+
+        # Final commit
+        write_paths, success_log_path = txn.seal()
+
+        assert txn.start_time < txn.end_time
+
+        # Read and verify written namespaces
+        for i, ns in enumerate([ns1, ns2, ns3]):
+            written_path = write_paths[i]
+
+            # Confirm file was created and is non-empty
+            assert os.path.exists(written_path), f"Missing file: {written_path}"
+            assert os.path.getsize(written_path) > 0, f"Empty file: {written_path}"
+
+            # Deserialize and verify content
+            deserialized_ns = Namespace.read(written_path)
+            assert ns.equivalent_to(deserialized_ns), f"Namespace mismatch at index {i}"
+            assert ns.locator.namespace == deserialized_ns.locator.namespace
+            assert ns.locator_alias == deserialized_ns.locator_alias
+            assert ns.properties == deserialized_ns.properties
+
+        # Verify success log
+        assert os.path.exists(success_log_path)
+        assert success_log_path.endswith(str(txn.end_time))
+
+
+class TestTransactionCommitMessage:
+    """Test commit message preservation and retrieval for transactions."""
+
+    def test_transaction_with_commit_message(self):
+        """Test that commit messages are stored and retrievable from transactions."""
+        commit_msg = "Test commit message for transaction functionality"
+
+        # Create transaction with commit message
+        txn = Transaction.of(commit_message=commit_msg)
+
+        # Verify commit message is stored correctly
+        assert txn.commit_message == commit_msg
+        assert txn.get("commit_message") == commit_msg
+
+    def test_transaction_without_commit_message(self):
+        """Test that transactions work normally without commit messages."""
+        # Create transaction without commit message
+        txn = Transaction.of()
+
+        # Verify no commit message is stored
+        assert txn.commit_message is None
+        assert txn.get("commit_message") is None
+
+    def test_transaction_commit_message_setter(self):
+        """Test that commit messages can be set after transaction creation."""
+        # Create transaction without commit message
+        txn = Transaction.of()
+        assert txn.commit_message is None
+
+        # Set commit message using property setter
+        commit_msg = "Added commit message after creation"
+        txn.commit_message = commit_msg
+
+        # Verify commit message is stored correctly
+        assert txn.commit_message == commit_msg
+        assert txn.get("commit_message") == commit_msg
+
+    def test_transaction_serialization_with_commit_message(self, temp_dir):
+        """Test that commit messages persist through transaction serialization."""
+        commit_msg = "Serialization test commit message"
+
+        # Create namespace for testing
+        ns = Namespace.of(locator=NamespaceLocator.of(namespace="serialization_test"))
+
+        # Create transaction with commit message
+        txn = Transaction.of(commit_message=commit_msg).start(temp_dir)
+        op = TransactionOperation.of(TransactionOperationType.CREATE, dest_metafile=ns)
+        txn.step(op)
+
+        # Commit transaction (this should serialize the transaction with commit message)
+        _, success_log_path = txn.seal()
+
+        # Read the transaction log and verify commit message persisted
+        txn_read = Transaction.read(success_log_path)
+        assert txn_read.commit_message == commit_msg
+
+        # Verify other transaction properties are intact
+        assert txn_read.start_time == txn.start_time
+        assert txn_read.end_time == txn.end_time
+        assert len(txn_read.operations) == 1