deltacat 1.1.6__py3-none-any.whl → 1.1.8__py3-none-any.whl

deltacat/tests/compute/compact_partition_test_cases.py

@@ -22,6 +22,7 @@ from deltacat.storage import (
 from deltacat.compute.compactor_v2.compaction_session import (
     compact_partition as compact_partition_v2,
 )
+from deltacat.storage import DeleteParameters
 
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
 
@@ -89,9 +90,13 @@ class IncrementalCompactionTestCaseParams(BaseCompactorTestCase):
     """
     Args:
         is_inplace: bool - argument to indicate whether to try compacting an in-place compacted table (the source table is the destination table). Also needed to control whether the destination table is created
+        add_late_deltas: List[Tuple[pa.Table, DeltaType]] - argument to indicate whether to add deltas to the source_partition after we've triggered compaction
     """
 
     is_inplace: bool
+    add_late_deltas: Optional[
+        List[Tuple[pa.Table, DeltaType, Optional[DeleteParameters]]]
+    ]
 
 
 @dataclass(frozen=True)
@@ -148,6 +153,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         is_inplace=False,
+        add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
     ),
     "2-incremental-pkstr-skstr-norcf": IncrementalCompactionTestCaseParams(
@@ -175,6 +181,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         is_inplace=False,
+        add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
     ),
     "3-incremental-pkstr-multiskstr-norcf": IncrementalCompactionTestCaseParams(
@@ -211,6 +218,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         is_inplace=False,
+        add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
     ),
     "4-incremental-duplicate-pk": IncrementalCompactionTestCaseParams(
@@ -246,6 +254,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         is_inplace=False,
+        add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
     ),
     "5-incremental-decimal-pk-simple": IncrementalCompactionTestCaseParams(
@@ -276,6 +285,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         is_inplace=False,
+        add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
     ),
     "6-incremental-integer-pk-simple": IncrementalCompactionTestCaseParams(
@@ -306,6 +316,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         is_inplace=False,
+        add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
     ),
     "7-incremental-timestamp-pk-simple": IncrementalCompactionTestCaseParams(
@@ -336,6 +347,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         is_inplace=False,
+        add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
     ),
     "8-incremental-decimal-timestamp-pk-multi": IncrementalCompactionTestCaseParams(
@@ -368,6 +380,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         is_inplace=False,
+        add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
     ),
     "9-incremental-decimal-pk-multi-dup": IncrementalCompactionTestCaseParams(
@@ -398,6 +411,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         is_inplace=False,
+        add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
     ),
     "10-incremental-decimal-pk-partitionless": IncrementalCompactionTestCaseParams(
@@ -428,6 +442,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         is_inplace=False,
+        add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
     ),
     "11-incremental-decimal-hash-bucket-single": IncrementalCompactionTestCaseParams(
@@ -458,6 +473,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         is_inplace=False,
+        add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
     ),
     "12-incremental-decimal-single-hash-bucket": IncrementalCompactionTestCaseParams(
@@ -488,6 +504,7 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         is_inplace=False,
+        add_late_deltas=None,
         skip_enabled_compact_partition_drivers=None,
     ),
     "13-incremental-pkstr-skexists-isinplacecompacted": IncrementalCompactionTestCaseParams(
@@ -518,6 +535,50 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
         read_kwargs_provider=None,
         drop_duplicates=True,
         is_inplace=True,
+        add_late_deltas=[
+            (
+                pa.Table.from_arrays(
+                    [
+                        pa.array([str(i) for i in range(20)]),
+                        pa.array([i for i in range(20)]),
+                    ],
+                    names=["pk_col_1", "sk_col_1"],
+                ),
+                DeltaType.UPSERT,
+                None,
+            )
+        ],
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+    ),
+    "14-incremental-pkstr-skexists-unhappy-hash-bucket-count-not-present": IncrementalCompactionTestCaseParams(
+        primary_keys={"pk_col_1"},
+        sort_keys=[SortKey.of(key_name="sk_col_1")],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([str(i) for i in range(10)]),
+                pa.array([i for i in range(10)]),
+            ],
+            names=["pk_col_1", "sk_col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([str(i) for i in range(10)]),
+                pa.array([i for i in range(10)]),
+            ],
+            names=["pk_col_1", "sk_col_1"],
+        ),
+        expected_terminal_exception=AssertionError,
+        expected_terminal_exception_message="hash_bucket_count is a required arg for compactor v2",
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=None,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        is_inplace=False,
+        add_late_deltas=False,
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
     ),
 }

deltacat/tests/compute/compactor_v2/test_compaction_session.py

@@ -35,6 +35,8 @@ class TestCompactionSession(unittest.TestCase):
     @classmethod
     def doClassCleanups(cls) -> None:
         os.remove(cls.DB_FILE_PATH)
+        ray.shutdown()
+        super().tearDownClass()
 
     @patch("deltacat.compute.compactor_v2.compaction_session.rcf")
     @patch("deltacat.compute.compactor_v2.compaction_session.s3_utils")

deltacat/tests/compute/test_compact_partition_incremental.py

@@ -2,8 +2,9 @@ import ray
 from moto import mock_s3
 import pytest
 import os
+import logging
 import boto3
-from typing import Any, Callable, Dict, List, Optional, Set
+from typing import Any, Callable, Dict, List, Optional, Set, Tuple
 from boto3.resources.base import ServiceResource
 import pyarrow as pa
 from pytest_benchmark.fixture import BenchmarkFixture
@@ -15,6 +16,7 @@ from deltacat.tests.compute.test_util_common import (
 from deltacat.tests.test_utils.utils import read_s3_contents
 from deltacat.tests.compute.test_util_create_table_deltas_repo import (
     create_src_w_deltas_destination_plus_destination,
+    add_late_deltas_to_partition,
 )
 from deltacat.tests.compute.compact_partition_test_cases import (
     INCREMENTAL_TEST_CASES,
@@ -27,12 +29,33 @@ from deltacat.tests.compute.test_util_constant import (
 from deltacat.compute.compactor import (
     RoundCompletionInfo,
 )
+from deltacat.storage import (
+    CommitState,
+    DeltaType,
+    Delta,
+    DeltaLocator,
+    Partition,
+    PartitionLocator,
+)
+from deltacat.types.media import ContentType
+from deltacat.compute.compactor.model.compaction_session_audit_info import (
+    CompactionSessionAuditInfo,
+)
+from deltacat.compute.compactor.model.compact_partition_params import (
+    CompactPartitionParams,
+)
+from deltacat.utils.placement import (
+    PlacementGroupManager,
+)
+from deltacat import logs
 
 DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
     "db_file_path",
     "deltacat/tests/local_deltacat_storage/db_test.sqlite",
 )
 
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
 
 """
 MODULE scoped fixtures
@@ -43,6 +66,7 @@ MODULE scoped fixtures
 def setup_ray_cluster():
     ray.init(local_mode=True, ignore_reinit_error=True)
     yield
+    ray.shutdown()
 
 
 @pytest.fixture(autouse=True, scope="module")
@@ -58,19 +82,20 @@ def mock_aws_credential():
 @pytest.fixture(autouse=True, scope="module")
 def cleanup_the_database_file_after_all_compaction_session_package_tests_complete():
     # make sure the database file is deleted after all the compactor package tests are completed
+    yield
     if os.path.exists(DATABASE_FILE_PATH_VALUE):
         os.remove(DATABASE_FILE_PATH_VALUE)
 
 
 @pytest.fixture(scope="module")
-def setup_s3_resource(mock_aws_credential):
+def s3_resource():
     with mock_s3():
         yield boto3.resource("s3")
 
 
 @pytest.fixture(autouse=True, scope="module")
-def setup_compaction_artifacts_s3_bucket(setup_s3_resource: ServiceResource):
-    setup_s3_resource.create_bucket(
+def setup_compaction_artifacts_s3_bucket(s3_resource: ServiceResource):
+    s3_resource.create_bucket(
         ACL="authenticated-read",
         Bucket=TEST_S3_RCF_BUCKET_NAME,
     )
@@ -112,6 +137,7 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         "drop_duplicates_param",
         "skip_enabled_compact_partition_drivers",
         "is_inplace",
+        "add_late_deltas",
         "compact_partition_func",
     ],
     [
@@ -133,6 +159,7 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             read_kwargs_provider,
             skip_enabled_compact_partition_drivers,
             is_inplace,
+            add_late_deltas,
             compact_partition_func,
         )
         for test_name, (
@@ -152,13 +179,14 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             read_kwargs_provider,
             skip_enabled_compact_partition_drivers,
             is_inplace,
+            add_late_deltas,
             compact_partition_func,
         ) in INCREMENTAL_TEST_CASES.items()
     ],
     ids=[test_name for test_name in INCREMENTAL_TEST_CASES],
 )
 def test_compact_partition_incremental(
-    setup_s3_resource: ServiceResource,
+    s3_resource: ServiceResource,
     offer_local_deltacat_storage_kwargs: Dict[str, Any],
     test_name: str,
     primary_keys: Set[str],
@@ -177,25 +205,11 @@ def test_compact_partition_incremental(
     read_kwargs_provider_param: Any,
     skip_enabled_compact_partition_drivers,
     is_inplace: bool,
+    add_late_deltas: Optional[List[Tuple[pa.Table, DeltaType]]],
     compact_partition_func: Callable,
     benchmark: BenchmarkFixture,
 ):
     import deltacat.tests.local_deltacat_storage as ds
-    from deltacat.types.media import ContentType
-    from deltacat.storage import (
-        DeltaLocator,
-        Partition,
-        PartitionLocator,
-    )
-    from deltacat.compute.compactor.model.compaction_session_audit_info import (
-        CompactionSessionAuditInfo,
-    )
-    from deltacat.compute.compactor.model.compact_partition_params import (
-        CompactPartitionParams,
-    )
-    from deltacat.utils.placement import (
-        PlacementGroupManager,
-    )
 
     ds_mock_kwargs: Dict[str, Any] = offer_local_deltacat_storage_kwargs
 
@@ -205,6 +219,9 @@ def test_compact_partition_incremental(
         source_table_stream,
         destination_table_stream,
         _,
+        source_table_namespace,
+        source_table_name,
+        source_table_version,
     ) = create_src_w_deltas_destination_plus_destination(
         primary_keys,
         sort_keys,
@@ -227,11 +244,13 @@ def test_compact_partition_incremental(
     )
     num_workers, worker_instance_cpu = DEFAULT_NUM_WORKERS, DEFAULT_WORKER_INSTANCE_CPUS
     total_cpus: int = num_workers * worker_instance_cpu
-    pgm: Optional[PlacementGroupManager] = None
-    if create_placement_group_param:
-        pgm = PlacementGroupManager(
+    pgm: Optional[PlacementGroupManager] = (
+        PlacementGroupManager(
             1, total_cpus, worker_instance_cpu, memory_per_bundle=4000000
         ).pgs[0]
+        if create_placement_group_param
+        else None
+    )
     compact_partition_params = CompactPartitionParams.of(
         {
             "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
@@ -265,24 +284,36 @@ def test_compact_partition_incremental(
 
         Returns: args, kwargs
         """
-        setup_s3_resource.Bucket(TEST_S3_RCF_BUCKET_NAME).objects.all().delete()
+        s3_resource.Bucket(TEST_S3_RCF_BUCKET_NAME).objects.all().delete()
         return (compact_partition_params,), {}
 
+    if add_late_deltas:
+        # NOTE: In the case of in-place compaction it is plausible that new deltas may be added to the source partition during compaction
+        # (so that the source_partitition.stream_position > last_stream_position_to_compact).
+        # This parameter helps simulate the case to check that no late deltas are dropped even when the compacted partition is created.
+        latest_delta, _ = add_late_deltas_to_partition(
+            add_late_deltas, source_partition, ds_mock_kwargs
+        )
+    if expected_terminal_exception:
+        with pytest.raises(expected_terminal_exception) as exc_info:
+            compact_partition_func(compact_partition_params)
+        assert expected_terminal_exception_message in str(exc_info.value)
+        return
     rcf_file_s3_uri = benchmark.pedantic(
         compact_partition_func, setup=_incremental_compaction_setup
     )
+
     # validate
-    round_completion_info: RoundCompletionInfo = get_rcf(
-        setup_s3_resource, rcf_file_s3_uri
-    )
+    round_completion_info: RoundCompletionInfo = get_rcf(s3_resource, rcf_file_s3_uri)
     compacted_delta_locator: DeltaLocator = (
         round_completion_info.compacted_delta_locator
     )
-    audit_bucket, audit_key = round_completion_info.compaction_audit_url.replace(
-        "s3://", ""
-    ).split("/", 1)
+    audit_bucket, audit_key = RoundCompletionInfo.get_audit_bucket_name_and_key(
+        round_completion_info.compaction_audit_url
+    )
+
     compaction_audit_obj: Dict[str, Any] = read_s3_contents(
-        setup_s3_resource, audit_bucket, audit_key
+        s3_resource, audit_bucket, audit_key
     )
     compaction_audit: CompactionSessionAuditInfo = CompactionSessionAuditInfo(
         **compaction_audit_obj
@@ -318,8 +349,34 @@ def test_compact_partition_incremental(
             == destination_partition_locator.partition_values
             and source_partition.locator.stream_id
             == destination_partition_locator.stream_id
-        ), "The source partition should match the destination partition"
+        ), f"The source partition: {source_partition.locator.canonical_string} should match the destination partition: {destination_partition_locator.canonical_string}"
         assert (
             compacted_delta_locator.stream_id == source_partition.locator.stream_id
         ), "The compacted delta should be in the same stream as the source"
+        source_partition: Partition = ds.get_partition(
+            source_table_stream.locator,
+            partition_values_param,
+            **ds_mock_kwargs,
+        )
+        compacted_partition: Optional[Partition] = ds.get_partition(
+            compacted_delta_locator.stream_locator,
+            partition_values_param,
+            **ds_mock_kwargs,
+        )
+        assert (
+            compacted_partition.state == source_partition.state == CommitState.COMMITTED
+        ), f"The compacted/source table partition should be in {CommitState.COMMITTED} state and not {CommitState.DEPRECATED}"
+        if add_late_deltas:
+            compacted_partition_deltas: List[Delta] = ds.list_partition_deltas(
+                partition_like=compacted_partition,
+                ascending_order=False,
+                **ds_mock_kwargs,
+            ).all_items()
+            assert (
+                len(compacted_partition_deltas) == len(add_late_deltas) + 1
+            ), f"Expected the number of deltas within the newly promoted partition to equal 1 (the compacted delta) + the # of late deltas: {len(add_late_deltas)}"
+            assert (
+                compacted_partition_deltas[0].stream_position
+                == latest_delta.stream_position
+            ), f"Expected the latest delta in the compacted partition: {compacted_partition_deltas[0].stream_position} to have the same stream position as the latest delta: {latest_delta.stream_position}"
     return

deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py

@@ -37,6 +37,19 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple
 from deltacat.types.media import StorageType
 from deltacat.storage import (
     DeltaType,
+    DeltaLocator,
+    Partition,
+    PartitionLocator,
+)
+from deltacat.types.media import ContentType
+from deltacat.compute.compactor.model.compact_partition_params import (
+    CompactPartitionParams,
+)
+from deltacat.utils.placement import (
+    PlacementGroupManager,
+)
+from deltacat.compute.compactor.model.compaction_session_audit_info import (
+    CompactionSessionAuditInfo,
 )
 
 DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
@@ -54,6 +67,7 @@ MODULE scoped fixtures
 def setup_ray_cluster():
     ray.init(local_mode=True, ignore_reinit_error=True)
     yield
+    ray.shutdown()
 
 
 @pytest.fixture(autouse=True, scope="module")
@@ -74,14 +88,14 @@ def cleanup_the_database_file_after_all_compaction_session_package_tests_complet
 
 
 @pytest.fixture(scope="module")
-def setup_s3_resource(mock_aws_credential):
+def s3_resource(mock_aws_credential):
     with mock_s3():
         yield boto3.resource("s3")
 
 
 @pytest.fixture(autouse=True, scope="module")
-def setup_compaction_artifacts_s3_bucket(setup_s3_resource: ServiceResource):
-    setup_s3_resource.create_bucket(
+def setup_compaction_artifacts_s3_bucket(s3_resource: ServiceResource):
+    s3_resource.create_bucket(
         ACL="authenticated-read",
         Bucket=TEST_S3_RCF_BUCKET_NAME,
     )
@@ -172,7 +186,7 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
     ids=[test_name for test_name in REBASE_THEN_INCREMENTAL_TEST_CASES],
 )
 def test_compact_partition_rebase_then_incremental(
-    setup_s3_resource: ServiceResource,
+    s3_resource: ServiceResource,
     local_deltacat_storage_kwargs: Dict[str, Any],
     test_name: str,
     primary_keys: Set[str],
@@ -196,25 +210,8 @@ def test_compact_partition_rebase_then_incremental(
     benchmark: BenchmarkFixture,
 ):
     import deltacat.tests.local_deltacat_storage as ds
-    from deltacat.types.media import ContentType
-    from deltacat.storage import (
-        DeltaLocator,
-        Partition,
-        PartitionLocator,
-    )
-    from deltacat.compute.compactor.model.compact_partition_params import (
-        CompactPartitionParams,
-    )
-    from deltacat.utils.placement import (
-        PlacementGroupManager,
-    )
-    from deltacat.compute.compactor.model.compaction_session_audit_info import (
-        CompactionSessionAuditInfo,
-    )
 
     ds_mock_kwargs = local_deltacat_storage_kwargs
-    ray.shutdown()
-    ray.init(local_mode=True, ignore_reinit_error=True)
     """
     REBASE
     """
@@ -280,7 +277,7 @@ def test_compact_partition_rebase_then_incremental(
     # execute
     rcf_file_s3_uri = benchmark(compact_partition_func, compact_partition_params)
     compacted_delta_locator: DeltaLocator = get_compacted_delta_locator_from_rcf(
-        setup_s3_resource, rcf_file_s3_uri
+        s3_resource, rcf_file_s3_uri
     )
     tables = ds.download_delta(
         compacted_delta_locator, storage_type=StorageType.LOCAL, **ds_mock_kwargs
@@ -346,16 +343,14 @@ def test_compact_partition_rebase_then_incremental(
         assert expected_terminal_exception_message in str(exc_info.value)
         return
     rcf_file_s3_uri = compact_partition_func(compact_partition_params)
-    round_completion_info = get_rcf(setup_s3_resource, rcf_file_s3_uri)
+    round_completion_info = get_rcf(s3_resource, rcf_file_s3_uri)
     compacted_delta_locator_incremental: DeltaLocator = (
         round_completion_info.compacted_delta_locator
     )
     audit_bucket, audit_key = round_completion_info.compaction_audit_url.replace(
         "s3://", ""
     ).split("/", 1)
-    compaction_audit_obj: dict = read_s3_contents(
-        setup_s3_resource, audit_bucket, audit_key
-    )
+    compaction_audit_obj: dict = read_s3_contents(s3_resource, audit_bucket, audit_key)
     compaction_audit: CompactionSessionAuditInfo = CompactionSessionAuditInfo(
         **compaction_audit_obj
     )

deltacat/tests/compute/test_util_create_table_deltas_repo.py

@@ -19,6 +19,42 @@ from deltacat.tests.compute.test_util_common import (
     create_destination_table,
     create_rebase_table,
 )
+import logging
+from deltacat import logs
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+def _add_deltas_to_partition(
+    deltas_ingredients: List[Tuple[pa.Table, DeltaType, Optional[Dict[str, str]]]],
+    partition: Optional[Partition],
+    ds_mock_kwargs: Optional[Dict[str, Any]],
+) -> List[Optional[Delta], int]:
+    import deltacat.tests.local_deltacat_storage as ds
+
+    all_deltas_length = 0
+    for (delta_data, delta_type, delete_parameters) in deltas_ingredients:
+        staged_delta: Delta = ds.stage_delta(
+            delta_data,
+            partition,
+            delta_type,
+            delete_parameters=delete_parameters,
+            **ds_mock_kwargs,
+        )
+        incremental_delta = ds.commit_delta(
+            staged_delta,
+            **ds_mock_kwargs,
+        )
+        all_deltas_length += len(delta_data) if delta_data else 0
+    return incremental_delta, all_deltas_length
+
+
+def add_late_deltas_to_partition(
+    late_deltas: List[Tuple[pa.Table, DeltaType, Optional[Dict[str, str]]]],
+    source_partition: Optional[Partition],
+    ds_mock_kwargs: Optional[Dict[str, Any]],
+) -> List[Optional[Delta], int]:
+    return _add_deltas_to_partition(late_deltas, source_partition, ds_mock_kwargs)
 
 
 def create_incremental_deltas_on_source_table(
@@ -85,7 +121,7 @@ def create_src_w_deltas_destination_plus_destination(
     partition_values: Optional[List[Any]],
     ds_mock_kwargs: Optional[Dict[str, Any]],
     simulate_is_inplace: bool = False,
-) -> Tuple[Stream, Stream, Optional[Stream]]:
+) -> Tuple[Stream, Stream, Optional[Stream], str, str, str]:
     import deltacat.tests.local_deltacat_storage as ds
 
     source_namespace, source_table_name, source_table_version = create_src_table(
@@ -137,7 +173,14 @@ def create_src_w_deltas_destination_plus_destination(
         table_version=destination_table_version,
         **ds_mock_kwargs,
     )
-    return source_table_stream_after_committed, destination_table_stream, None
+    return (
+        source_table_stream_after_committed,
+        destination_table_stream,
+        None,
+        source_namespace,
+        source_table_name,
+        source_table_version,
+    )
 
 
 def create_src_w_deltas_destination_rebase_w_deltas_strategy(