deltacat 1.1.14__py3-none-any.whl → 1.1.16__py3-none-any.whl

deltacat/tests/compute/compactor_v2/test_compaction_session.py

@@ -87,6 +87,7 @@ class TestCompactionSession:
     INCREMENTAL_FILE_PATH = (
         "deltacat/tests/compute/compactor_v2/data/incremental_source_date_pk.csv"
     )
+    ERROR_RATE = 0.05
 
     def test_compact_partition_when_no_input_deltas_to_compact(
         self, local_deltacat_storage_kwargs
@@ -178,7 +179,7 @@ class TestCompactionSession:
                     },
                     "primary_keys": [],
                     "rebase_source_partition_locator": source_delta.partition_locator,
-                    "rebase_source_partition_high_watermark": None,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
                     "records_per_compacted_file": 4000,
                     "s3_client_kwargs": {},
                     "source_partition_locator": source_delta.partition_locator,
@@ -253,3 +254,148 @@ class TestCompactionSession:
         # as it should be running incremental
         assert compaction_audit.uniform_deltas_created == 1
         assert compaction_audit.input_records == 6
+
+    def test_compact_partition_when_incremental_then_rcf_stats_accurate(
+        self, s3_resource, local_deltacat_storage_kwargs
+    ):
+        """
+        A test case which asserts the RCF stats are correctly generated for
+        a rebase and incremental use-case.
+        """
+
+        # setup
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
+        )
+
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, [self.BACKFILL_FILE_PATH], **local_deltacat_storage_kwargs
+        )
+
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
+
+        # action
+        rcf_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 2,
+                    "last_stream_position_to_compact": source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                }
+            )
+        )
+
+        backfill_rcf = get_rcf(s3_resource, rcf_url)
+        _, compaction_audit_key = backfill_rcf.compaction_audit_url.strip(
+            "s3://"
+        ).split("/", 1)
+        compaction_audit = CompactionSessionAuditInfo(
+            **read_s3_contents(
+                s3_resource, TEST_S3_RCF_BUCKET_NAME, compaction_audit_key
+            )
+        )
+
+        assert abs(backfill_rcf.input_inflation - 0.05235042735042735) <= 1e-5
+        assert abs(backfill_rcf.input_average_record_size_bytes - 12.25) <= 1e-5
+
+        assert compaction_audit.input_records == 4
+        assert compaction_audit.records_deduped == 0
+        assert compaction_audit.records_deleted == 0
+        assert compaction_audit.untouched_file_count == 0
+        assert compaction_audit.untouched_record_count == 0
+        assert compaction_audit.untouched_size_bytes == 0
+        assert compaction_audit.untouched_file_ratio == 0
+        assert compaction_audit.uniform_deltas_created == 1
+        assert compaction_audit.hash_bucket_count == 2
+        assert compaction_audit.input_file_count == 1
+        assert compaction_audit.output_file_count == 2
+        assert abs(compaction_audit.output_size_bytes - 1832) / 1832 <= self.ERROR_RATE
+        assert abs(compaction_audit.input_size_bytes - 936) / 936 <= self.ERROR_RATE
+
+        # Now run an incremental compaction and verify if the previous RCF was read properly.
+        new_source_delta = commit_delta_to_partition(
+            source_delta.partition_locator,
+            [self.INCREMENTAL_FILE_PATH],
+            **local_deltacat_storage_kwargs,
+        )
+
+        new_destination_partition = ds.get_partition(
+            dest_partition.stream_locator, [], **local_deltacat_storage_kwargs
+        )
+
+        new_rcf_url = compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": new_destination_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 2,
+                    "last_stream_position_to_compact": new_source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": None,
+                    "rebase_source_partition_high_watermark": None,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": new_source_delta.partition_locator,
+                }
+            )
+        )
+
+        new_rcf = get_rcf(s3_resource, new_rcf_url)
+        _, compaction_audit_key = new_rcf.compaction_audit_url.strip("s3://").split(
+            "/", 1
+        )
+        compaction_audit = CompactionSessionAuditInfo(
+            **read_s3_contents(
+                s3_resource, TEST_S3_RCF_BUCKET_NAME, compaction_audit_key
+            )
+        )
+
+        # as it should be running incremental
+        assert abs(new_rcf.input_inflation - 0.027292576419213975) <= 1e-5
+        assert abs(new_rcf.input_average_record_size_bytes - 12.5) <= 1e-5
+
+        assert compaction_audit.input_records == 6
+        assert compaction_audit.records_deduped == 1
+        assert compaction_audit.records_deleted == 0
+        assert compaction_audit.untouched_file_count == 1
+        assert compaction_audit.untouched_record_count == 2
+        assert (
+            abs(compaction_audit.untouched_size_bytes - 916) / 916 <= self.ERROR_RATE
+        )  # 5% error
+        assert abs(compaction_audit.untouched_file_ratio - 50) <= 1e-5
+        assert compaction_audit.uniform_deltas_created == 1
+        assert compaction_audit.hash_bucket_count == 2
+        assert compaction_audit.input_file_count == 3
+        assert compaction_audit.output_file_count == 2
+        assert abs(compaction_audit.output_size_bytes - 1843) / 1843 <= self.ERROR_RATE
+        assert abs(compaction_audit.input_size_bytes - 2748) / 2748 <= self.ERROR_RATE

deltacat/tests/compute/test_compact_partition_multiple_rounds.py

@@ -0,0 +1,330 @@
+import ray
+import os
+from moto import mock_s3
+import pytest
+import boto3
+from boto3.resources.base import ServiceResource
+import pyarrow as pa
+from deltacat.io.ray_plasma_object_store import RayPlasmaObjectStore
+from pytest_benchmark.fixture import BenchmarkFixture
+
+from deltacat.tests.compute.test_util_constant import (
+    TEST_S3_RCF_BUCKET_NAME,
+    DEFAULT_NUM_WORKERS,
+    DEFAULT_WORKER_INSTANCE_CPUS,
+)
+from deltacat.tests.compute.test_util_common import (
+    get_rcf,
+)
+from deltacat.tests.test_utils.utils import read_s3_contents
+from deltacat.compute.compactor.model.compactor_version import CompactorVersion
+from deltacat.tests.compute.test_util_common import (
+    get_compacted_delta_locator_from_rcf,
+)
+from deltacat.compute.compactor.model.compaction_session_audit_info import (
+    CompactionSessionAuditInfo,
+)
+from deltacat.tests.compute.test_util_create_table_deltas_repo import (
+    multiple_rounds_create_src_w_deltas_destination_rebase_w_deltas_strategy,
+)
+from deltacat.tests.compute.compact_partition_multiple_rounds_test_cases import (
+    MULTIPLE_ROUNDS_TEST_CASES,
+)
+from typing import Any, Callable, Dict, List, Optional, Set
+from deltacat.types.media import StorageType
+from deltacat.storage import (
+    DeltaLocator,
+    Partition,
+)
+from deltacat.types.media import ContentType
+from deltacat.compute.compactor.model.compact_partition_params import (
+    CompactPartitionParams,
+)
+from deltacat.compute.compactor import (
+    RoundCompletionInfo,
+)
+from deltacat.utils.placement import (
+    PlacementGroupManager,
+)
+
+DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
+    "db_file_path",
+    "deltacat/tests/local_deltacat_storage/db_test.sqlite",
+)
+
+
+"""
+MODULE scoped fixtures
+"""
+
+
+@pytest.fixture(autouse=True, scope="module")
+def setup_ray_cluster():
+    ray.init(local_mode=True, ignore_reinit_error=True)
+    yield
+    ray.shutdown()
+
+
+@pytest.fixture(autouse=True, scope="module")
+def mock_aws_credential():
+    os.environ["AWS_ACCESS_KEY_ID"] = "testing"
+    os.environ["AWS_SECRET_ACCESS_ID"] = "testing"
+    os.environ["AWS_SECURITY_TOKEN"] = "testing"
+    os.environ["AWS_SESSION_TOKEN"] = "testing"
+    os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
+    yield
+
+
+@pytest.fixture(autouse=True, scope="module")
+def cleanup_the_database_file_after_all_compaction_session_package_tests_complete():
+    # make sure the database file is deleted after all the compactor package tests are completed
+    if os.path.exists(DATABASE_FILE_PATH_VALUE):
+        os.remove(DATABASE_FILE_PATH_VALUE)
+
+
+@pytest.fixture(scope="module")
+def s3_resource(mock_aws_credential):
+    with mock_s3():
+        yield boto3.resource("s3")
+
+
+@pytest.fixture(autouse=True, scope="module")
+def setup_compaction_artifacts_s3_bucket(s3_resource: ServiceResource):
+    s3_resource.create_bucket(
+        ACL="authenticated-read",
+        Bucket=TEST_S3_RCF_BUCKET_NAME,
+    )
+    yield
+
+
+"""
+FUNCTION scoped fixtures
+"""
+
+
+@pytest.fixture(scope="function")
+def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
+    # see deltacat/tests/local_deltacat_storage/README.md for documentation
+    kwargs_for_local_deltacat_storage: Dict[str, Any] = {
+        DATABASE_FILE_PATH_KEY: DATABASE_FILE_PATH_VALUE,
+    }
+    yield kwargs_for_local_deltacat_storage
+    if os.path.exists(DATABASE_FILE_PATH_VALUE):
+        os.remove(DATABASE_FILE_PATH_VALUE)
+
+
+@pytest.mark.parametrize(
+    [
+        "test_name",
+        "primary_keys",
+        "sort_keys",
+        "partition_keys_param",
+        "partition_values_param",
+        "input_deltas_param",
+        "expected_terminal_compact_partition_result",
+        "expected_terminal_exception",
+        "expected_terminal_exception_message",
+        "create_placement_group_param",
+        "records_per_compacted_file_param",
+        "hash_bucket_count_param",
+        "read_kwargs_provider_param",
+        "drop_duplicates_param",
+        "skip_enabled_compact_partition_drivers",
+        "assert_compaction_audit",
+        "rebase_expected_compact_partition_result",
+        "num_rounds_param",
+        "compact_partition_func",
+        "compactor_version",
+    ],
+    [
+        (
+            test_name,
+            primary_keys,
+            sort_keys,
+            partition_keys_param,
+            partition_values_param,
+            input_deltas,
+            expected_terminal_compact_partition_result,
+            expected_terminal_exception,
+            expected_terminal_exception_message,
+            create_placement_group_param,
+            records_per_compacted_file_param,
+            hash_bucket_count_param,
+            drop_duplicates_param,
+            read_kwargs_provider,
+            skip_enabled_compact_partition_drivers,
+            assert_compaction_audit,
+            rebase_expected_compact_partition_result,
+            num_rounds_param,
+            compact_partition_func,
+            compactor_version,
+        )
+        for test_name, (
+            primary_keys,
+            sort_keys,
+            partition_keys_param,
+            partition_values_param,
+            input_deltas,
+            expected_terminal_compact_partition_result,
+            expected_terminal_exception,
+            expected_terminal_exception_message,
+            create_placement_group_param,
+            records_per_compacted_file_param,
+            hash_bucket_count_param,
+            drop_duplicates_param,
+            read_kwargs_provider,
+            skip_enabled_compact_partition_drivers,
+            assert_compaction_audit,
+            rebase_expected_compact_partition_result,
+            num_rounds_param,
+            compact_partition_func,
+            compactor_version,
+        ) in MULTIPLE_ROUNDS_TEST_CASES.items()
+    ],
+    ids=[test_name for test_name in MULTIPLE_ROUNDS_TEST_CASES],
+)
+def test_compact_partition_rebase_multiple_rounds_same_source_and_destination(
+    mocker,
+    s3_resource: ServiceResource,
+    local_deltacat_storage_kwargs: Dict[str, Any],
+    test_name: str,
+    primary_keys: Set[str],
+    sort_keys: List[Optional[Any]],
+    partition_keys_param: Optional[List[Any]],
+    partition_values_param: List[Optional[str]],
+    input_deltas_param: List[pa.Array],
+    expected_terminal_compact_partition_result: pa.Table,
+    expected_terminal_exception: BaseException,
+    expected_terminal_exception_message: Optional[str],
+    create_placement_group_param: bool,
+    records_per_compacted_file_param: int,
+    hash_bucket_count_param: int,
+    drop_duplicates_param: bool,
+    read_kwargs_provider_param: Any,
+    rebase_expected_compact_partition_result: pa.Table,
+    skip_enabled_compact_partition_drivers: List[CompactorVersion],
+    assert_compaction_audit: Optional[Callable],
+    compactor_version: Optional[CompactorVersion],
+    compact_partition_func: Callable,
+    num_rounds_param: int,
+    benchmark: BenchmarkFixture,
+):
+    import deltacat.tests.local_deltacat_storage as ds
+
+    ds_mock_kwargs = local_deltacat_storage_kwargs
+    """
+    This test tests different multi-round compaction rebase configurations,
+    as specified in compact_partition_multiple_rounds_test_cases.py
+    These tests do not test multi-round compaction backfill, which is
+    currently unsupported.
+    """
+    (
+        source_table_stream,
+        _,
+        rebased_table_stream,
+        _,
+    ) = multiple_rounds_create_src_w_deltas_destination_rebase_w_deltas_strategy(
+        primary_keys,
+        sort_keys,
+        partition_keys_param,
+        input_deltas_param,
+        partition_values_param,
+        ds_mock_kwargs,
+    )
+    source_partition: Partition = ds.get_partition(
+        source_table_stream.locator,
+        partition_values_param,
+        **ds_mock_kwargs,
+    )
+    rebased_partition: Partition = ds.get_partition(
+        rebased_table_stream.locator,
+        partition_values_param,
+        **ds_mock_kwargs,
+    )
+    total_cpus = DEFAULT_NUM_WORKERS * DEFAULT_WORKER_INSTANCE_CPUS
+    pgm = None
+    if create_placement_group_param:
+        pgm = PlacementGroupManager(
+            1, total_cpus, DEFAULT_WORKER_INSTANCE_CPUS, memory_per_bundle=4000000
+        ).pgs[0]
+    compact_partition_params = CompactPartitionParams.of(
+        {
+            "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+            "compacted_file_content_type": ContentType.PARQUET,
+            "dd_max_parallelism_ratio": 1.0,
+            "deltacat_storage": ds,
+            "deltacat_storage_kwargs": ds_mock_kwargs,
+            "destination_partition_locator": rebased_partition.locator,
+            "hash_bucket_count": hash_bucket_count_param,
+            "last_stream_position_to_compact": source_partition.stream_position,
+            "list_deltas_kwargs": {**ds_mock_kwargs, **{"equivalent_table_types": []}},
+            "object_store": RayPlasmaObjectStore(),
+            "pg_config": pgm,
+            "primary_keys": primary_keys,
+            "read_kwargs_provider": read_kwargs_provider_param,
+            "rebase_source_partition_locator": source_partition.locator,
+            "rebase_source_partition_high_watermark": rebased_partition.stream_position,
+            "records_per_compacted_file": records_per_compacted_file_param,
+            "s3_client_kwargs": {},
+            "source_partition_locator": rebased_partition.locator,
+            "sort_keys": sort_keys if sort_keys else None,
+            "num_rounds": num_rounds_param,
+            "drop_duplicates": drop_duplicates_param,
+            "min_delta_bytes": 560,
+        }
+    )
+    if expected_terminal_exception:
+        with pytest.raises(expected_terminal_exception) as exc_info:
+            benchmark(compact_partition_func, compact_partition_params)
+        assert expected_terminal_exception_message in str(exc_info.value)
+        return
+    from deltacat.compute.compactor_v2.model.evaluate_compaction_result import (
+        ExecutionCompactionResult,
+    )
+
+    execute_compaction_result_spy = mocker.spy(ExecutionCompactionResult, "__init__")
+
+    # execute
+    rcf_file_s3_uri = benchmark(compact_partition_func, compact_partition_params)
+
+    round_completion_info: RoundCompletionInfo = get_rcf(s3_resource, rcf_file_s3_uri)
+    audit_bucket, audit_key = RoundCompletionInfo.get_audit_bucket_name_and_key(
+        round_completion_info.compaction_audit_url
+    )
+
+    compaction_audit_obj: Dict[str, Any] = read_s3_contents(
+        s3_resource, audit_bucket, audit_key
+    )
+    compaction_audit: CompactionSessionAuditInfo = CompactionSessionAuditInfo(
+        **compaction_audit_obj
+    )
+
+    # Assert not in-place compacted
+    assert (
+        execute_compaction_result_spy.call_args.args[-1] is False
+    ), "Table version erroneously marked as in-place compacted!"
+    compacted_delta_locator: DeltaLocator = get_compacted_delta_locator_from_rcf(
+        s3_resource, rcf_file_s3_uri
+    )
+    tables = ds.download_delta(
+        compacted_delta_locator, storage_type=StorageType.LOCAL, **ds_mock_kwargs
+    )
+    actual_rebase_compacted_table = pa.concat_tables(tables)
+    # if no primary key is specified then sort by sort_key for consistent assertion
+    sorting_cols: List[Any] = (
+        [(val, "ascending") for val in primary_keys] if primary_keys else sort_keys
+    )
+    rebase_expected_compact_partition_result = (
+        rebase_expected_compact_partition_result.combine_chunks().sort_by(sorting_cols)
+    )
+    actual_rebase_compacted_table = (
+        actual_rebase_compacted_table.combine_chunks().sort_by(sorting_cols)
+    )
+    assert actual_rebase_compacted_table.equals(
+        rebase_expected_compact_partition_result
+    ), f"{actual_rebase_compacted_table} does not match {rebase_expected_compact_partition_result}"
+
+    if assert_compaction_audit:
+        if not assert_compaction_audit(compactor_version, compaction_audit):
+            assert False, "Compaction audit assertion failed"
+    return

deltacat/tests/compute/test_compact_partition_rebase.py

@@ -279,7 +279,7 @@ def test_compact_partition_rebase_same_source_and_destination(
     execute_compaction_result_spy = mocker.spy(ExecutionCompactionResult, "__init__")
 
     # execute
-    rcf_file_s3_uri = compact_partition_func(compact_partition_params)
+    rcf_file_s3_uri = benchmark(compact_partition_func, compact_partition_params)
 
     round_completion_info: RoundCompletionInfo = get_rcf(s3_resource, rcf_file_s3_uri)
     audit_bucket, audit_key = RoundCompletionInfo.get_audit_bucket_name_and_key(

deltacat/tests/compute/test_util_create_table_deltas_repo.py

@@ -269,3 +269,121 @@ def create_src_w_deltas_destination_rebase_w_deltas_strategy(
         destination_table_stream,
         rebased_stream_after_committed,
     )
+
+
+def multiple_rounds_create_src_w_deltas_destination_rebase_w_deltas_strategy(
+    primary_keys: Set[str],
+    sort_keys: Optional[List[Any]],
+    partition_keys: Optional[List[PartitionKey]],
+    input_deltas: List[pa.Table],
+    partition_values: Optional[List[Any]],
+    ds_mock_kwargs: Optional[Dict[str, Any]],
+) -> Tuple[Stream, Stream, Optional[Stream], bool]:
+    import deltacat.tests.local_deltacat_storage as ds
+    from deltacat.storage import Partition, Stream
+
+    source_namespace, source_table_name, source_table_version = create_src_table(
+        primary_keys, sort_keys, partition_keys, ds_mock_kwargs
+    )
+
+    source_table_stream: Stream = ds.get_stream(
+        namespace=source_namespace,
+        table_name=source_table_name,
+        table_version=source_table_version,
+        **ds_mock_kwargs,
+    )
+    staged_partition: Partition = ds.stage_partition(
+        source_table_stream, partition_values, **ds_mock_kwargs
+    )
+    is_delete = False
+    input_delta_length = 0
+    for (
+        input_delta,
+        input_delta_type,
+        input_delta_parameters,
+    ) in input_deltas:
+        if input_delta_type is DeltaType.DELETE:
+            is_delete = True
+        staged_delta = ds.stage_delta(
+            input_delta,
+            staged_partition,
+            input_delta_type,
+            delete_parameters=input_delta_parameters,
+            **ds_mock_kwargs,
+        )
+        ds.commit_delta(
+            staged_delta,
+            **ds_mock_kwargs,
+        )
+        input_delta_length += len(input_delta) if input_delta else 0
+    ds.commit_partition(staged_partition, **ds_mock_kwargs)
+    source_table_stream_after_committed: Stream = ds.get_stream(
+        namespace=source_namespace,
+        table_name=source_table_name,
+        table_version=source_table_version,
+        **ds_mock_kwargs,
+    )
+    # create the destination table
+    (
+        destination_table_namespace,
+        destination_table_name,
+        destination_table_version,
+    ) = create_destination_table(
+        primary_keys, sort_keys, partition_keys, ds_mock_kwargs
+    )
+    # create the rebase table
+    (
+        rebase_table_namespace,
+        rebase_table_name,
+        rebase_table_version,
+    ) = create_rebase_table(primary_keys, sort_keys, partition_keys, ds_mock_kwargs)
+    rebasing_table_stream: Stream = ds.get_stream(
+        namespace=rebase_table_namespace,
+        table_name=rebase_table_name,
+        table_version=rebase_table_version,
+        **ds_mock_kwargs,
+    )
+    staged_partition: Partition = ds.stage_partition(
+        rebasing_table_stream, partition_values, **ds_mock_kwargs
+    )
+    input_delta_length = 0
+    for (
+        input_delta,
+        input_delta_type,
+        input_delta_parameters,
+    ) in input_deltas:
+        if input_delta_type is DeltaType.DELETE:
+            is_delete = True
+        staged_delta = ds.stage_delta(
+            input_delta,
+            staged_partition,
+            input_delta_type,
+            delete_parameters=input_delta_parameters,
+            **ds_mock_kwargs,
+        )
+        ds.commit_delta(
+            staged_delta,
+            **ds_mock_kwargs,
+        )
+        input_delta_length += len(input_delta) if input_delta else 0
+    ds.commit_partition(staged_partition, **ds_mock_kwargs)
+
+    # get streams
+    destination_table_stream: Stream = ds.get_stream(
+        namespace=destination_table_namespace,
+        table_name=destination_table_name,
+        table_version=destination_table_version,
+        **ds_mock_kwargs,
+    )
+    rebased_stream_after_committed: Stream = ds.get_stream(
+        namespace=rebase_table_namespace,
+        table_name=rebase_table_name,
+        table_version=rebase_table_version,
+        **ds_mock_kwargs,
+    )
+    return (
+        source_table_stream_after_committed,
+        destination_table_stream,
+        rebased_stream_after_committed,
+        is_delete,
+    )

deltacat/tests/local_deltacat_storage/__init__.py

@@ -394,12 +394,13 @@ def download_delta(
     **kwargs,
 ) -> Union[LocalDataset, DistributedDataset]:  # type: ignore
     result = []
-    manifest = get_delta_manifest(delta_like, *args, **kwargs)
-
+    if isinstance(delta_like, Delta) and delta_like.manifest is not None:
+        manifest = Delta(delta_like).manifest
+    else:
+        manifest = get_delta_manifest(delta_like, *args, **kwargs)
     partition_values: PartitionValues = None
     if partition_filter is not None:
         partition_values = partition_filter.partition_values
-
     for entry_index in range(len(manifest.entries)):
         if (
             partition_values is not None
@@ -440,8 +441,10 @@ def download_delta_manifest_entry(
     **kwargs,
 ) -> LocalTable:
     cur, con = _get_sqlite3_cursor_con(kwargs)
-
-    manifest = get_delta_manifest(delta_like, *args, **kwargs)
+    if isinstance(delta_like, Delta) and delta_like.manifest is not None:
+        manifest = Delta(delta_like).manifest
+    else:
+        manifest = get_delta_manifest(delta_like, *args, **kwargs)
     if entry_index >= len(manifest.entries):
         raise IndexError(
             f"Manifest entry index {entry_index} does not exist. "

{deltacat-1.1.14.dist-info → deltacat-1.1.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.14
+Version: 1.1.16
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team