deltacat 1.1.17__py3-none-any.whl → 1.1.19__py3-none-any.whl

deltacat/tests/compute/compact_partition_rebase_test_cases.py

@@ -84,6 +84,314 @@ REBASE_TEST_CASES = {
         skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
         assert_compaction_audit=None,
     ),
+    "2-rebase-with-null-pk": RebaseCompactionTestCaseParams(
+        primary_keys={"pk_col_1"},
+        sort_keys=[
+            SortKey.of(key_name="sk_col_1"),
+            SortKey.of(key_name="sk_col_2"),
+        ],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1]),
+                pa.array([1, 2, 3, 4, 5, 6]),
+                pa.array(["foo"] * 6),
+                pa.array([5, 6, 7, 8, 9, 10]),
+            ],
+            names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([None, 1, 2]),
+                pa.array([5, 6, 4]),
+                pa.array(["foo"] * 3),
+                pa.array([9, 10, 8]),
+            ],
+            names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([None, 1, 2]),
+                pa.array([5, 6, 4]),
+                pa.array(["foo"] * 3),
+                pa.array([7, 10, 8]),
+            ],
+            names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+    ),
+    "3-rebase-with-null-two-pk": RebaseCompactionTestCaseParams(
+        primary_keys={"pk_col_1", "pk_col_2"},
+        sort_keys=[
+            SortKey.of(key_name="sk_col_1"),
+        ],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5]),
+                pa.array([1, None, 3, None, None, 1, 5]),
+                pa.array(["foo"] * 7),
+                pa.array([5, 6, 7, 8, 9, 10, 11]),
+            ],
+            names=["pk_col_1", "pk_col_2", "sk_col_1", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 5, None]),
+                pa.array([1, None, 3, 5, None]),
+                pa.array(["foo"] * 5),
+                pa.array([10, 8, 7, 11, 9]),
+            ],
+            names=["pk_col_1", "pk_col_2", "sk_col_1", "col_1"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 5, None]),
+                pa.array([1, None, 3, 5, None]),
+                pa.array(["foo"] * 5),
+                pa.array([10, 8, 7, 11, 9]),
+            ],
+            names=["pk_col_1", "pk_col_2", "sk_col_1", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+    ),
+    "4-rebase-with-null-multiple-pk-different-types": RebaseCompactionTestCaseParams(
+        primary_keys={"pk_col_1", "pk_col_2", "pk_col_3"},
+        sort_keys=[],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 5, None, None]),
+                pa.array([1, None, 3, 5, None, None]),
+                pa.array(["a", "b", "c", "g", "e", None]),
+                pa.array([10, 8, 7, 11, 12, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 5, None, None]),
+                pa.array([1, None, 3, 5, None, None]),
+                pa.array(["a", "b", "c", "g", "e", None]),
+                pa.array([10, 8, 7, 11, 12, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+    ),
+    "5-rebase-with-null-multiple-pk-one-hash-bucket": RebaseCompactionTestCaseParams(
+        primary_keys={"pk_col_1", "pk_col_2", "pk_col_3"},
+        sort_keys=[],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 5, None, None]),
+                pa.array([1, None, 3, 5, None, None]),
+                pa.array(["a", "b", "c", "g", "e", None]),
+                pa.array([10, 8, 7, 11, 12, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 5, None, None]),
+                pa.array([1, None, 3, 5, None, None]),
+                pa.array(["a", "b", "c", "g", "e", None]),
+                pa.array([10, 8, 7, 11, 12, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=1,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+    ),
+    "6-rebase-with-null-multiple-pk-drop-duplicates-false": RebaseCompactionTestCaseParams(
+        primary_keys={"pk_col_1", "pk_col_2", "pk_col_3"},
+        sort_keys=[],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=False,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+    ),
+    "7-rebase-drop-duplicates-false": RebaseCompactionTestCaseParams(
+        primary_keys={"pk_col_1"},
+        sort_keys=[
+            SortKey.of(key_name="sk_col_1"),
+        ],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, 2, 3, 3, 1]),
+                pa.array([1, 2, 3, 4, 5, 6]),
+                pa.array(["a", "b", "c", "b", "e", "a"]),
+                pa.array([5, 6, 7, 8, 9, 10]),
+            ],
+            names=["pk_col_1", "sk_col_1", "col_1", "col_2"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, 2, 3, 3, 1]),
+                pa.array([1, 2, 3, 4, 5, 6]),
+                pa.array(["a", "b", "c", "b", "e", "a"]),
+                pa.array([5, 6, 7, 8, 9, 10]),
+            ],
+            names=["pk_col_1", "sk_col_1", "col_1", "col_2"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, 2, 3, 3, 1]),
+                pa.array([1, 2, 3, 4, 5, 6]),
+                pa.array(["a", "b", "c", "b", "e", "a"]),
+                pa.array([5, 6, 7, 8, 9, 10]),
+            ],
+            names=["pk_col_1", "sk_col_1", "col_1", "col_2"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=False,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+    ),
+    "8-rebase-with-with-null-pk-duplicates-false-hash-bucket-1": RebaseCompactionTestCaseParams(
+        primary_keys={"pk_col_1", "pk_col_2", "pk_col_3"},
+        sort_keys=[],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([1, 2, None, 2, None, 1, 5, None, None, None]),
+                pa.array([1, None, 3, None, None, 1, 5, None, None, None]),
+                pa.array(["a", "b", "c", "b", "e", "a", "g", "e", None, None]),
+                pa.array([5, 6, 7, 8, 9, 10, 11, 12, 13, 14]),
+            ],
+            names=["pk_col_1", "pk_col_2", "pk_col_3", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=1,
+        hash_bucket_count=1,
+        read_kwargs_provider=None,
+        drop_duplicates=False,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=None,
+    ),
 }
 
 REBASE_TEST_CASES = with_compactor_version_func_test_param(REBASE_TEST_CASES)

deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py

@@ -798,6 +798,67 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
         skip_enabled_compact_partition_drivers=None,
         assert_compaction_audit=None,
     ),
+    "14-rebase-then-incremental-with-null-pk": RebaseThenIncrementalCompactionTestCaseParams(
+        primary_keys={"pk_col_1"},
+        sort_keys=[
+            SortKey.of(key_name="sk_col_1"),
+            SortKey.of(key_name="sk_col_2"),
+        ],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([str(i) for i in range(9)] + [None]),
+                pa.array([i for i in range(0, 10)]),
+                pa.array(["foo"] * 10),
+                pa.array([i / 10 for i in range(10, 20)]),
+            ],
+            names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([str(i) for i in range(9)] + [None]),
+                pa.array([i for i in range(0, 10)]),
+                pa.array(["foo"] * 10),
+                pa.array([i / 10 for i in range(10, 20)]),
+            ],
+            names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
+        ),
+        incremental_deltas=[
+            (
+                pa.Table.from_arrays(
+                    [
+                        pa.array([str(i) for i in range(9)] + [None]),
+                        pa.array([i for i in range(20, 30)]),
+                        pa.array(["foo"] * 10),
+                        pa.array([i / 10 for i in range(40, 50)]),
+                    ],
+                    names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
+                ),
+                DeltaType.UPSERT,
+                None,
+            )
+        ],
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([str(i) for i in range(9)] + [None]),
+                pa.array([i for i in range(20, 30)]),
+                pa.array(["foo"] * 10),
+                pa.array([i / 10 for i in range(40, 50)]),
+            ],
+            names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        skip_enabled_compact_partition_drivers=None,
+        assert_compaction_audit=assert_compaction_audit,
+    ),
 }
 
 REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
@@ -1983,6 +2044,104 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
         skip_enabled_compact_partition_drivers=None,
         assert_compaction_audit=assert_compaction_audit_no_hash_bucket,
     ),
+    "31-rebase-then-incremental-delete-delta-on-incremental-null-pk-delete-null": RebaseThenIncrementalCompactionTestCaseParams(
+        primary_keys={"pk_col_1"},
+        sort_keys=ZERO_VALUED_SORT_KEY,
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([i for i in range(11)] + [None]),
+                pa.array([str(i) for i in range(0, 12)]),
+            ],
+            names=["pk_col_1", "col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        rebase_expected_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([i for i in range(11)] + [None]),
+                pa.array([str(i) for i in range(0, 12)]),
+            ],
+            names=["pk_col_1", "col_1"],
+        ),
+        incremental_deltas=[
+            (
+                pa.Table.from_arrays(
+                    [
+                        pa.array([10, 11, None, 13]),
+                        pa.array(["a", "b", "c", "d"]),
+                    ],
+                    names=["pk_col_1", "col_1"],
+                ),
+                DeltaType.UPSERT,
+                None,
+            ),
+            (
+                pa.Table.from_arrays(
+                    [pa.array([10, 11]), pa.array(["a", "b"])],
+                    names=["pk_col_1", "col_1"],
+                ),
+                DeltaType.DELETE,
+                DeleteParameters.of(["pk_col_1", "col_1"]),
+            ),
+            (
+                pa.Table.from_arrays(
+                    [pa.array([None])],  # Support deleting null PK records
+                    names=["pk_col_1"],
+                ),
+                DeltaType.DELETE,
+                DeleteParameters.of(["pk_col_1"]),
+            ),
+            (
+                pa.Table.from_arrays(
+                    [pa.array(["c"])],
+                    names=["col_1"],
+                ),
+                DeltaType.DELETE,
+                DeleteParameters.of(["col_1"]),
+            ),
+            (
+                pa.Table.from_arrays(
+                    [pa.array(["c"])],
+                    names=["col_1"],
+                ),
+                DeltaType.DELETE,
+                DeleteParameters.of(["col_1"]),
+            ),
+            (
+                pa.Table.from_arrays(
+                    [pa.array([10, 11]), pa.array(["a", "b"])],
+                    names=["pk_col_1", "col_1"],
+                ),
+                DeltaType.DELETE,
+                DeleteParameters.of(["pk_col_1", "col_1"]),
+            ),
+            (
+                pa.Table.from_arrays(
+                    [pa.array(["c"])],
+                    names=["col_1"],
+                ),
+                DeltaType.DELETE,
+                DeleteParameters.of(["col_1"]),
+            ),
+        ],
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([i for i in range(10)] + [13]),
+                pa.array([str(i) for i in range(0, 10)] + ["d"]),
+            ],
+            names=["pk_col_1", "col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+        assert_compaction_audit=assert_compaction_audit,
+    ),
 }
 
 REBASE_THEN_INCREMENTAL_TEST_CASES = with_compactor_version_func_test_param(

deltacat/tests/compute/compactor_v2/test_compaction_session.py

@@ -19,6 +19,7 @@ from deltacat.tests.test_utils.utils import read_s3_contents
 from deltacat.tests.compute.test_util_constant import (
     TEST_S3_RCF_BUCKET_NAME,
 )
+from deltacat.compute.resource_estimation import ResourceEstimationMethod
 from deltacat.tests.compute.test_util_common import get_rcf
 from deltacat.tests.test_utils.pyarrow import (
     stage_partition_from_file_paths,
@@ -399,3 +400,159 @@ class TestCompactionSession:
         assert compaction_audit.output_file_count == 2
         assert abs(compaction_audit.output_size_bytes - 1843) / 1843 <= self.ERROR_RATE
         assert abs(compaction_audit.input_size_bytes - 2748) / 2748 <= self.ERROR_RATE
+
+    def test_compact_partition_when_incremental_then_intelligent_estimation_sanity(
+        self, s3_resource, local_deltacat_storage_kwargs
+    ):
+        """
+        A test case which asserts the RCF stats are correctly generated for
+        a rebase and incremental use-case.
+        """
+
+        # setup
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
+        )
+
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, [self.BACKFILL_FILE_PATH], **local_deltacat_storage_kwargs
+        )
+
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
+
+        # action
+        compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 2,
+                    "last_stream_position_to_compact": source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                    "resource_estimation_method": ResourceEstimationMethod.INTELLIGENT_ESTIMATION,
+                }
+            )
+        )
+
+    def test_compact_partition_when_incremental_then_content_type_meta_estimation_sanity(
+        self, s3_resource, local_deltacat_storage_kwargs
+    ):
+        """
+        A test case which asserts the RCF stats are correctly generated for
+        a rebase and incremental use-case.
+        """
+
+        # setup
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
+        )
+
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, [self.BACKFILL_FILE_PATH], **local_deltacat_storage_kwargs
+        )
+
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
+
+        # action
+        compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 2,
+                    "last_stream_position_to_compact": source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                    "resource_estimation_method": ResourceEstimationMethod.CONTENT_TYPE_META,
+                }
+            )
+        )
+
+    def test_compact_partition_when_incremental_then_previous_inflation_estimation_sanity(
+        self, s3_resource, local_deltacat_storage_kwargs
+    ):
+        """
+        A test case which asserts the RCF stats are correctly generated for
+        a rebase and incremental use-case.
+        """
+
+        # setup
+        staged_source = stage_partition_from_file_paths(
+            self.NAMESPACE, ["source"], **local_deltacat_storage_kwargs
+        )
+
+        source_delta = commit_delta_to_staged_partition(
+            staged_source, [self.BACKFILL_FILE_PATH], **local_deltacat_storage_kwargs
+        )
+
+        staged_dest = stage_partition_from_file_paths(
+            self.NAMESPACE, ["destination"], **local_deltacat_storage_kwargs
+        )
+        dest_partition = ds.commit_partition(
+            staged_dest, **local_deltacat_storage_kwargs
+        )
+
+        # action
+        compact_partition(
+            CompactPartitionParams.of(
+                {
+                    "compaction_artifact_s3_bucket": TEST_S3_RCF_BUCKET_NAME,
+                    "compacted_file_content_type": ContentType.PARQUET,
+                    "dd_max_parallelism_ratio": 1.0,
+                    "deltacat_storage": ds,
+                    "deltacat_storage_kwargs": local_deltacat_storage_kwargs,
+                    "destination_partition_locator": dest_partition.locator,
+                    "drop_duplicates": True,
+                    "hash_bucket_count": 2,
+                    "last_stream_position_to_compact": source_delta.stream_position,
+                    "list_deltas_kwargs": {
+                        **local_deltacat_storage_kwargs,
+                        **{"equivalent_table_types": []},
+                    },
+                    "primary_keys": ["pk"],
+                    "rebase_source_partition_locator": source_delta.partition_locator,
+                    "rebase_source_partition_high_watermark": source_delta.stream_position,
+                    "records_per_compacted_file": 4000,
+                    "s3_client_kwargs": {},
+                    "source_partition_locator": source_delta.partition_locator,
+                    "resource_estimation_method": ResourceEstimationMethod.PREVIOUS_INFLATION,
+                }
+            )
+        )

deltacat/tests/compute/compactor_v2/utils/test_task_options.py

@@ -1,6 +1,6 @@
 import unittest
 import ray
-from deltacat.compute.compactor_v2.utils.task_options import get_task_options
+from deltacat.compute.compactor_v2.utils.task_options import _get_task_options
 
 
 @ray.remote
@@ -20,14 +20,14 @@ class TestTaskOptions(unittest.TestCase):
         super().setUpClass()
 
     def test_get_task_options_sanity(self):
-        opts = get_task_options(0.01, 0.01)
+        opts = _get_task_options(0.01, 0.01)
         result_ref = valid_func.options(**opts).remote()
         result = ray.get(result_ref)
 
         self.assertEqual(result, 2)
 
     def test_get_task_options_when_exception_is_thrown(self):
-        opts = get_task_options(0.01, 0.01)
+        opts = _get_task_options(0.01, 0.01)
         result_ref = throwing_func.options(**opts).remote()
 
         self.assertRaises(ConnectionAbortedError, lambda: ray.get(result_ref))