deltacat 1.1.3__py3-none-any.whl → 1.1.5__py3-none-any.whl

deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py

@@ -107,6 +107,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -186,6 +187,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "pk_col_2", "sk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -234,6 +236,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -282,6 +285,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -336,6 +340,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -395,6 +400,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=10,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT + 10,
@@ -445,6 +451,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["sk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=10,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -500,6 +507,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -559,6 +567,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=1,
@@ -613,6 +622,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -658,6 +668,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=3,
@@ -717,6 +728,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=1,
@@ -762,6 +774,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=1,
@@ -861,6 +874,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -911,6 +925,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "pk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -967,6 +982,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             ),
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1019,6 +1035,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             ),
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1099,6 +1116,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1168,6 +1186,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1231,6 +1250,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             ),
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1321,6 +1341,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1384,6 +1405,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             ),
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1453,6 +1475,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1516,6 +1539,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             ),
         ),
         expected_terminal_exception=AssertionError,
+        expected_terminal_exception_message="Delete type deltas are required to have delete parameters defined",
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1580,6 +1604,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1638,6 +1663,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1706,6 +1732,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1795,6 +1822,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=1,
@@ -1865,6 +1893,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["sk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=10,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1913,6 +1942,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,

deltacat/tests/compute/compact_partition_test_cases.py

@@ -54,6 +54,7 @@ class BaseCompactorTestCase:
         input_deltas_delta_type: DeltaType - enumerated argument required for delta creation during compact_partition test setup. Available values are (DeltaType.APPEND, DeltaType.UPSERT, DeltaType.DELETE). DeltaType.APPEND is not supported by compactor v1 or v2
         expected_terminal_compact_partition_result: pa.Table - expected PyArrow table after compaction (i.e,. the state of the table after applying all row UPDATES/DELETES/INSERTS)
         expected_terminal_exception: BaseException - expected exception during compaction
+        expected_terminal_exception_message: Optional[str] - expected exception message if present.
         do_create_placement_group: bool - toggles whether to create a placement group (https://docs.ray.io/en/latest/ray-core/scheduling/placement-group.html) or not
         records_per_compacted_file: int - argument for the records_per_compacted_file parameter in compact_partition
         hash_bucket_count_param: int - argument for the hash_bucket_count parameter in compact_partition
@@ -70,6 +71,7 @@ class BaseCompactorTestCase:
     input_deltas_delta_type: DeltaType
     expected_terminal_compact_partition_result: pa.Table
     expected_terminal_exception: BaseException
+    expected_terminal_exception_message: str
     do_create_placement_group: bool
     records_per_compacted_file: int
     hash_bucket_count: int
@@ -84,7 +86,12 @@ class BaseCompactorTestCase:
 
 @dataclass(frozen=True)
 class IncrementalCompactionTestCaseParams(BaseCompactorTestCase):
-    pass
+    """
+    Args:
+        is_inplace: bool - argument to indicate whether to try compacting an in-place compacted table (the source table is the destination table). Also needed to control whether the destination table is created
+    """
+
+    is_inplace: bool
 
 
 @dataclass(frozen=True)
@@ -134,11 +141,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "2-incremental-pkstr-skstr-norcf": IncrementalCompactionTestCaseParams(
@@ -159,11 +168,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "3-incremental-pkstr-multiskstr-norcf": IncrementalCompactionTestCaseParams(
@@ -193,11 +204,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1", "sk_col_2"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "4-incremental-duplicate-pk": IncrementalCompactionTestCaseParams(
@@ -226,11 +239,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1", "sk_col_2"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "5-incremental-decimal-pk-simple": IncrementalCompactionTestCaseParams(
@@ -254,11 +269,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "6-incremental-integer-pk-simple": IncrementalCompactionTestCaseParams(
@@ -282,11 +299,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "7-incremental-timestamp-pk-simple": IncrementalCompactionTestCaseParams(
@@ -310,11 +329,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "8-incremental-decimal-timestamp-pk-multi": IncrementalCompactionTestCaseParams(
@@ -340,11 +361,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "pk_col_2", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "9-incremental-decimal-pk-multi-dup": IncrementalCompactionTestCaseParams(
@@ -368,11 +391,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "10-incremental-decimal-pk-partitionless": IncrementalCompactionTestCaseParams(
@@ -396,11 +421,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "11-incremental-decimal-hash-bucket-single": IncrementalCompactionTestCaseParams(
@@ -424,11 +451,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "12-incremental-decimal-single-hash-bucket": IncrementalCompactionTestCaseParams(
@@ -452,13 +481,45 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=1,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
+    "13-incremental-pkstr-skexists-isinplacecompacted": IncrementalCompactionTestCaseParams(
+        primary_keys={"pk_col_1"},
+        sort_keys=[SortKey.of(key_name="sk_col_1")],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([str(i) for i in range(10)]),
+                pa.array([i for i in range(10)]),
+            ],
+            names=["pk_col_1", "sk_col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([str(i) for i in range(10)]),
+                pa.array([i for i in range(10)]),
+            ],
+            names=["pk_col_1", "sk_col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        is_inplace=True,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+    ),
 }
 
 INCREMENTAL_TEST_CASES = with_compactor_version_func_test_param(INCREMENTAL_TEST_CASES)

deltacat/tests/compute/test_compact_partition_incremental.py

@@ -24,6 +24,9 @@ from deltacat.tests.compute.test_util_constant import (
     DEFAULT_NUM_WORKERS,
     DEFAULT_WORKER_INSTANCE_CPUS,
 )
+from deltacat.compute.compactor import (
+    RoundCompletionInfo,
+)
 
 DATABASE_FILE_PATH_KEY, DATABASE_FILE_PATH_VALUE = (
     "db_file_path",
@@ -101,12 +104,14 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         "input_deltas_delta_type",
         "expected_terminal_compact_partition_result",
         "expected_terminal_exception",
+        "expected_terminal_exception_message",
         "create_placement_group_param",
         "records_per_compacted_file_param",
         "hash_bucket_count_param",
         "read_kwargs_provider_param",
         "drop_duplicates_param",
         "skip_enabled_compact_partition_drivers",
+        "is_inplace",
         "compact_partition_func",
     ],
     [
@@ -120,12 +125,14 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             input_deltas_delta_type,
             expected_terminal_compact_partition_result,
             expected_terminal_exception,
+            expected_terminal_exception_message,
             create_placement_group_param,
             records_per_compacted_file_param,
             hash_bucket_count_param,
             drop_duplicates_param,
             read_kwargs_provider,
             skip_enabled_compact_partition_drivers,
+            is_inplace,
             compact_partition_func,
         )
         for test_name, (
@@ -137,17 +144,18 @@ def offer_local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             input_deltas_delta_type,
             expected_terminal_compact_partition_result,
             expected_terminal_exception,
+            expected_terminal_exception_message,
             create_placement_group_param,
             records_per_compacted_file_param,
             hash_bucket_count_param,
             drop_duplicates_param,
             read_kwargs_provider,
             skip_enabled_compact_partition_drivers,
+            is_inplace,
             compact_partition_func,
         ) in INCREMENTAL_TEST_CASES.items()
     ],
     ids=[test_name for test_name in INCREMENTAL_TEST_CASES],
-    indirect=[],
 )
 def test_compact_partition_incremental(
     setup_s3_resource: ServiceResource,
@@ -161,12 +169,14 @@ def test_compact_partition_incremental(
     input_deltas_delta_type: str,
     expected_terminal_compact_partition_result: pa.Table,
     expected_terminal_exception: BaseException,
+    expected_terminal_exception_message: Optional[str],
     create_placement_group_param: bool,
     records_per_compacted_file_param: int,
     hash_bucket_count_param: int,
     drop_duplicates_param: bool,
     read_kwargs_provider_param: Any,
     skip_enabled_compact_partition_drivers,
+    is_inplace: bool,
     compact_partition_func: Callable,
     benchmark: BenchmarkFixture,
 ):
@@ -174,6 +184,7 @@ def test_compact_partition_incremental(
     from deltacat.types.media import ContentType
     from deltacat.storage import (
         DeltaLocator,
+        Partition,
         PartitionLocator,
     )
     from deltacat.compute.compactor.model.compaction_session_audit_info import (
@@ -186,7 +197,7 @@ def test_compact_partition_incremental(
         PlacementGroupManager,
     )
 
-    ds_mock_kwargs = offer_local_deltacat_storage_kwargs
+    ds_mock_kwargs: Dict[str, Any] = offer_local_deltacat_storage_kwargs
 
     # setup
     partition_keys = partition_keys_param
@@ -202,20 +213,21 @@ def test_compact_partition_incremental(
         input_deltas_delta_type,
         partition_values_param,
         ds_mock_kwargs,
+        is_inplace,
     )
-    source_partition = ds.get_partition(
+    source_partition: Partition = ds.get_partition(
         source_table_stream.locator,
         partition_values_param,
         **ds_mock_kwargs,
     )
-    destination_partition_locator = PartitionLocator.of(
+    destination_partition_locator: PartitionLocator = PartitionLocator.of(
         destination_table_stream.locator,
         partition_values_param,
         None,
     )
     num_workers, worker_instance_cpu = DEFAULT_NUM_WORKERS, DEFAULT_WORKER_INSTANCE_CPUS
-    total_cpus = num_workers * worker_instance_cpu
-    pgm = None
+    total_cpus: int = num_workers * worker_instance_cpu
+    pgm: Optional[PlacementGroupManager] = None
     if create_placement_group_param:
         pgm = PlacementGroupManager(
             1, total_cpus, worker_instance_cpu, memory_per_bundle=4000000
@@ -260,14 +272,16 @@ def test_compact_partition_incremental(
         compact_partition_func, setup=_incremental_compaction_setup
     )
     # validate
-    round_completion_info = get_rcf(setup_s3_resource, rcf_file_s3_uri)
+    round_completion_info: RoundCompletionInfo = get_rcf(
+        setup_s3_resource, rcf_file_s3_uri
+    )
     compacted_delta_locator: DeltaLocator = (
         round_completion_info.compacted_delta_locator
     )
     audit_bucket, audit_key = round_completion_info.compaction_audit_url.replace(
         "s3://", ""
     ).split("/", 1)
-    compaction_audit_obj: dict = read_s3_contents(
+    compaction_audit_obj: Dict[str, Any] = read_s3_contents(
         setup_s3_resource, audit_bucket, audit_key
     )
     compaction_audit: CompactionSessionAuditInfo = CompactionSessionAuditInfo(
@@ -281,7 +295,7 @@ def test_compact_partition_incremental(
     sorting_cols: List[Any] = [(val, "ascending") for val in primary_keys]
     # the compacted table may contain multiple files and chunks
     # and order of records may be incorrect due to multiple files.
-    expected_terminal_compact_partition_result = (
+    expected_terminal_compact_partition_result: pa.Table = (
         expected_terminal_compact_partition_result.combine_chunks().sort_by(
             sorting_cols
         )
@@ -297,4 +311,15 @@ def test_compact_partition_incremental(
     assert actual_compacted_table.equals(
         expected_terminal_compact_partition_result
     ), f"{actual_compacted_table} does not match {expected_terminal_compact_partition_result}"
+
+    if is_inplace:
+        assert (
+            source_partition.locator.partition_values
+            == destination_partition_locator.partition_values
+            and source_partition.locator.stream_id
+            == destination_partition_locator.stream_id
+        ), "The source partition should match the destination partition"
+        assert (
+            compacted_delta_locator.stream_id == source_partition.locator.stream_id
+        ), "The compacted delta should be in the same stream as the source"
     return

deltacat/tests/compute/test_compact_partition_rebase_then_incremental.py

@@ -115,6 +115,7 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
         "input_deltas_delta_type",
         "expected_terminal_compact_partition_result",
         "expected_terminal_exception",
+        "expected_terminal_exception_message",
         "create_placement_group_param",
         "records_per_compacted_file_param",
         "hash_bucket_count_param",
@@ -136,6 +137,7 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             input_deltas_delta_type,
             expected_terminal_compact_partition_result,
             expected_terminal_exception,
+            expected_terminal_exception_message,
             create_placement_group_param,
             records_per_compacted_file_param,
             hash_bucket_count_param,
@@ -155,6 +157,7 @@ def local_deltacat_storage_kwargs(request: pytest.FixtureRequest):
             input_deltas_delta_type,
             expected_terminal_compact_partition_result,
             expected_terminal_exception,
+            expected_terminal_exception_message,
             create_placement_group_param,
             records_per_compacted_file_param,
             hash_bucket_count_param,
@@ -180,6 +183,7 @@ def test_compact_partition_rebase_then_incremental(
     input_deltas_delta_type: str,
     expected_terminal_compact_partition_result: pa.Table,
     expected_terminal_exception: BaseException,
+    expected_terminal_exception_message: Optional[str],
     create_placement_group_param: bool,
     records_per_compacted_file_param: int,
     hash_bucket_count_param: int,
@@ -337,8 +341,9 @@ def test_compact_partition_rebase_then_incremental(
         }
     )
     if expected_terminal_exception:
-        with pytest.raises(expected_terminal_exception):
+        with pytest.raises(expected_terminal_exception) as exc_info:
             compact_partition_func(compact_partition_params)
+        assert expected_terminal_exception_message in str(exc_info.value)
         return
     rcf_file_s3_uri = compact_partition_func(compact_partition_params)
     round_completion_info = get_rcf(setup_s3_resource, rcf_file_s3_uri)

deltacat/tests/compute/test_util_common.py

@@ -20,6 +20,9 @@ from deltacat.tests.compute.test_util_constant import (
     REBASING_TABLE_NAME,
     REBASING_TABLE_VERSION,
 )
+from deltacat.compute.compactor import (
+    RoundCompletionInfo,
+)
 
 
 class PartitionKeyType(str, Enum):
@@ -134,11 +137,8 @@ def create_rebase_table(
     )
 
 
-def get_rcf(s3_resource, rcf_file_s3_uri: str):
+def get_rcf(s3_resource, rcf_file_s3_uri: str) -> RoundCompletionInfo:
     from deltacat.tests.test_utils.utils import read_s3_contents
-    from deltacat.compute.compactor import (
-        RoundCompletionInfo,
-    )
 
     _, rcf_object_key = rcf_file_s3_uri.rsplit("/", 1)
     rcf_file_output: Dict[str, Any] = read_s3_contents(