deltacat 1.1.4__tar.gz → 1.1.5__tar.gz

{deltacat-1.1.4 → deltacat-1.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 1.1.4
+Version: 1.1.5
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team

{deltacat-1.1.4 → deltacat-1.1.5}/deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.4"
+__version__ = "1.1.5"
 
 
 __all__ = [

{deltacat-1.1.4 → deltacat-1.1.5}/deltacat/compute/compactor_v2/compaction_session.py

@@ -17,7 +17,11 @@ from deltacat.compute.compactor_v2.model.merge_input import MergeInput
 from deltacat.aws import s3u as s3_utils
 import deltacat
 from deltacat import logs
-from deltacat.compute.compactor import PyArrowWriteResult, RoundCompletionInfo
+from deltacat.compute.compactor import (
+    HighWatermark,
+    PyArrowWriteResult,
+    RoundCompletionInfo,
+)
 from deltacat.compute.compactor_v2.model.merge_result import MergeResult
 from deltacat.compute.compactor_v2.model.hash_bucket_result import HashBucketResult
 from deltacat.compute.compactor.model.materialize_result import MaterializeResult
@@ -37,6 +41,7 @@ from deltacat.compute.compactor_v2.deletes.utils import prepare_deletes
 from deltacat.storage import (
     Delta,
     DeltaLocator,
+    Manifest,
     Partition,
 )
 from deltacat.compute.compactor.model.compact_partition_params import (
@@ -96,7 +101,7 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
         round_completion_file_s3_url = None
         if new_partition:
             logger.info(f"Committing compacted partition to: {new_partition.locator}")
-            partition = params.deltacat_storage.commit_partition(
+            partition: Partition = params.deltacat_storage.commit_partition(
                 new_partition, **params.deltacat_storage_kwargs
             )
             logger.info(f"Committed compacted partition: {partition}")
@@ -150,9 +155,9 @@ def _execute_compaction(
         compaction_audit.set_total_cluster_memory_bytes(cluster_memory)
 
     # read the results from any previously completed compaction round
-    round_completion_info = None
-    high_watermark = None
-    previous_compacted_delta_manifest = None
+    round_completion_info: Optional[RoundCompletionInfo] = None
+    high_watermark: Optional[HighWatermark] = None
+    previous_compacted_delta_manifest: Optional[Manifest] = None
 
     if not params.rebase_source_partition_locator:
         round_completion_info = rcf.read_round_completion_file(
@@ -271,6 +276,7 @@ def _execute_compaction(
     total_hb_record_count = np.int64(0)
     telemetry_time_hb = 0
     if params.hash_bucket_count == 1:
+        logger.info("Hash bucket count set to 1. Running local merge")
         merge_start = time.monotonic()
         local_merge_input = generate_local_merge_input(
             params,
@@ -634,6 +640,19 @@ def _execute_compaction(
         f"partition-{params.source_partition_locator.partition_values},"
         f"compacted at: {params.last_stream_position_to_compact},"
     )
+    is_inplace_compacted: bool = (
+        params.source_partition_locator.partition_values
+        == params.destination_partition_locator.partition_values
+        and params.source_partition_locator.stream_id
+        == params.destination_partition_locator.stream_id
+    )
+    if is_inplace_compacted:
+        logger.info(
+            "Overriding round completion file source partition locator as in-place compacted. "
+            + f"Got compacted partition partition_id of {compacted_partition.locator.partition_id} "
+            f"and rcf source partition_id of {rcf_source_partition_locator.partition_id}."
+        )
+        rcf_source_partition_locator = compacted_partition.locator
     return (
         compacted_partition,
         new_round_completion_info,

{deltacat-1.1.4 → deltacat-1.1.5}/deltacat/compute/compactor_v2/utils/task_options.py

@@ -21,6 +21,9 @@ from deltacat.compute.compactor_v2.constants import (
     PARQUET_TO_PYARROW_INFLATION,
 )
 
+from daft.exceptions import DaftTransientError
+
+
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
@@ -76,6 +79,7 @@ def get_task_options(
         botocore.exceptions.HTTPClientError,
         ConnectionError,
         TimeoutError,
+        DaftTransientError,
     ]
 
     return task_opts

{deltacat-1.1.4 → deltacat-1.1.5}/deltacat/tests/compute/compact_partition_rebase_then_incremental_test_cases.py

@@ -107,6 +107,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -186,6 +187,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "pk_col_2", "sk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -234,6 +236,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -282,6 +285,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -336,6 +340,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -395,6 +400,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=10,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT + 10,
@@ -445,6 +451,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["sk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=10,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -500,6 +507,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -559,6 +567,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=1,
@@ -613,6 +622,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -658,6 +668,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=3,
@@ -717,6 +728,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=1,
@@ -762,6 +774,7 @@ REBASE_THEN_INCREMENTAL_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=1,
@@ -861,6 +874,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -911,6 +925,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "pk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -967,6 +982,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             ),
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1019,6 +1035,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             ),
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1099,6 +1116,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1168,6 +1186,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1231,6 +1250,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             ),
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1321,6 +1341,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1384,6 +1405,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             ),
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1453,6 +1475,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1516,6 +1539,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             ),
         ),
         expected_terminal_exception=AssertionError,
+        expected_terminal_exception_message="Delete type deltas are required to have delete parameters defined",
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1580,6 +1604,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1638,6 +1663,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1706,6 +1732,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=True,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1795,6 +1822,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "sk_col_1", "sk_col_2", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=1,
@@ -1865,6 +1893,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["sk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=10,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
@@ -1913,6 +1942,7 @@ REBASE_THEN_INCREMENTAL_DELETE_DELTA_TYPE_TEST_CASES = {
             names=["pk_col_1", "col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,

{deltacat-1.1.4 → deltacat-1.1.5}/deltacat/tests/compute/compact_partition_test_cases.py

@@ -54,6 +54,7 @@ class BaseCompactorTestCase:
         input_deltas_delta_type: DeltaType - enumerated argument required for delta creation during compact_partition test setup. Available values are (DeltaType.APPEND, DeltaType.UPSERT, DeltaType.DELETE). DeltaType.APPEND is not supported by compactor v1 or v2
         expected_terminal_compact_partition_result: pa.Table - expected PyArrow table after compaction (i.e,. the state of the table after applying all row UPDATES/DELETES/INSERTS)
         expected_terminal_exception: BaseException - expected exception during compaction
+        expected_terminal_exception_message: Optional[str] - expected exception message if present.
         do_create_placement_group: bool - toggles whether to create a placement group (https://docs.ray.io/en/latest/ray-core/scheduling/placement-group.html) or not
         records_per_compacted_file: int - argument for the records_per_compacted_file parameter in compact_partition
         hash_bucket_count_param: int - argument for the hash_bucket_count parameter in compact_partition
@@ -70,6 +71,7 @@ class BaseCompactorTestCase:
     input_deltas_delta_type: DeltaType
     expected_terminal_compact_partition_result: pa.Table
     expected_terminal_exception: BaseException
+    expected_terminal_exception_message: str
     do_create_placement_group: bool
     records_per_compacted_file: int
     hash_bucket_count: int
@@ -84,7 +86,12 @@ class BaseCompactorTestCase:
 
 @dataclass(frozen=True)
 class IncrementalCompactionTestCaseParams(BaseCompactorTestCase):
-    pass
+    """
+    Args:
+        is_inplace: bool - argument to indicate whether to try compacting an in-place compacted table (the source table is the destination table). Also needed to control whether the destination table is created
+    """
+
+    is_inplace: bool
 
 
 @dataclass(frozen=True)
@@ -134,11 +141,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "2-incremental-pkstr-skstr-norcf": IncrementalCompactionTestCaseParams(
@@ -159,11 +168,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "3-incremental-pkstr-multiskstr-norcf": IncrementalCompactionTestCaseParams(
@@ -193,11 +204,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1", "sk_col_2"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "4-incremental-duplicate-pk": IncrementalCompactionTestCaseParams(
@@ -226,11 +239,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1", "sk_col_2"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "5-incremental-decimal-pk-simple": IncrementalCompactionTestCaseParams(
@@ -254,11 +269,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "6-incremental-integer-pk-simple": IncrementalCompactionTestCaseParams(
@@ -282,11 +299,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "7-incremental-timestamp-pk-simple": IncrementalCompactionTestCaseParams(
@@ -310,11 +329,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "8-incremental-decimal-timestamp-pk-multi": IncrementalCompactionTestCaseParams(
@@ -340,11 +361,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "pk_col_2", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "9-incremental-decimal-pk-multi-dup": IncrementalCompactionTestCaseParams(
@@ -368,11 +391,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "10-incremental-decimal-pk-partitionless": IncrementalCompactionTestCaseParams(
@@ -396,11 +421,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "11-incremental-decimal-hash-bucket-single": IncrementalCompactionTestCaseParams(
@@ -424,11 +451,13 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
     "12-incremental-decimal-single-hash-bucket": IncrementalCompactionTestCaseParams(
@@ -452,13 +481,45 @@ INCREMENTAL_TEST_CASES: Dict[str, IncrementalCompactionTestCaseParams] = {
             names=["pk_col_1", "sk_col_1"],
         ),
         expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
         do_create_placement_group=False,
         records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
         hash_bucket_count=1,
         read_kwargs_provider=None,
         drop_duplicates=True,
+        is_inplace=False,
         skip_enabled_compact_partition_drivers=None,
     ),
+    "13-incremental-pkstr-skexists-isinplacecompacted": IncrementalCompactionTestCaseParams(
+        primary_keys={"pk_col_1"},
+        sort_keys=[SortKey.of(key_name="sk_col_1")],
+        partition_keys=[PartitionKey.of("region_id", PartitionKeyType.INT)],
+        partition_values=["1"],
+        input_deltas=pa.Table.from_arrays(
+            [
+                pa.array([str(i) for i in range(10)]),
+                pa.array([i for i in range(10)]),
+            ],
+            names=["pk_col_1", "sk_col_1"],
+        ),
+        input_deltas_delta_type=DeltaType.UPSERT,
+        expected_terminal_compact_partition_result=pa.Table.from_arrays(
+            [
+                pa.array([str(i) for i in range(10)]),
+                pa.array([i for i in range(10)]),
+            ],
+            names=["pk_col_1", "sk_col_1"],
+        ),
+        expected_terminal_exception=None,
+        expected_terminal_exception_message=None,
+        do_create_placement_group=False,
+        records_per_compacted_file=DEFAULT_MAX_RECORDS_PER_FILE,
+        hash_bucket_count=DEFAULT_HASH_BUCKET_COUNT,
+        read_kwargs_provider=None,
+        drop_duplicates=True,
+        is_inplace=True,
+        skip_enabled_compact_partition_drivers=[CompactorVersion.V1],
+    ),
 }
 
 INCREMENTAL_TEST_CASES = with_compactor_version_func_test_param(INCREMENTAL_TEST_CASES)