deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/compute/compactor/model/delta_file_envelope.py

@@ -37,7 +37,9 @@ class DeltaFileEnvelope(dict):
                 pointing to a file from the uncompacted source table, False if
                 this Locator is pointing to a file in the compacted destination
                 table.
-            table_storage_strategy: The way the table object is stored in the delta file envelope. If None just stores the table normally
+            file_record_count: Record count in the delta file table.
+            table_storage_strategy: The way the table object is stored in the
+                delta file envelope. If None just stores the table normally
         Returns:
             A delta file envelope.
 

deltacat/compute/compactor/model/delta_file_locator.py

@@ -31,9 +31,11 @@ class DeltaFileLocator(Locator, tuple):
 
             file_index: Index of the file in the Delta Manifest.
 
+            file_record_count: Count of records in the Delta File.
+
         Returns:
             delta_file_locator: The Delta File Locator Tuple as
-            (is_source_delta, stream_position, file_index).
+            (is_src_delta, stream_position, file_index, file_record_count).
         """
         return DeltaFileLocator(
             (is_src_delta, stream_position, file_index, file_record_count)

deltacat/compute/compactor/model/round_completion_info.py

@@ -1,7 +1,7 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-from typing import Tuple
+from typing import Tuple, Union
 from deltacat.storage import DeltaLocator, PartitionLocator
 from deltacat.compute.compactor.model.pyarrow_write_result import PyArrowWriteResult
 from typing import Any, Dict, Optional
@@ -10,7 +10,7 @@ from typing import Any, Dict, Optional
 class HighWatermark(dict):
     """
     Inherit from dict to make it easy for serialization/deserialization.
-    Keep both partition locator and high watermark as a tuple to be persisted in the rcf
+    Keep both partition locator and high watermark as a tuple to be persisted in the rci
     """
 
     def set(self, partition_locator: PartitionLocator, delta_stream_position: int):
@@ -34,7 +34,7 @@ class RoundCompletionInfo(dict):
 
     @staticmethod
     def of(
-        high_watermark: HighWatermark,
+        high_watermark: Union[HighWatermark, int],
         compacted_delta_locator: DeltaLocator,
         compacted_pyarrow_write_result: PyArrowWriteResult,
         sort_keys_bit_width: int,
@@ -46,6 +46,7 @@ class RoundCompletionInfo(dict):
         compactor_version: Optional[str] = None,
         input_inflation: Optional[float] = None,
         input_average_record_size_bytes: Optional[float] = None,
+        prev_source_partition_locator: Optional[PartitionLocator] = None,
     ) -> RoundCompletionInfo:
 
         rci = RoundCompletionInfo()
@@ -63,10 +64,11 @@ class RoundCompletionInfo(dict):
         rci["compactorVersion"] = compactor_version
         rci["inputInflation"] = input_inflation
         rci["inputAverageRecordSizeBytes"] = input_average_record_size_bytes
+        rci["prevSourcePartitionLocator"] = prev_source_partition_locator
         return rci
 
     @property
-    def high_watermark(self) -> HighWatermark:
+    def high_watermark(self) -> Union[HighWatermark, int]:
         val: Dict[str, Any] = self.get("highWatermark")
         if (
             val is not None
@@ -100,7 +102,11 @@ class RoundCompletionInfo(dict):
 
     @property
     def rebase_source_partition_locator(self) -> Optional[PartitionLocator]:
-        return self.get("rebaseSourcePartitionLocator")
+        val = self.get("rebaseSourcePartitionLocator")
+        if val is not None and not isinstance(val, PartitionLocator):
+            val = PartitionLocator(val)
+            self["rebaseSourcePartitionLocator"] = val  # Cache the converted value
+        return val
 
     @property
     def manifest_entry_copied_by_reference_ratio(self) -> Optional[float]:
@@ -111,7 +117,7 @@ class RoundCompletionInfo(dict):
         return self["hashBucketCount"]
 
     @property
-    def hb_index_to_entry_range(self) -> Optional[Dict[int, Tuple[int, int]]]:
+    def hb_index_to_entry_range(self) -> Optional[Dict[str, Tuple[int, int]]]:
         """
         The start index is inclusive and end index is exclusive by default.
         """
@@ -129,6 +135,10 @@ class RoundCompletionInfo(dict):
     def input_average_record_size_bytes(self) -> Optional[float]:
         return self.get("inputAverageRecordSizeBytes")
 
-    @staticmethod
-    def get_audit_bucket_name_and_key(compaction_audit_url: str) -> Tuple[str, str]:
-        return compaction_audit_url.replace("s3://", "").split("/", 1)
+    @property
+    def prev_source_partition_locator(self) -> Optional[PartitionLocator]:
+        val = self.get("prevSourcePartitionLocator")
+        if val is not None and not isinstance(val, PartitionLocator):
+            val = PartitionLocator(val)
+            self["prevSourcePartitionLocator"] = val  # Cache the converted value
+        return val

deltacat/compute/compactor/model/table_object_store.py

@@ -4,7 +4,7 @@ from ray.types import ObjectRef
 
 from typing import Any, Union
 
-from abc import ABC, abstractmethod, abstractproperty
+from abc import ABC, abstractmethod
 from deltacat.io.ray_plasma_object_store import RayPlasmaObjectStore
 from deltacat.storage import (
     LocalTable,
@@ -15,7 +15,8 @@ LocalTableReference = Union[ObjectRef, LocalTable]
 
 
 class LocalTableStorageStrategy(ABC):
-    @abstractproperty
+    @property
+    @abstractmethod
     def object_store(cls) -> IObjectStore:
         pass
 

deltacat/compute/compactor/repartition_session.py

@@ -21,14 +21,13 @@ from deltacat.utils.placement import PlacementGroupConfig
 from typing import List, Optional, Dict, Any
 from deltacat.utils.ray_utils.runtime import live_node_resource_keys
 from deltacat.compute.compactor.utils import io
-from deltacat.compute.compactor.utils import round_completion_file as rcf
 from deltacat.compute.compactor.steps import repartition as repar
 from deltacat.compute.compactor.steps.repartition import RepartitionType
 from deltacat.storage import (
     Delta,
     DeltaLocator,
     PartitionLocator,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from deltacat.utils.metrics import MetricsConfig
 from deltacat.compute.compactor.utils.sort_key import validate_sort_keys
@@ -41,7 +40,6 @@ def repartition(
     source_partition_locator: PartitionLocator,
     destination_partition_locator: PartitionLocator,
     repartition_args: Any,
-    repartition_completion_file_s3_url: str,
     last_stream_position_to_compact: int,
     repartition_type: RepartitionType = RepartitionType.RANGE,
     sort_keys: List[SortKey] = None,
@@ -54,9 +52,8 @@ def repartition(
     pg_config: Optional[PlacementGroupConfig] = None,
     list_deltas_kwargs: Optional[Dict[str, Any]] = None,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
-    s3_client_kwargs: Optional[Dict[str, Any]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    deltacat_storage=metastore,
     **kwargs,
 ) -> Optional[str]:
 
@@ -132,13 +129,13 @@ def repartition(
         enable_profiler=enable_profiler,
         metrics_config=metrics_config,
         read_kwargs_provider=read_kwargs_provider,
-        s3_table_writer_kwargs=s3_table_writer_kwargs,
+        table_writer_kwargs=table_writer_kwargs,
         repartitioned_file_content_type=repartitioned_file_content_type,
         deltacat_storage=deltacat_storage,
     )
     logger.info(f"Getting {len(repar_tasks_pending)} task results...")
     repar_results: List[RepartitionResult] = ray.get(repar_tasks_pending)
-    repar_results: List[Delta] = [rp.range_deltas for rp in repar_results]
+    repar_results: List[List[Delta]] = [rp.range_deltas for rp in repar_results]
     transposed = list(itertools.zip_longest(*repar_results, fillvalue=None))
     ordered_deltas: List[Delta] = [
         i for sublist in transposed for i in sublist if i is not None
@@ -153,9 +150,6 @@ def repartition(
     compacted_delta = deltacat_storage.commit_delta(
         merged_delta, properties=kwargs.get("properties", {})
     )
-    deltacat_storage.commit_partition(partition)
-    logger.info(f"Committed final delta: {compacted_delta}")
-    logger.info(f"Job run completed successfully!")
     new_compacted_delta_locator = DeltaLocator.of(
         new_compacted_partition_locator,
         compacted_delta.stream_position,
@@ -173,14 +167,7 @@ def repartition(
         bit_width_of_sort_keys,
         None,
     )
-    if s3_client_kwargs is None:
-        s3_client_kwargs = {}
-
-    return rcf.write_round_completion_file(
-        None,
-        None,
-        None,
-        repartition_completion_info,
-        repartition_completion_file_s3_url,
-        **s3_client_kwargs,
-    )
+    partition.compaction_round_completion_info = repartition_completion_info
+    deltacat_storage.commit_partition(partition)
+    logger.info(f"Committed final delta: {compacted_delta}")
+    logger.info(f"Job run completed successfully!")

deltacat/compute/compactor/steps/dedupe.py

@@ -15,7 +15,8 @@ from deltacat.compute.compactor import (
     DeltaFileEnvelope,
     DeltaFileLocator,
 )
-from deltacat.storage.model.sort_key import SortKey, SortOrder
+from deltacat.storage.model.sort_key import SortKey
+from deltacat.storage import SortOrder
 from deltacat.compute.compactor.model.dedupe_result import DedupeResult
 from deltacat.compute.compactor.utils import system_columns as sc
 from deltacat.utils.ray_utils.runtime import (
@@ -155,15 +156,21 @@ def _timed_dedupe(
                 sort_keys.extend(
                     [
                         SortKey.of(
-                            sc._PARTITION_STREAM_POSITION_COLUMN_NAME,
+                            [sc._PARTITION_STREAM_POSITION_COLUMN_NAME],
                             SortOrder.ASCENDING,
                         ),
                         SortKey.of(
-                            sc._ORDERED_FILE_IDX_COLUMN_NAME, SortOrder.ASCENDING
+                            [sc._ORDERED_FILE_IDX_COLUMN_NAME],
+                            SortOrder.ASCENDING,
                         ),
                     ]
                 )
-                table = table.take(pc.sort_indices(table, sort_keys=sort_keys))
+                table = table.take(
+                    pc.sort_indices(
+                        table,
+                        sort_keys=[pa_key for key in sort_keys for pa_key in key.arrow],
+                    )
+                )
 
             # drop duplicates by primary key hash column
             logger.info(

deltacat/compute/compactor/steps/hash_bucket.py

@@ -21,7 +21,7 @@ from deltacat.compute.compactor.utils.primary_key_index import (
     group_hash_bucket_indices,
     group_record_indices_by_hash_bucket,
 )
-from deltacat.storage import interface as unimplemented_deltacat_storage
+from deltacat.storage import metastore
 from deltacat.types.media import StorageType
 from deltacat.utils.common import sha1_digest
 from deltacat.utils.ray_utils.runtime import (
@@ -90,7 +90,7 @@ def _group_file_records_by_pk_hash_bucket(
     sort_key_names: List[str],
     is_src_delta: np.bool_ = True,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> Tuple[Optional[DeltaFileEnvelopeGroups], int]:
@@ -139,7 +139,7 @@ def _read_delta_file_envelopes(
     primary_keys: List[str],
     sort_key_names: List[str],
     read_kwargs_provider: Optional[ReadKwargsProvider],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> Tuple[Optional[List[DeltaFileEnvelope]], int]:
@@ -190,7 +190,7 @@ def _timed_hash_bucket(
     enable_profiler: bool,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
     object_store: Optional[IObjectStore] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ):
@@ -201,7 +201,7 @@ def _timed_hash_bucket(
     with memray.Tracker(
         f"hash_bucket_{worker_id}_{task_id}.bin"
     ) if enable_profiler else nullcontext():
-        sort_key_names = [key.key_name for key in sort_keys]
+        sort_key_names = list(chain.from_iterable([key.key for key in sort_keys]))
         if not round_completion_info:
             is_src_delta = True
         else:
@@ -249,7 +249,7 @@ def hash_bucket(
     metrics_config: MetricsConfig,
     read_kwargs_provider: Optional[ReadKwargsProvider],
     object_store: Optional[IObjectStore],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> HashBucketResult:

deltacat/compute/compactor/steps/materialize.py

@@ -25,10 +25,11 @@ from deltacat.storage import (
     DeltaType,
     Partition,
     PartitionLocator,
-    Manifest,
     ManifestEntry,
+    ManifestEntryList,
 )
-from deltacat.storage import interface as unimplemented_deltacat_storage
+from deltacat.storage.model.manifest import Manifest
+
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.types.media import DELIMITED_TEXT_CONTENT_TYPES, ContentType
 from deltacat.types.tables import TABLE_CLASS_TO_SIZE_FUNC
@@ -45,6 +46,7 @@ from deltacat.utils.ray_utils.runtime import (
 )
 from deltacat.utils.metrics import emit_timer_metrics, MetricsConfig
 from deltacat.utils.resources import get_current_process_peak_memory_usage_in_bytes
+from deltacat.storage import metastore
 
 if importlib.util.find_spec("memray"):
     import memray
@@ -66,9 +68,9 @@ def materialize(
     metrics_config: MetricsConfig,
     schema: Optional[pa.Schema] = None,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     object_store: Optional[IObjectStore] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
 ):
     if deltacat_storage_kwargs is None:
@@ -77,12 +79,15 @@ def materialize(
     def _stage_delta_from_manifest_entry_reference_list(
         manifest_entry_list_reference: List[ManifestEntry],
         partition: Partition,
-        delta_type: DeltaType = DeltaType.UPSERT,
+        delta_type: DeltaType = DeltaType.APPEND,
     ) -> Delta:
         assert (
-            delta_type == DeltaType.UPSERT
-        ), "Stage delta with existing manifest entries only supports UPSERT delta type!"
-        manifest = Manifest.of(entries=manifest_entry_list_reference, uuid=str(uuid4()))
+            delta_type == DeltaType.APPEND
+        ), "Compaction should always produce APPEND deltas for consistent read operations!"
+        manifest = Manifest.of(
+            entries=ManifestEntryList.of(manifest_entry_list_reference),
+            uuid=str(uuid4()),
+        )
         delta = Delta.of(
             locator=DeltaLocator.of(partition.locator),
             delta_type=delta_type,
@@ -106,9 +111,10 @@ def materialize(
             deltacat_storage.stage_delta,
             compacted_table,
             partition,
+            delta_type=DeltaType.APPEND,  # Compaction always produces APPEND deltas
             max_records_per_entry=max_records_per_output_file,
             content_type=compacted_file_content_type,
-            s3_table_writer_kwargs=s3_table_writer_kwargs,
+            table_writer_kwargs=table_writer_kwargs,
             **deltacat_storage_kwargs,
         )
         compacted_table_size = TABLE_CLASS_TO_SIZE_FUNC[type(compacted_table)](

deltacat/compute/compactor/steps/repartition.py

@@ -10,7 +10,7 @@ import ray
 from deltacat import logs
 from deltacat.compute.compactor import DeltaAnnotated
 from deltacat.compute.compactor.model.repartition_result import RepartitionResult
-from deltacat.storage import interface as unimplemented_deltacat_storage
+from deltacat.storage import metastore
 from deltacat.storage import Partition
 from deltacat.utils.ray_utils.runtime import (
     get_current_ray_task_id,
@@ -19,7 +19,7 @@ from deltacat.utils.ray_utils.runtime import (
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.utils.performance import timed_invocation
 from deltacat.utils.metrics import emit_timer_metrics, MetricsConfig
-from deltacat.storage import Delta
+from deltacat.storage import Delta, DeltaType
 from enum import Enum
 
 if importlib.util.find_spec("memray"):
@@ -56,9 +56,9 @@ def repartition_range(
     destination_partition: Partition,
     repartition_args: dict,
     max_records_per_output_file: int,
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     repartitioned_file_content_type: ContentType = ContentType.PARQUET,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ):
@@ -144,9 +144,10 @@ def repartition_range(
                 partition_delta: Delta = deltacat_storage.stage_delta(
                     partition_table,
                     destination_partition,
+                    delta_type=DeltaType.APPEND,  # Repartition always produces APPEND deltas
                     max_records_per_entry=max_records_per_output_file,
                     content_type=repartitioned_file_content_type,
-                    s3_table_writer_kwargs=s3_table_writer_kwargs,
+                    table_writer_kwargs=table_writer_kwargs,
                     **deltacat_storage_kwargs,
                 )
                 partition_deltas.append(partition_delta)
@@ -168,9 +169,9 @@ def _timed_repartition(
     max_records_per_output_file: int,
     enable_profiler: bool,
     read_kwargs_provider: Optional[ReadKwargsProvider],
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     repartitioned_file_content_type: ContentType = ContentType.PARQUET,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> RepartitionResult:
@@ -192,7 +193,7 @@ def _timed_repartition(
                 destination_partition=destination_partition,
                 repartition_args=repartition_args,
                 max_records_per_output_file=max_records_per_output_file,
-                s3_table_writer_kwargs=s3_table_writer_kwargs,
+                table_writer_kwargs=table_writer_kwargs,
                 repartitioned_file_content_type=repartitioned_file_content_type,
                 deltacat_storage=deltacat_storage,
                 deltacat_storage_kwargs=deltacat_storage_kwargs,
@@ -213,9 +214,9 @@ def repartition(
     enable_profiler: bool,
     metrics_config: Optional[MetricsConfig],
     read_kwargs_provider: Optional[ReadKwargsProvider],
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     repartitioned_file_content_type: ContentType = ContentType.PARQUET,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> RepartitionResult:
@@ -231,7 +232,7 @@ def repartition(
         max_records_per_output_file=max_records_per_output_file,
         enable_profiler=enable_profiler,
         read_kwargs_provider=read_kwargs_provider,
-        s3_table_writer_kwargs=s3_table_writer_kwargs,
+        table_writer_kwargs=table_writer_kwargs,
         repartitioned_file_content_type=repartitioned_file_content_type,
         deltacat_storage=deltacat_storage,
         deltacat_storage_kwargs=deltacat_storage_kwargs,

deltacat/compute/compactor/utils/io.py

@@ -11,7 +11,7 @@ from deltacat.storage import (
     PartitionLocator,
     Delta,
     ManifestEntry,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from deltacat import logs
 from deltacat.compute.compactor import DeltaAnnotated
@@ -31,12 +31,13 @@ def discover_deltas(
     compacted_partition_locator: Optional[PartitionLocator],
     rebase_source_partition_locator: Optional[PartitionLocator],
     rebase_source_partition_high_watermark: Optional[int],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
     list_deltas_kwargs: Optional[Dict[str, Any]] = {},
 ) -> Tuple[List[Delta], int]:
     if deltacat_storage_kwargs is None:
         deltacat_storage_kwargs = {}
+
     # Source One: new deltas from uncompacted table for incremental compaction or deltas from compacted table for rebase
     start_position_exclusive = (
         high_watermark.get(source_partition_locator)
@@ -109,7 +110,7 @@ def limit_input_deltas(
     user_hash_bucket_chunk_size: int,
     input_deltas_stats: Dict[int, DeltaStats],
     compaction_audit: CompactionSessionAuditInfo,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> Tuple[List[DeltaAnnotated], int, HighWatermark, bool]:
@@ -272,7 +273,7 @@ def fit_input_deltas(
     cluster_resources: Dict[str, float],
     compaction_audit: CompactionSessionAuditInfo,
     hash_bucket_count: Optional[int],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> Tuple[List[DeltaAnnotated], int, HighWatermark, bool]:
@@ -358,8 +359,8 @@ def fit_input_deltas(
 def _discover_deltas(
     source_partition_locator: PartitionLocator,
     start_position_exclusive: Optional[int],
-    end_position_inclusive: int,
-    deltacat_storage=unimplemented_deltacat_storage,
+    end_position_inclusive: Optional[int],
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
     list_deltas_kwargs: Optional[Dict[str, Any]] = {},
 ) -> List[Delta]:

deltacat/compute/compactor/utils/round_completion_reader.py

@@ -0,0 +1,117 @@
+import logging
+from typing import Optional
+from deltacat import logs
+from deltacat.compute.compactor import RoundCompletionInfo
+from deltacat.storage import PartitionLocator
+from deltacat.storage.model.partition import Partition
+from deltacat.utils.metrics import metrics
+from deltacat.exceptions import PartitionNotFoundError
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+@metrics
+def read_round_completion_info(
+    source_partition_locator: PartitionLocator,
+    destination_partition_locator: PartitionLocator,
+    deltacat_storage,
+    deltacat_storage_kwargs: Optional[dict] = None,
+    destination_partition: Optional[Partition] = None,
+) -> Optional[RoundCompletionInfo]:
+    """
+    Read round completion info from the partition metafile.
+
+    Args:
+        source_partition_locator: Source partition locator for validation
+        destination_partition_locator: Destination partition locator
+        deltacat_storage: Storage implementation
+        deltacat_storage_kwargs: Optional storage kwargs
+        destination_partition: Optional destination partition to avoid redundant get_partition calls
+
+    Returns:
+        RoundCompletionInfo if found in partition, None otherwise
+    """
+    if not destination_partition_locator:
+        return None
+
+    if deltacat_storage_kwargs is None:
+        deltacat_storage_kwargs = {}
+
+    try:
+        # Use provided partition or get it from storage
+        if destination_partition:
+            partition = destination_partition
+        else:
+            # First get the current partition to access its previous_partition_id
+            current_partition: Partition = deltacat_storage.get_partition(
+                destination_partition_locator.stream_locator,
+                destination_partition_locator.partition_values,
+                **deltacat_storage_kwargs,
+            )
+
+            # If current partition has round completion info, use it
+            if current_partition.compaction_round_completion_info:
+                partition = current_partition
+            elif current_partition.previous_partition_id is not None:
+                # For incremental compaction, we need to get the previous committed partition
+                # that contains the round completion info.
+                # Get the previous partition by ID - this is where the round completion info should be
+                logger.info(
+                    f"Current partition {destination_partition_locator} does not have round completion info, "
+                    f"getting previous partition with ID: {current_partition.previous_partition_id}"
+                )
+                previous_partition = deltacat_storage.get_partition_by_id(
+                    destination_partition_locator.stream_locator,
+                    current_partition.previous_partition_id,
+                    **deltacat_storage_kwargs,
+                )
+                if previous_partition is not None:
+                    logger.info(
+                        f"Found previous partition: {previous_partition.locator}"
+                    )
+                    partition = previous_partition
+                else:
+                    raise PartitionNotFoundError(
+                        f"Previous partition with ID {current_partition.previous_partition_id} not found"
+                    )
+            else:
+                logger.info(f"No previous partition ID found, using current partition")
+                partition = current_partition
+
+        if partition:
+            round_completion_info = partition.compaction_round_completion_info
+            if round_completion_info:
+                # Validate that prev_source_partition_locator matches current source
+                if (
+                    not source_partition_locator
+                    or not round_completion_info.prev_source_partition_locator
+                ):
+                    raise ValueError(
+                        f"Source partition locator ({source_partition_locator}) and "
+                        f"prev_source_partition_locator ({round_completion_info.prev_source_partition_locator}) "
+                        f"must both be provided."
+                    )
+
+                if (
+                    round_completion_info.prev_source_partition_locator.canonical_string()
+                    != source_partition_locator.canonical_string()
+                ):
+                    logger.warning(
+                        f"Previous source partition locator mismatch: "
+                        f"expected {source_partition_locator.canonical_string()}, "
+                        f"but found {round_completion_info.prev_source_partition_locator.canonical_string()} "
+                        f"in round completion info. Ignoring cached round completion info."
+                    )
+                    return None
+
+                logger.info(
+                    f"Read round completion info from partition metafile: {round_completion_info}"
+                )
+                return round_completion_info
+
+    except Exception as e:
+        logger.debug(
+            f"Failed to read round completion info from partition metafile: {e}"
+        )
+
+    return None

deltacat/compute/compactor/utils/sort_key.py

@@ -1,6 +1,7 @@
 import pyarrow as pa
 from typing import List
-from deltacat.storage import PartitionLocator, SortKey
+from itertools import chain
+from deltacat.storage import PartitionLocator, SortKey, TransformName
 
 MAX_SORT_KEYS_BIT_WIDTH = 256
 
@@ -22,7 +23,13 @@ def validate_sort_keys(
         deltacat_storage_kwargs = {}
     total_sort_keys_bit_width = 0
     if sort_keys:
-        sort_key_names = [key.key_name for key in sort_keys]
+        sort_key_names = list(chain.from_iterable([key.key for key in sort_keys]))
+        assert all(
+            [
+                key.transform is None or key.transform.name == TransformName.IDENTITY
+                for key in sort_keys
+            ]
+        ), f"Sort key transforms are not supported: {sort_keys}"
         assert len(sort_key_names) == len(
             set(sort_key_names)
         ), f"Sort key names must be unique: {sort_key_names}"

deltacat/compute/compactor/utils/system_columns.py

@@ -294,7 +294,9 @@ def append_dedupe_task_idx_col(table: pa.Table, dedupe_task_indices) -> pa.Table
 
 
 def delta_type_to_field(delta_type: DeltaType) -> bool:
-    return True if delta_type is DeltaType.UPSERT else False
+    # For deduplication purposes, treat both UPSERT and APPEND as UPSERT (True)
+    # Only DELETE should be treated as DELETE (False)
+    return delta_type is not DeltaType.DELETE
 
 
 def delta_type_from_field(delta_type_field: bool) -> DeltaType: