deltacat 2.0.0b11__py3-none-any.whl → 2.0.0.post1__py3-none-any.whl

deltacat/compute/compactor/model/compact_partition_params.py

@@ -2,17 +2,19 @@ from __future__ import annotations
 import importlib
 import copy
 import json
-from typing import Any, Dict, List, Optional
+import posixpath
+from typing import Any, Dict, List, Optional, Set
 from deltacat.io.object_store import IObjectStore
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.types.media import ContentType
 from deltacat.utils.placement import PlacementGroupConfig
 from deltacat.io.ray_plasma_object_store import RayPlasmaObjectStore
 from deltacat.storage import (
-    interface as unimplemented_deltacat_storage,
+    metastore,
     PartitionLocator,
     SortKey,
 )
+from deltacat.catalog.model.properties import CatalogProperties
 from deltacat.compute.resource_estimation import (
     ResourceEstimationMethod,
     EstimateResourcesParams,
@@ -52,11 +54,22 @@ class CompactPartitionParams(dict):
         assert (
             params.get("source_partition_locator") is not None
         ), "source_partition_locator is a required arg"
+        assert params.get("catalog") is not None, "catalog is a required arg"
         assert (
-            params.get("compaction_artifact_s3_bucket") is not None
-        ), "compaction_artifact_s3_bucket is a required arg"
+            params.get("all_column_names") is not None
+        ), "all_column_names is a required arg"
 
         result = CompactPartitionParams(params)
+        assert (
+            result.destination_partition_locator.partition_id
+        ), "destination_partition_locator must have a globally unique partition_id"
+        assert (
+            result.source_partition_locator.partition_id
+        ), "source_partition_locator must have a globally unique partition_id"
+        if result.rebase_source_partition_locator:
+            assert (
+                result.rebase_source_partition_locator.partition_id
+            ), "rebase_source_partition_locator must have a globally unique partition_id"
 
         result.records_per_compacted_file = params.get(
             "records_per_compacted_file", MAX_RECORDS_PER_COMPACTED_FILE
@@ -65,15 +78,18 @@ class CompactPartitionParams(dict):
             "compacted_file_content_type", ContentType.PARQUET
         )
         result.object_store = params.get("object_store", RayPlasmaObjectStore())
+        result.table_writer_kwargs = params.get("table_writer_kwargs", {})
 
         result.enable_profiler = params.get("enable_profiler", False)
-        result.deltacat_storage = params.get(
-            "deltacat_storage", unimplemented_deltacat_storage
-        )
-        result.s3_client_kwargs = params.get("s3_client_kwargs", {})
+        result.deltacat_storage = params.get("deltacat_storage", metastore)
+        result.catalog = params.get("catalog")
         result.deltacat_storage_kwargs = params.get("deltacat_storage_kwargs", {})
         result.list_deltas_kwargs = params.get("list_deltas_kwargs", {})
-        result.s3_table_writer_kwargs = params.get("s3_table_writer_kwargs", {})
+        result.all_column_names = params.get("all_column_names")
+
+        # Add catalog to deltacat_storage_kwargs
+        result.deltacat_storage_kwargs["catalog"] = result.catalog
+
         result.bit_width_of_sort_keys = validate_sort_keys(
             result.source_partition_locator,
             result.sort_keys,
@@ -133,6 +149,8 @@ class CompactPartitionParams(dict):
         if result.primary_keys:
             result.primary_keys = sorted(result.primary_keys)
 
+        result.original_fields = params.get("original_fields")
+
         # assertions
         assert (
             result.source_partition_locator.partition_values
@@ -177,21 +195,32 @@ class CompactPartitionParams(dict):
         self["source_partition_locator"] = locator
 
     @property
-    def compaction_artifact_s3_bucket(self) -> str:
-        return self["compaction_artifact_s3_bucket"]
-
-    @compaction_artifact_s3_bucket.setter
-    def compaction_artifact_s3_bucket(self, s3_bucket: str) -> None:
-        self["compaction_artifact_s3_bucket"] = s3_bucket
+    def compaction_artifact_path(self) -> str:
+        """
+        Returns the compaction artifact path based on catalog root.
+        """
+        return posixpath.join(self.catalog.root, "compute", "compactor")
 
     @property
-    def deltacat_storage(self) -> unimplemented_deltacat_storage:
+    def deltacat_storage(self) -> metastore:
         return self["deltacat_storage"]
 
     @deltacat_storage.setter
-    def deltacat_storage(self, storage: unimplemented_deltacat_storage) -> None:
+    def deltacat_storage(self, storage: metastore) -> None:
         self["deltacat_storage"] = storage
 
+    @property
+    def catalog(self) -> CatalogProperties:
+        return self["catalog"]
+
+    @catalog.setter
+    def catalog(self, catalog: CatalogProperties) -> None:
+        self["catalog"] = catalog
+        # Update deltacat_storage_kwargs when catalog is set
+        if "deltacat_storage_kwargs" not in self:
+            self["deltacat_storage_kwargs"] = {}
+        self["deltacat_storage_kwargs"]["catalog"] = catalog
+
     @property
     def object_store(self) -> IObjectStore:
         return self["object_store"]
@@ -286,14 +315,6 @@ class CompactPartitionParams(dict):
     def list_deltas_kwargs(self, kwargs: dict) -> None:
         self["list_deltas_kwargs"] = kwargs
 
-    @property
-    def s3_table_writer_kwargs(self) -> dict:
-        return self["s3_table_writer_kwargs"]
-
-    @s3_table_writer_kwargs.setter
-    def s3_table_writer_kwargs(self, kwargs: dict) -> None:
-        self["s3_table_writer_kwargs"] = kwargs
-
     @property
     def deltacat_storage_kwargs(self) -> dict:
         return self["deltacat_storage_kwargs"]
@@ -303,12 +324,12 @@ class CompactPartitionParams(dict):
         self["deltacat_storage_kwargs"] = kwargs
 
     @property
-    def s3_client_kwargs(self) -> dict:
-        return self["s3_client_kwargs"]
+    def all_column_names(self) -> List[str]:
+        return self.get("all_column_names")
 
-    @s3_client_kwargs.setter
-    def s3_client_kwargs(self, kwargs: dict) -> None:
-        self["s3_client_kwargs"] = kwargs
+    @all_column_names.setter
+    def all_column_names(self, column_names: List[str]) -> None:
+        self["all_column_names"] = column_names
 
     @property
     def records_per_compacted_file(self) -> int:
@@ -489,6 +510,30 @@ class CompactPartitionParams(dict):
             average_record_size_bytes=self.average_record_size_bytes,
         )
 
+    @property
+    def table_writer_kwargs(self) -> dict:
+        return self["table_writer_kwargs"]
+
+    @table_writer_kwargs.setter
+    def table_writer_kwargs(self, kwargs: dict) -> None:
+        self["table_writer_kwargs"] = kwargs
+
+    @property
+    def expected_previous_partition_id(self) -> Optional[str]:
+        return self.get("expected_previous_partition_id")
+
+    @expected_previous_partition_id.setter
+    def expected_previous_partition_id(self, partition_id: Optional[str]) -> None:
+        self["expected_previous_partition_id"] = partition_id
+
+    @property
+    def original_fields(self) -> Optional[Set[str]]:
+        return self.get("original_fields")
+
+    @original_fields.setter
+    def original_fields(self, fields: Optional[Set[str]]) -> None:
+        self["original_fields"] = fields
+
     @staticmethod
     def json_handler_for_compact_partition_params(obj):
         """

deltacat/compute/compactor/model/compaction_session_audit_info.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from typing import Optional
 import pyarrow as pa
 import logging
+from pathlib import PosixPath
 from deltacat import logs
 from typing import List, Union
 from deltacat.compute.compactor.model.hash_bucket_result import HashBucketResult
@@ -919,3 +920,19 @@ class CompactionSessionAuditInfo(dict):
         )
 
         self.set_pyarrow_version(pa.__version__)
+
+    def to_serializable(self, catalog_root: str) -> CompactionSessionAuditInfo:
+        root_path = PosixPath(catalog_root)
+        target_path = PosixPath(self.audit_url)
+        if root_path == target_path:
+            raise ValueError(
+                "Target and root are identical, but expected target to be a child of root."
+            )
+        try:
+            relative_path = target_path.relative_to(root_path)
+            # Create a copy of the audit info with the relative path
+            audit_copy = CompactionSessionAuditInfo(**dict(self))
+            audit_copy["auditUrl"] = str(relative_path)
+            return audit_copy
+        except ValueError:
+            raise ValueError("Expected target to be a child of root.")

deltacat/compute/compactor/model/round_completion_info.py

@@ -1,7 +1,7 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-from typing import List, Tuple, Union
+from typing import Tuple, Union
 from deltacat.storage import DeltaLocator, PartitionLocator
 from deltacat.compute.compactor.model.pyarrow_write_result import PyArrowWriteResult
 from typing import Any, Dict, Optional
@@ -10,7 +10,7 @@ from typing import Any, Dict, Optional
 class HighWatermark(dict):
     """
     Inherit from dict to make it easy for serialization/deserialization.
-    Keep both partition locator and high watermark as a tuple to be persisted in the rcf
+    Keep both partition locator and high watermark as a tuple to be persisted in the rci
     """
 
     def set(self, partition_locator: PartitionLocator, delta_stream_position: int):
@@ -46,6 +46,7 @@ class RoundCompletionInfo(dict):
         compactor_version: Optional[str] = None,
         input_inflation: Optional[float] = None,
         input_average_record_size_bytes: Optional[float] = None,
+        prev_source_partition_locator: Optional[PartitionLocator] = None,
     ) -> RoundCompletionInfo:
 
         rci = RoundCompletionInfo()
@@ -63,6 +64,7 @@ class RoundCompletionInfo(dict):
         rci["compactorVersion"] = compactor_version
         rci["inputInflation"] = input_inflation
         rci["inputAverageRecordSizeBytes"] = input_average_record_size_bytes
+        rci["prevSourcePartitionLocator"] = prev_source_partition_locator
         return rci
 
     @property
@@ -100,7 +102,11 @@ class RoundCompletionInfo(dict):
 
     @property
     def rebase_source_partition_locator(self) -> Optional[PartitionLocator]:
-        return self.get("rebaseSourcePartitionLocator")
+        val = self.get("rebaseSourcePartitionLocator")
+        if val is not None and not isinstance(val, PartitionLocator):
+            val = PartitionLocator(val)
+            self["rebaseSourcePartitionLocator"] = val  # Cache the converted value
+        return val
 
     @property
     def manifest_entry_copied_by_reference_ratio(self) -> Optional[float]:
@@ -129,6 +135,10 @@ class RoundCompletionInfo(dict):
     def input_average_record_size_bytes(self) -> Optional[float]:
         return self.get("inputAverageRecordSizeBytes")
 
-    @staticmethod
-    def get_audit_bucket_name_and_key(compaction_audit_url: str) -> List[str]:
-        return compaction_audit_url.replace("s3://", "").split("/", 1)
+    @property
+    def prev_source_partition_locator(self) -> Optional[PartitionLocator]:
+        val = self.get("prevSourcePartitionLocator")
+        if val is not None and not isinstance(val, PartitionLocator):
+            val = PartitionLocator(val)
+            self["prevSourcePartitionLocator"] = val  # Cache the converted value
+        return val

deltacat/compute/compactor/repartition_session.py

@@ -21,14 +21,13 @@ from deltacat.utils.placement import PlacementGroupConfig
 from typing import List, Optional, Dict, Any
 from deltacat.utils.ray_utils.runtime import live_node_resource_keys
 from deltacat.compute.compactor.utils import io
-from deltacat.compute.compactor.utils import round_completion_file as rcf
 from deltacat.compute.compactor.steps import repartition as repar
 from deltacat.compute.compactor.steps.repartition import RepartitionType
 from deltacat.storage import (
     Delta,
     DeltaLocator,
     PartitionLocator,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from deltacat.utils.metrics import MetricsConfig
 from deltacat.compute.compactor.utils.sort_key import validate_sort_keys
@@ -41,7 +40,6 @@ def repartition(
     source_partition_locator: PartitionLocator,
     destination_partition_locator: PartitionLocator,
     repartition_args: Any,
-    repartition_completion_file_s3_url: str,
     last_stream_position_to_compact: int,
     repartition_type: RepartitionType = RepartitionType.RANGE,
     sort_keys: List[SortKey] = None,
@@ -54,9 +52,8 @@ def repartition(
     pg_config: Optional[PlacementGroupConfig] = None,
     list_deltas_kwargs: Optional[Dict[str, Any]] = None,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
-    s3_client_kwargs: Optional[Dict[str, Any]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    deltacat_storage=metastore,
     **kwargs,
 ) -> Optional[str]:
 
@@ -132,7 +129,7 @@ def repartition(
         enable_profiler=enable_profiler,
         metrics_config=metrics_config,
         read_kwargs_provider=read_kwargs_provider,
-        s3_table_writer_kwargs=s3_table_writer_kwargs,
+        table_writer_kwargs=table_writer_kwargs,
         repartitioned_file_content_type=repartitioned_file_content_type,
         deltacat_storage=deltacat_storage,
     )
@@ -153,9 +150,6 @@ def repartition(
     compacted_delta = deltacat_storage.commit_delta(
         merged_delta, properties=kwargs.get("properties", {})
     )
-    deltacat_storage.commit_partition(partition)
-    logger.info(f"Committed final delta: {compacted_delta}")
-    logger.info(f"Job run completed successfully!")
     new_compacted_delta_locator = DeltaLocator.of(
         new_compacted_partition_locator,
         compacted_delta.stream_position,
@@ -173,14 +167,7 @@ def repartition(
         bit_width_of_sort_keys,
         None,
     )
-    if s3_client_kwargs is None:
-        s3_client_kwargs = {}
-
-    return rcf.write_round_completion_file(
-        None,
-        None,
-        None,
-        repartition_completion_info,
-        repartition_completion_file_s3_url,
-        **s3_client_kwargs,
-    )
+    partition.compaction_round_completion_info = repartition_completion_info
+    deltacat_storage.commit_partition(partition)
+    logger.info(f"Committed final delta: {compacted_delta}")
+    logger.info(f"Job run completed successfully!")

deltacat/compute/compactor/steps/hash_bucket.py

@@ -21,7 +21,7 @@ from deltacat.compute.compactor.utils.primary_key_index import (
     group_hash_bucket_indices,
     group_record_indices_by_hash_bucket,
 )
-from deltacat.storage import interface as unimplemented_deltacat_storage
+from deltacat.storage import metastore
 from deltacat.types.media import StorageType
 from deltacat.utils.common import sha1_digest
 from deltacat.utils.ray_utils.runtime import (
@@ -90,7 +90,7 @@ def _group_file_records_by_pk_hash_bucket(
     sort_key_names: List[str],
     is_src_delta: np.bool_ = True,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> Tuple[Optional[DeltaFileEnvelopeGroups], int]:
@@ -139,7 +139,7 @@ def _read_delta_file_envelopes(
     primary_keys: List[str],
     sort_key_names: List[str],
     read_kwargs_provider: Optional[ReadKwargsProvider],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> Tuple[Optional[List[DeltaFileEnvelope]], int]:
@@ -190,7 +190,7 @@ def _timed_hash_bucket(
     enable_profiler: bool,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
     object_store: Optional[IObjectStore] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ):
@@ -249,7 +249,7 @@ def hash_bucket(
     metrics_config: MetricsConfig,
     read_kwargs_provider: Optional[ReadKwargsProvider],
     object_store: Optional[IObjectStore],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> HashBucketResult:

deltacat/compute/compactor/steps/materialize.py

@@ -29,7 +29,7 @@ from deltacat.storage import (
     ManifestEntryList,
 )
 from deltacat.storage.model.manifest import Manifest
-from deltacat.storage import interface as unimplemented_deltacat_storage
+
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.types.media import DELIMITED_TEXT_CONTENT_TYPES, ContentType
 from deltacat.types.tables import TABLE_CLASS_TO_SIZE_FUNC
@@ -46,6 +46,7 @@ from deltacat.utils.ray_utils.runtime import (
 )
 from deltacat.utils.metrics import emit_timer_metrics, MetricsConfig
 from deltacat.utils.resources import get_current_process_peak_memory_usage_in_bytes
+from deltacat.storage import metastore
 
 if importlib.util.find_spec("memray"):
     import memray
@@ -67,9 +68,9 @@ def materialize(
     metrics_config: MetricsConfig,
     schema: Optional[pa.Schema] = None,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     object_store: Optional[IObjectStore] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
 ):
     if deltacat_storage_kwargs is None:
@@ -78,11 +79,11 @@ def materialize(
     def _stage_delta_from_manifest_entry_reference_list(
         manifest_entry_list_reference: List[ManifestEntry],
         partition: Partition,
-        delta_type: DeltaType = DeltaType.UPSERT,
+        delta_type: DeltaType = DeltaType.APPEND,
     ) -> Delta:
         assert (
-            delta_type == DeltaType.UPSERT
-        ), "Stage delta with existing manifest entries only supports UPSERT delta type!"
+            delta_type == DeltaType.APPEND
+        ), "Compaction should always produce APPEND deltas for consistent read operations!"
         manifest = Manifest.of(
             entries=ManifestEntryList.of(manifest_entry_list_reference),
             uuid=str(uuid4()),
@@ -110,9 +111,10 @@ def materialize(
             deltacat_storage.stage_delta,
             compacted_table,
             partition,
+            delta_type=DeltaType.APPEND,  # Compaction always produces APPEND deltas
             max_records_per_entry=max_records_per_output_file,
             content_type=compacted_file_content_type,
-            s3_table_writer_kwargs=s3_table_writer_kwargs,
+            table_writer_kwargs=table_writer_kwargs,
             **deltacat_storage_kwargs,
         )
         compacted_table_size = TABLE_CLASS_TO_SIZE_FUNC[type(compacted_table)](

deltacat/compute/compactor/steps/repartition.py

@@ -10,7 +10,7 @@ import ray
 from deltacat import logs
 from deltacat.compute.compactor import DeltaAnnotated
 from deltacat.compute.compactor.model.repartition_result import RepartitionResult
-from deltacat.storage import interface as unimplemented_deltacat_storage
+from deltacat.storage import metastore
 from deltacat.storage import Partition
 from deltacat.utils.ray_utils.runtime import (
     get_current_ray_task_id,
@@ -19,7 +19,7 @@ from deltacat.utils.ray_utils.runtime import (
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.utils.performance import timed_invocation
 from deltacat.utils.metrics import emit_timer_metrics, MetricsConfig
-from deltacat.storage import Delta
+from deltacat.storage import Delta, DeltaType
 from enum import Enum
 
 if importlib.util.find_spec("memray"):
@@ -56,9 +56,9 @@ def repartition_range(
     destination_partition: Partition,
     repartition_args: dict,
     max_records_per_output_file: int,
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     repartitioned_file_content_type: ContentType = ContentType.PARQUET,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ):
@@ -144,9 +144,10 @@ def repartition_range(
                 partition_delta: Delta = deltacat_storage.stage_delta(
                     partition_table,
                     destination_partition,
+                    delta_type=DeltaType.APPEND,  # Repartition always produces APPEND deltas
                     max_records_per_entry=max_records_per_output_file,
                     content_type=repartitioned_file_content_type,
-                    s3_table_writer_kwargs=s3_table_writer_kwargs,
+                    table_writer_kwargs=table_writer_kwargs,
                     **deltacat_storage_kwargs,
                 )
                 partition_deltas.append(partition_delta)
@@ -168,9 +169,9 @@ def _timed_repartition(
     max_records_per_output_file: int,
     enable_profiler: bool,
     read_kwargs_provider: Optional[ReadKwargsProvider],
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     repartitioned_file_content_type: ContentType = ContentType.PARQUET,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> RepartitionResult:
@@ -192,7 +193,7 @@ def _timed_repartition(
                 destination_partition=destination_partition,
                 repartition_args=repartition_args,
                 max_records_per_output_file=max_records_per_output_file,
-                s3_table_writer_kwargs=s3_table_writer_kwargs,
+                table_writer_kwargs=table_writer_kwargs,
                 repartitioned_file_content_type=repartitioned_file_content_type,
                 deltacat_storage=deltacat_storage,
                 deltacat_storage_kwargs=deltacat_storage_kwargs,
@@ -213,9 +214,9 @@ def repartition(
     enable_profiler: bool,
     metrics_config: Optional[MetricsConfig],
     read_kwargs_provider: Optional[ReadKwargsProvider],
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     repartitioned_file_content_type: ContentType = ContentType.PARQUET,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> RepartitionResult:
@@ -231,7 +232,7 @@ def repartition(
         max_records_per_output_file=max_records_per_output_file,
         enable_profiler=enable_profiler,
         read_kwargs_provider=read_kwargs_provider,
-        s3_table_writer_kwargs=s3_table_writer_kwargs,
+        table_writer_kwargs=table_writer_kwargs,
         repartitioned_file_content_type=repartitioned_file_content_type,
         deltacat_storage=deltacat_storage,
         deltacat_storage_kwargs=deltacat_storage_kwargs,

deltacat/compute/compactor/utils/io.py

@@ -11,7 +11,7 @@ from deltacat.storage import (
     PartitionLocator,
     Delta,
     ManifestEntry,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from deltacat import logs
 from deltacat.compute.compactor import DeltaAnnotated
@@ -31,12 +31,13 @@ def discover_deltas(
     compacted_partition_locator: Optional[PartitionLocator],
     rebase_source_partition_locator: Optional[PartitionLocator],
     rebase_source_partition_high_watermark: Optional[int],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
     list_deltas_kwargs: Optional[Dict[str, Any]] = {},
 ) -> Tuple[List[Delta], int]:
     if deltacat_storage_kwargs is None:
         deltacat_storage_kwargs = {}
+
     # Source One: new deltas from uncompacted table for incremental compaction or deltas from compacted table for rebase
     start_position_exclusive = (
         high_watermark.get(source_partition_locator)
@@ -109,7 +110,7 @@ def limit_input_deltas(
     user_hash_bucket_chunk_size: int,
     input_deltas_stats: Dict[int, DeltaStats],
     compaction_audit: CompactionSessionAuditInfo,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> Tuple[List[DeltaAnnotated], int, HighWatermark, bool]:
@@ -272,7 +273,7 @@ def fit_input_deltas(
     cluster_resources: Dict[str, float],
     compaction_audit: CompactionSessionAuditInfo,
     hash_bucket_count: Optional[int],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> Tuple[List[DeltaAnnotated], int, HighWatermark, bool]:
@@ -359,7 +360,7 @@ def _discover_deltas(
     source_partition_locator: PartitionLocator,
     start_position_exclusive: Optional[int],
     end_position_inclusive: Optional[int],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
     list_deltas_kwargs: Optional[Dict[str, Any]] = {},
 ) -> List[Delta]: