deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/compute/compactor/repartition_session.py

@@ -21,14 +21,13 @@ from deltacat.utils.placement import PlacementGroupConfig
 from typing import List, Optional, Dict, Any
 from deltacat.utils.ray_utils.runtime import live_node_resource_keys
 from deltacat.compute.compactor.utils import io
-from deltacat.compute.compactor.utils import round_completion_file as rcf
 from deltacat.compute.compactor.steps import repartition as repar
 from deltacat.compute.compactor.steps.repartition import RepartitionType
 from deltacat.storage import (
     Delta,
     DeltaLocator,
     PartitionLocator,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from deltacat.utils.metrics import MetricsConfig
 from deltacat.compute.compactor.utils.sort_key import validate_sort_keys
@@ -41,7 +40,6 @@ def repartition(
     source_partition_locator: PartitionLocator,
     destination_partition_locator: PartitionLocator,
     repartition_args: Any,
-    repartition_completion_file_s3_url: str,
     last_stream_position_to_compact: int,
     repartition_type: RepartitionType = RepartitionType.RANGE,
     sort_keys: List[SortKey] = None,
@@ -54,9 +52,8 @@ def repartition(
     pg_config: Optional[PlacementGroupConfig] = None,
     list_deltas_kwargs: Optional[Dict[str, Any]] = None,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
-    s3_client_kwargs: Optional[Dict[str, Any]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    deltacat_storage=metastore,
     **kwargs,
 ) -> Optional[str]:
 
@@ -132,7 +129,7 @@ def repartition(
         enable_profiler=enable_profiler,
         metrics_config=metrics_config,
         read_kwargs_provider=read_kwargs_provider,
-        s3_table_writer_kwargs=s3_table_writer_kwargs,
+        table_writer_kwargs=table_writer_kwargs,
         repartitioned_file_content_type=repartitioned_file_content_type,
         deltacat_storage=deltacat_storage,
     )
@@ -153,9 +150,6 @@ def repartition(
     compacted_delta = deltacat_storage.commit_delta(
         merged_delta, properties=kwargs.get("properties", {})
     )
-    deltacat_storage.commit_partition(partition)
-    logger.info(f"Committed final delta: {compacted_delta}")
-    logger.info(f"Job run completed successfully!")
     new_compacted_delta_locator = DeltaLocator.of(
         new_compacted_partition_locator,
         compacted_delta.stream_position,
@@ -173,14 +167,7 @@ def repartition(
         bit_width_of_sort_keys,
         None,
     )
-    if s3_client_kwargs is None:
-        s3_client_kwargs = {}
-
-    return rcf.write_round_completion_file(
-        None,
-        None,
-        None,
-        repartition_completion_info,
-        repartition_completion_file_s3_url,
-        **s3_client_kwargs,
-    )
+    partition.compaction_round_completion_info = repartition_completion_info
+    deltacat_storage.commit_partition(partition)
+    logger.info(f"Committed final delta: {compacted_delta}")
+    logger.info(f"Job run completed successfully!")

deltacat/compute/compactor/steps/hash_bucket.py

@@ -21,7 +21,7 @@ from deltacat.compute.compactor.utils.primary_key_index import (
     group_hash_bucket_indices,
     group_record_indices_by_hash_bucket,
 )
-from deltacat.storage import interface as unimplemented_deltacat_storage
+from deltacat.storage import metastore
 from deltacat.types.media import StorageType
 from deltacat.utils.common import sha1_digest
 from deltacat.utils.ray_utils.runtime import (
@@ -90,7 +90,7 @@ def _group_file_records_by_pk_hash_bucket(
     sort_key_names: List[str],
     is_src_delta: np.bool_ = True,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> Tuple[Optional[DeltaFileEnvelopeGroups], int]:
@@ -139,7 +139,7 @@ def _read_delta_file_envelopes(
     primary_keys: List[str],
     sort_key_names: List[str],
     read_kwargs_provider: Optional[ReadKwargsProvider],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> Tuple[Optional[List[DeltaFileEnvelope]], int]:
@@ -190,7 +190,7 @@ def _timed_hash_bucket(
     enable_profiler: bool,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
     object_store: Optional[IObjectStore] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ):
@@ -249,7 +249,7 @@ def hash_bucket(
     metrics_config: MetricsConfig,
     read_kwargs_provider: Optional[ReadKwargsProvider],
     object_store: Optional[IObjectStore],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> HashBucketResult:

deltacat/compute/compactor/steps/materialize.py

@@ -29,7 +29,7 @@ from deltacat.storage import (
     ManifestEntryList,
 )
 from deltacat.storage.model.manifest import Manifest
-from deltacat.storage import interface as unimplemented_deltacat_storage
+
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.types.media import DELIMITED_TEXT_CONTENT_TYPES, ContentType
 from deltacat.types.tables import TABLE_CLASS_TO_SIZE_FUNC
@@ -46,6 +46,7 @@ from deltacat.utils.ray_utils.runtime import (
 )
 from deltacat.utils.metrics import emit_timer_metrics, MetricsConfig
 from deltacat.utils.resources import get_current_process_peak_memory_usage_in_bytes
+from deltacat.storage import metastore
 
 if importlib.util.find_spec("memray"):
     import memray
@@ -67,9 +68,9 @@ def materialize(
     metrics_config: MetricsConfig,
     schema: Optional[pa.Schema] = None,
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     object_store: Optional[IObjectStore] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
 ):
     if deltacat_storage_kwargs is None:
@@ -78,11 +79,11 @@ def materialize(
     def _stage_delta_from_manifest_entry_reference_list(
         manifest_entry_list_reference: List[ManifestEntry],
         partition: Partition,
-        delta_type: DeltaType = DeltaType.UPSERT,
+        delta_type: DeltaType = DeltaType.APPEND,
     ) -> Delta:
         assert (
-            delta_type == DeltaType.UPSERT
-        ), "Stage delta with existing manifest entries only supports UPSERT delta type!"
+            delta_type == DeltaType.APPEND
+        ), "Compaction should always produce APPEND deltas for consistent read operations!"
         manifest = Manifest.of(
             entries=ManifestEntryList.of(manifest_entry_list_reference),
             uuid=str(uuid4()),
@@ -110,9 +111,10 @@ def materialize(
             deltacat_storage.stage_delta,
             compacted_table,
             partition,
+            delta_type=DeltaType.APPEND,  # Compaction always produces APPEND deltas
             max_records_per_entry=max_records_per_output_file,
             content_type=compacted_file_content_type,
-            s3_table_writer_kwargs=s3_table_writer_kwargs,
+            table_writer_kwargs=table_writer_kwargs,
             **deltacat_storage_kwargs,
         )
         compacted_table_size = TABLE_CLASS_TO_SIZE_FUNC[type(compacted_table)](

deltacat/compute/compactor/steps/repartition.py

@@ -10,7 +10,7 @@ import ray
 from deltacat import logs
 from deltacat.compute.compactor import DeltaAnnotated
 from deltacat.compute.compactor.model.repartition_result import RepartitionResult
-from deltacat.storage import interface as unimplemented_deltacat_storage
+from deltacat.storage import metastore
 from deltacat.storage import Partition
 from deltacat.utils.ray_utils.runtime import (
     get_current_ray_task_id,
@@ -19,7 +19,7 @@ from deltacat.utils.ray_utils.runtime import (
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.utils.performance import timed_invocation
 from deltacat.utils.metrics import emit_timer_metrics, MetricsConfig
-from deltacat.storage import Delta
+from deltacat.storage import Delta, DeltaType
 from enum import Enum
 
 if importlib.util.find_spec("memray"):
@@ -56,9 +56,9 @@ def repartition_range(
     destination_partition: Partition,
     repartition_args: dict,
     max_records_per_output_file: int,
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     repartitioned_file_content_type: ContentType = ContentType.PARQUET,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ):
@@ -144,9 +144,10 @@ def repartition_range(
                 partition_delta: Delta = deltacat_storage.stage_delta(
                     partition_table,
                     destination_partition,
+                    delta_type=DeltaType.APPEND,  # Repartition always produces APPEND deltas
                     max_records_per_entry=max_records_per_output_file,
                     content_type=repartitioned_file_content_type,
-                    s3_table_writer_kwargs=s3_table_writer_kwargs,
+                    table_writer_kwargs=table_writer_kwargs,
                     **deltacat_storage_kwargs,
                 )
                 partition_deltas.append(partition_delta)
@@ -168,9 +169,9 @@ def _timed_repartition(
     max_records_per_output_file: int,
     enable_profiler: bool,
     read_kwargs_provider: Optional[ReadKwargsProvider],
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     repartitioned_file_content_type: ContentType = ContentType.PARQUET,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> RepartitionResult:
@@ -192,7 +193,7 @@ def _timed_repartition(
                 destination_partition=destination_partition,
                 repartition_args=repartition_args,
                 max_records_per_output_file=max_records_per_output_file,
-                s3_table_writer_kwargs=s3_table_writer_kwargs,
+                table_writer_kwargs=table_writer_kwargs,
                 repartitioned_file_content_type=repartitioned_file_content_type,
                 deltacat_storage=deltacat_storage,
                 deltacat_storage_kwargs=deltacat_storage_kwargs,
@@ -213,9 +214,9 @@ def repartition(
     enable_profiler: bool,
     metrics_config: Optional[MetricsConfig],
     read_kwargs_provider: Optional[ReadKwargsProvider],
-    s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+    table_writer_kwargs: Optional[Dict[str, Any]] = None,
     repartitioned_file_content_type: ContentType = ContentType.PARQUET,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> RepartitionResult:
@@ -231,7 +232,7 @@ def repartition(
         max_records_per_output_file=max_records_per_output_file,
         enable_profiler=enable_profiler,
         read_kwargs_provider=read_kwargs_provider,
-        s3_table_writer_kwargs=s3_table_writer_kwargs,
+        table_writer_kwargs=table_writer_kwargs,
         repartitioned_file_content_type=repartitioned_file_content_type,
         deltacat_storage=deltacat_storage,
         deltacat_storage_kwargs=deltacat_storage_kwargs,

deltacat/compute/compactor/utils/io.py

@@ -11,7 +11,7 @@ from deltacat.storage import (
     PartitionLocator,
     Delta,
     ManifestEntry,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from deltacat import logs
 from deltacat.compute.compactor import DeltaAnnotated
@@ -31,12 +31,13 @@ def discover_deltas(
     compacted_partition_locator: Optional[PartitionLocator],
     rebase_source_partition_locator: Optional[PartitionLocator],
     rebase_source_partition_high_watermark: Optional[int],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
     list_deltas_kwargs: Optional[Dict[str, Any]] = {},
 ) -> Tuple[List[Delta], int]:
     if deltacat_storage_kwargs is None:
         deltacat_storage_kwargs = {}
+
     # Source One: new deltas from uncompacted table for incremental compaction or deltas from compacted table for rebase
     start_position_exclusive = (
         high_watermark.get(source_partition_locator)
@@ -109,7 +110,7 @@ def limit_input_deltas(
     user_hash_bucket_chunk_size: int,
     input_deltas_stats: Dict[int, DeltaStats],
     compaction_audit: CompactionSessionAuditInfo,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> Tuple[List[DeltaAnnotated], int, HighWatermark, bool]:
@@ -272,7 +273,7 @@ def fit_input_deltas(
     cluster_resources: Dict[str, float],
     compaction_audit: CompactionSessionAuditInfo,
     hash_bucket_count: Optional[int],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
     **kwargs,
 ) -> Tuple[List[DeltaAnnotated], int, HighWatermark, bool]:
@@ -359,7 +360,7 @@ def _discover_deltas(
     source_partition_locator: PartitionLocator,
     start_position_exclusive: Optional[int],
     end_position_inclusive: Optional[int],
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
     list_deltas_kwargs: Optional[Dict[str, Any]] = {},
 ) -> List[Delta]:

deltacat/compute/compactor/utils/round_completion_reader.py

@@ -0,0 +1,117 @@
+import logging
+from typing import Optional
+from deltacat import logs
+from deltacat.compute.compactor import RoundCompletionInfo
+from deltacat.storage import PartitionLocator
+from deltacat.storage.model.partition import Partition
+from deltacat.utils.metrics import metrics
+from deltacat.exceptions import PartitionNotFoundError
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+@metrics
+def read_round_completion_info(
+    source_partition_locator: PartitionLocator,
+    destination_partition_locator: PartitionLocator,
+    deltacat_storage,
+    deltacat_storage_kwargs: Optional[dict] = None,
+    destination_partition: Optional[Partition] = None,
+) -> Optional[RoundCompletionInfo]:
+    """
+    Read round completion info from the partition metafile.
+
+    Args:
+        source_partition_locator: Source partition locator for validation
+        destination_partition_locator: Destination partition locator
+        deltacat_storage: Storage implementation
+        deltacat_storage_kwargs: Optional storage kwargs
+        destination_partition: Optional destination partition to avoid redundant get_partition calls
+
+    Returns:
+        RoundCompletionInfo if found in partition, None otherwise
+    """
+    if not destination_partition_locator:
+        return None
+
+    if deltacat_storage_kwargs is None:
+        deltacat_storage_kwargs = {}
+
+    try:
+        # Use provided partition or get it from storage
+        if destination_partition:
+            partition = destination_partition
+        else:
+            # First get the current partition to access its previous_partition_id
+            current_partition: Partition = deltacat_storage.get_partition(
+                destination_partition_locator.stream_locator,
+                destination_partition_locator.partition_values,
+                **deltacat_storage_kwargs,
+            )
+
+            # If current partition has round completion info, use it
+            if current_partition.compaction_round_completion_info:
+                partition = current_partition
+            elif current_partition.previous_partition_id is not None:
+                # For incremental compaction, we need to get the previous committed partition
+                # that contains the round completion info.
+                # Get the previous partition by ID - this is where the round completion info should be
+                logger.info(
+                    f"Current partition {destination_partition_locator} does not have round completion info, "
+                    f"getting previous partition with ID: {current_partition.previous_partition_id}"
+                )
+                previous_partition = deltacat_storage.get_partition_by_id(
+                    destination_partition_locator.stream_locator,
+                    current_partition.previous_partition_id,
+                    **deltacat_storage_kwargs,
+                )
+                if previous_partition is not None:
+                    logger.info(
+                        f"Found previous partition: {previous_partition.locator}"
+                    )
+                    partition = previous_partition
+                else:
+                    raise PartitionNotFoundError(
+                        f"Previous partition with ID {current_partition.previous_partition_id} not found"
+                    )
+            else:
+                logger.info(f"No previous partition ID found, using current partition")
+                partition = current_partition
+
+        if partition:
+            round_completion_info = partition.compaction_round_completion_info
+            if round_completion_info:
+                # Validate that prev_source_partition_locator matches current source
+                if (
+                    not source_partition_locator
+                    or not round_completion_info.prev_source_partition_locator
+                ):
+                    raise ValueError(
+                        f"Source partition locator ({source_partition_locator}) and "
+                        f"prev_source_partition_locator ({round_completion_info.prev_source_partition_locator}) "
+                        f"must both be provided."
+                    )
+
+                if (
+                    round_completion_info.prev_source_partition_locator.canonical_string()
+                    != source_partition_locator.canonical_string()
+                ):
+                    logger.warning(
+                        f"Previous source partition locator mismatch: "
+                        f"expected {source_partition_locator.canonical_string()}, "
+                        f"but found {round_completion_info.prev_source_partition_locator.canonical_string()} "
+                        f"in round completion info. Ignoring cached round completion info."
+                    )
+                    return None
+
+                logger.info(
+                    f"Read round completion info from partition metafile: {round_completion_info}"
+                )
+                return round_completion_info
+
+    except Exception as e:
+        logger.debug(
+            f"Failed to read round completion info from partition metafile: {e}"
+        )
+
+    return None

deltacat/compute/compactor/utils/system_columns.py

@@ -294,7 +294,9 @@ def append_dedupe_task_idx_col(table: pa.Table, dedupe_task_indices) -> pa.Table
 
 
 def delta_type_to_field(delta_type: DeltaType) -> bool:
-    return True if delta_type is DeltaType.UPSERT else False
+    # For deduplication purposes, treat both UPSERT and APPEND as UPSERT (True)
+    # Only DELETE should be treated as DELETE (False)
+    return delta_type is not DeltaType.DELETE
 
 
 def delta_type_from_field(delta_type_field: bool) -> DeltaType:

deltacat/compute/compactor_v2/compaction_session.py

@@ -14,7 +14,6 @@ from deltacat.compute.compactor_v2.model.evaluate_compaction_result import (
     ExecutionCompactionResult,
 )
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
-from deltacat.compute.compactor.utils import round_completion_file as rcf
 from deltacat.compute.compactor import DeltaAnnotated
 from deltacat.compute.compactor_v2.deletes.delete_strategy import (
     DeleteStrategy,
@@ -27,6 +26,7 @@ from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
 from deltacat.storage import (
     Delta,
     DeltaLocator,
+    PartitionLocator,
 )
 from deltacat.storage.model.manifest import Manifest
 from deltacat.compute.compactor.model.compact_partition_params import (
@@ -36,13 +36,14 @@ from deltacat.utils.resources import (
     get_current_process_peak_memory_usage_in_bytes,
 )
 from deltacat.compute.compactor_v2.private.compaction_utils import (
+    _get_rci_source_partition_locator,
     _fetch_compaction_metadata,
     _build_uniform_deltas,
     _group_uniform_deltas,
     _stage_new_partition,
     _run_hash_and_merge,
     _process_merge_results,
-    _write_new_round_completion_file,
+    _create_round_completion_info,
     _commit_compaction_result,
 )
 from deltacat.utils.metrics import metrics
@@ -64,24 +65,26 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 @metrics(prefix=COMPACT_PARTITION_METRIC_PREFIX)
 @categorize_errors
-def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]:
+def compact_partition(params: CompactPartitionParams, **kwargs) -> None:
     assert (
         params.hash_bucket_count is not None and params.hash_bucket_count >= 1
     ), "hash_bucket_count is a required arg for compactor v2"
+    assert type(params.hash_bucket_count) is int, "Hash bucket count must be an integer"
     if params.num_rounds > 1:
         assert (
             not params.drop_duplicates
         ), "num_rounds > 1, drop_duplicates must be False but is True"
 
-    with memray.Tracker(
-        "compaction_partition.bin"
-    ) if params.enable_profiler else nullcontext():
+    with (
+        memray.Tracker("compaction_partition.bin")
+        if params.enable_profiler
+        else nullcontext()
+    ):
         execute_compaction_result: ExecutionCompactionResult = _execute_compaction(
             params,
             **kwargs,
         )
         _commit_compaction_result(params, execute_compaction_result)
-        return execute_compaction_result.round_completion_file_s3_url
 
 
 def _execute_compaction(
@@ -96,12 +99,12 @@ def _execute_compaction(
         previous_compacted_delta_manifest,
         round_completion_info,
     ) = fetch_compaction_metadata_result
-    rcf_source_partition_locator: rcf.PartitionLocator = (
-        params.rebase_source_partition_locator or params.source_partition_locator
+    rci_source_partition_locator: PartitionLocator = _get_rci_source_partition_locator(
+        params
     )
 
-    base_audit_url: str = rcf_source_partition_locator.path(
-        f"s3://{params.compaction_artifact_s3_bucket}/compaction-audit"
+    base_audit_url: str = rci_source_partition_locator.path(
+        f"{params.compaction_artifact_path}/compaction-audit"
     )
     audit_url: str = f"{base_audit_url}.json"
     logger.info(f"Compaction audit will be written to {audit_url}")
@@ -136,7 +139,7 @@ def _execute_compaction(
     )
     if not input_deltas:
         logger.info("No input deltas found to compact.")
-        return ExecutionCompactionResult(None, None, None, False)
+        return ExecutionCompactionResult(None, None, False)
     build_uniform_deltas_result: tuple[
         List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope]
     ] = _build_uniform_deltas(
@@ -199,13 +202,13 @@ def _execute_compaction(
 
     compaction_audit.save_round_completion_stats(mat_results)
 
-    compaction_result: ExecutionCompactionResult = _write_new_round_completion_file(
+    compaction_result: ExecutionCompactionResult = _create_round_completion_info(
         params,
         compaction_audit,
         compacted_partition,
         audit_url,
         hb_id_to_entry_indices_range,
-        rcf_source_partition_locator,
+        rci_source_partition_locator,
         new_compacted_delta_locator,
         pyarrow_write_result,
         round_completion_info,

deltacat/compute/compactor_v2/constants.py

@@ -1,3 +1,5 @@
+from deltacat.utils.common import env_bool, env_integer, env_string
+
 TOTAL_BYTES_IN_SHA1_HASH = 20
 
 PK_DELIMITER = "L6kl7u5f"
@@ -31,7 +33,9 @@ TOTAL_MEMORY_BUFFER_PERCENTAGE = 30
 # The total size of records that will be hash bucketed at once
 # Since, sorting is nlogn, we ensure that is not performed
 # on a very large dataset for best performance.
-MAX_SIZE_OF_RECORD_BATCH_IN_GIB = 2 * 1024 * 1024 * 1024
+MAX_SIZE_OF_RECORD_BATCH_IN_GIB = env_integer(
+    "MAX_SIZE_OF_RECORD_BATCH_IN_GIB", 2 * 1024 * 1024 * 1024
+)
 
 # Whether to drop duplicates during merge.
 DROP_DUPLICATES = True
@@ -78,3 +82,28 @@ COMPACT_PARTITION_METRIC_PREFIX = "compact_partition"
 # Number of rounds to run hash/merge for a single
 # partition. (For large table support)
 DEFAULT_NUM_ROUNDS = 1
+
+# Whether to perform sha1 hashing when required to
+# optimize memory. For example, hashing is always
+# required for bucketing where it's not mandatory
+# when dropping duplicates. Setting this to True
+# will disable sha1 hashing in cases where it isn't
+# mandatory. This flag is False by default.
+SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED = env_bool(
+    "SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED", False
+)
+
+# This env variable specifies whether to check bucketing spec
+# compliance of the existing compacted table.
+# PRINT_LOG: Enable logging if any partition is found
+# to be non-compliant with the bucketing spec.
+# ASSERT: Fail the job with ValidationError if the
+# current compacted partition is found to be non-compliant
+# with bucketing spec. Note, logging is implicitly enabled
+# in this case.
+BUCKETING_SPEC_COMPLIANCE_PROFILE = env_string(
+    "BUCKETING_SPEC_COMPLIANCE_PROFILE", None
+)
+
+BUCKETING_SPEC_COMPLIANCE_PRINT_LOG = "PRINT_LOG"
+BUCKETING_SPEC_COMPLIANCE_ASSERT = "ASSERT"

deltacat/compute/compactor_v2/model/evaluate_compaction_result.py

@@ -13,7 +13,6 @@ from typing import Optional
 class ExecutionCompactionResult:
     new_compacted_partition: Optional[Partition]
     new_round_completion_info: Optional[RoundCompletionInfo]
-    round_completion_file_s3_url: Optional[str]
     is_inplace_compacted: bool
 
     def __iter__(self):

deltacat/compute/compactor_v2/model/hash_bucket_input.py

@@ -4,7 +4,7 @@ from typing import Dict, List, Optional, Any
 from deltacat.utils.metrics import MetricsConfig
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.io.object_store import IObjectStore
-from deltacat.storage import interface as unimplemented_deltacat_storage
+from deltacat.storage import metastore
 from deltacat.compute.compactor import DeltaAnnotated
 
 
@@ -15,12 +15,13 @@ class HashBucketInput(Dict):
         primary_keys: List[str],
         num_hash_buckets: int,
         num_hash_groups: int,
+        all_column_names: List[str],
         hb_task_index: Optional[int] = 0,
         enable_profiler: Optional[bool] = False,
         metrics_config: Optional[MetricsConfig] = None,
         read_kwargs_provider: Optional[ReadKwargsProvider] = None,
         object_store: Optional[IObjectStore] = None,
-        deltacat_storage=unimplemented_deltacat_storage,
+        deltacat_storage=metastore,
         deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
         memory_logs_enabled: Optional[bool] = None,
     ) -> HashBucketInput:
@@ -31,6 +32,7 @@ class HashBucketInput(Dict):
         result["hb_task_index"] = hb_task_index
         result["num_hash_buckets"] = num_hash_buckets
         result["num_hash_groups"] = num_hash_groups
+        result["all_column_names"] = all_column_names
         result["enable_profiler"] = enable_profiler
         result["metrics_config"] = metrics_config
         result["read_kwargs_provider"] = read_kwargs_provider
@@ -61,6 +63,10 @@ class HashBucketInput(Dict):
     def num_hash_groups(self) -> int:
         return self["num_hash_groups"]
 
+    @property
+    def all_column_names(self) -> List[str]:
+        return self["all_column_names"]
+
     @property
     def enable_profiler(self) -> Optional[bool]:
         return self.get("enable_profiler")
@@ -78,7 +84,7 @@ class HashBucketInput(Dict):
         return self.get("object_store")
 
     @property
-    def deltacat_storage(self) -> unimplemented_deltacat_storage:
+    def deltacat_storage(self) -> metastore:
         return self.get("deltacat_storage")
 
     @property