deltacat 1.1.13__py3-none-any.whl → 1.1.15__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.13"
+__version__ = "1.1.15"
 
 
 __all__ = [

deltacat/compute/compactor/compaction_session.py

@@ -688,8 +688,9 @@ def _execute_compaction_round(
         session_peak_memory
     )
 
-    compaction_audit.save_round_completion_stats(
-        mat_results, telemetry_time_hb + telemetry_time_dd + telemetry_time_materialize
+    compaction_audit.save_round_completion_stats(mat_results)
+    compaction_audit.set_telemetry_time_in_seconds(
+        telemetry_time_hb + telemetry_time_dd + telemetry_time_materialize
     )
 
     s3_utils.upload(

deltacat/compute/compactor/model/compact_partition_params.py

@@ -22,6 +22,7 @@ from deltacat.compute.compactor_v2.constants import (
     DROP_DUPLICATES,
     TOTAL_MEMORY_BUFFER_PERCENTAGE,
     DEFAULT_DISABLE_COPY_BY_REFERENCE,
+    DEFAULT_NUM_ROUNDS,
 )
 from deltacat.constants import PYARROW_INFLATION_MULTIPLIER
 from deltacat.compute.compactor.utils.sort_key import validate_sort_keys
@@ -102,6 +103,8 @@ class CompactPartitionParams(dict):
 
         result.metrics_config = params.get("metrics_config")
 
+        result.num_rounds = params.get("num_rounds", DEFAULT_NUM_ROUNDS)
+
         if not importlib.util.find_spec("memray"):
             result.enable_profiler = False
 
@@ -189,7 +192,6 @@ class CompactPartitionParams(dict):
             cluster_resources = self.pg_config.resource
             cluster_cpus = cluster_resources["CPU"]
             self.task_max_parallelism = cluster_cpus
-            self["task_max_parallelism"] = self.task_max_parallelism
         return self["task_max_parallelism"]
 
     @task_max_parallelism.setter
@@ -403,6 +405,14 @@ class CompactPartitionParams(dict):
     def metrics_config(self, config: MetricsConfig) -> None:
         self["metrics_config"] = config
 
+    @property
+    def num_rounds(self) -> int:
+        return self["num_rounds"]
+
+    @num_rounds.setter
+    def num_rounds(self, num_rounds: int) -> None:
+        self["num_rounds"] = num_rounds
+
     @staticmethod
     def json_handler_for_compact_partition_params(obj):
         """

deltacat/compute/compactor/model/compaction_session_audit_info.py

@@ -818,7 +818,8 @@ class CompactionSessionAuditInfo(dict):
         return cluster_util_after_task_latency + telemetry_time
 
     def save_round_completion_stats(
-        self, mat_results: List[MaterializeResult], total_telemetry_time: float
+        self,
+        mat_results: List[MaterializeResult],
     ) -> None:
         """
         This method saves all the relevant stats after all the steps are completed.
@@ -888,4 +889,3 @@ class CompactionSessionAuditInfo(dict):
         )
 
         self.set_pyarrow_version(pa.__version__)
-        self.set_telemetry_time_in_seconds(total_telemetry_time)

deltacat/compute/compactor/model/delta_annotated.py

@@ -97,8 +97,7 @@ class DeltaAnnotated(Delta):
         for src_da in split_annotated_deltas:
             src_da_annotations = src_da.annotations
             src_da_entries = src_da.manifest.entries
-            assert (
-                len(src_da_annotations) == len(src_da_entries),
+            assert len(src_da_annotations) == len(src_da_entries), (
                 f"Unexpected Error: Length of delta annotations "
                 f"({len(src_da_annotations)}) doesn't mach the length of "
                 f"delta manifest entries ({len(src_da_entries)}).",
@@ -152,8 +151,7 @@ class DeltaAnnotated(Delta):
         da_group_entry_count = 0
         src_da_annotations = src_da.annotations
         src_da_entries = src_da.manifest.entries
-        assert (
-            len(src_da_annotations) == len(src_da_entries),
+        assert len(src_da_annotations) == len(src_da_entries), (
             f"Unexpected Error: Length of delta annotations "
             f"({len(src_da_annotations)}) doesn't mach the length of "
             f"delta manifest entries ({len(src_da_entries)}).",

deltacat/compute/compactor/steps/hash_bucket.py

@@ -157,11 +157,10 @@ def _read_delta_file_envelopes(
         **deltacat_storage_kwargs,
     )
     annotations = annotated_delta.annotations
-    assert (
-        len(tables) == len(annotations),
+    assert len(tables) == len(annotations), (
         f"Unexpected Error: Length of downloaded delta manifest tables "
         f"({len(tables)}) doesn't match the length of delta manifest "
-        f"annotations ({len(annotations)}).",
+        f"annotations ({len(annotations)})."
     )
     if not tables:
         return None, 0

deltacat/compute/compactor_v2/compaction_session.py

@@ -1,4 +1,3 @@
-import numpy as np
 import importlib
 from contextlib import nullcontext
 import logging
@@ -40,6 +39,8 @@ from deltacat.utils.resources import (
 from deltacat.compute.compactor_v2.private.compaction_utils import (
     _fetch_compaction_metadata,
     _build_uniform_deltas,
+    _group_uniform_deltas,
+    _stage_new_partition,
     _run_hash_and_merge,
     _process_merge_results,
     _upload_compaction_audit,
@@ -69,6 +70,10 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
     assert (
         params.hash_bucket_count is not None and params.hash_bucket_count >= 1
     ), "hash_bucket_count is a required arg for compactor v2"
+    if params.num_rounds > 1:
+        assert (
+            not params.drop_duplicates
+        ), "num_rounds > 1, drop_duplicates must be False but is True"
 
     with memray.Tracker(
         "compaction_partition.bin"
@@ -144,32 +149,28 @@ def _execute_compaction(
         delete_strategy,
         delete_file_envelopes,
     ) = build_uniform_deltas_result
-
-    # run merge
-    _run_hash_and_merge_result: tuple[
-        Optional[List[MergeResult]],
-        np.float64,
-        np.float64,
-        Partition,
-    ] = _run_hash_and_merge(
-        params,
-        uniform_deltas,
-        round_completion_info,
-        delete_strategy,
-        delete_file_envelopes,
-        compaction_audit,
-        previous_compacted_delta_manifest,
-    )
-    (
-        merge_results,
-        telemetry_time_hb,
-        telemetry_time_merge,
-        compacted_partition,
-    ) = _run_hash_and_merge_result
+    logger.info(f"Number of rounds parameter is set to: {params.num_rounds}")
+    uniform_deltas_grouped = _group_uniform_deltas(params, uniform_deltas)
+    logger.info(f"Length of grouped uniform deltas is: {len(uniform_deltas_grouped)}")
+    merge_result_list: List[MergeResult] = []
+    compacted_partition = _stage_new_partition(params)
+    for uniform_deltas in uniform_deltas_grouped:
+        # run hash and merge
+        _run_hash_and_merge_result: List[MergeResult] = _run_hash_and_merge(
+            params,
+            uniform_deltas,
+            round_completion_info,
+            delete_strategy,
+            delete_file_envelopes,
+            compaction_audit,
+            previous_compacted_delta_manifest,
+            compacted_partition,
+        )
+        merge_result_list.extend(_run_hash_and_merge_result)
     # process merge results
     process_merge_results: tuple[
         Delta, list[MaterializeResult], dict
-    ] = _process_merge_results(params, merge_results, compaction_audit)
+    ] = _process_merge_results(params, merge_result_list, compaction_audit)
     merged_delta, mat_results, hb_id_to_entry_indices_range = process_merge_results
     # Record information, logging, and return ExecutionCompactionResult
     record_info_msg: str = f" Materialized records: {merged_delta.meta.record_count}"
@@ -198,9 +199,7 @@ def _execute_compaction(
         session_peak_memory
     )
 
-    compaction_audit.save_round_completion_stats(
-        mat_results, telemetry_time_hb + telemetry_time_merge
-    )
+    compaction_audit.save_round_completion_stats(mat_results)
 
     _upload_compaction_audit(
         params,

deltacat/compute/compactor_v2/constants.py

@@ -71,3 +71,7 @@ PREPARE_DELETES_METRIC_PREFIX = "prepare_deletes"
 
 # Metric prefix for compact partition method
 COMPACT_PARTITION_METRIC_PREFIX = "compact_partition"
+
+# Number of rounds to run hash/merge for a single
+# partition. (For large table support)
+DEFAULT_NUM_ROUNDS = 1