deltacat 1.1.14__py3-none-any.whl → 1.1.16__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.1.14"
+__version__ = "1.1.16"
 
 
 __all__ = [

deltacat/compute/compactor/compaction_session.py

@@ -688,8 +688,9 @@ def _execute_compaction_round(
         session_peak_memory
     )
 
-    compaction_audit.save_round_completion_stats(
-        mat_results, telemetry_time_hb + telemetry_time_dd + telemetry_time_materialize
+    compaction_audit.save_round_completion_stats(mat_results)
+    compaction_audit.set_telemetry_time_in_seconds(
+        telemetry_time_hb + telemetry_time_dd + telemetry_time_materialize
     )
 
     s3_utils.upload(

deltacat/compute/compactor/model/compact_partition_params.py

@@ -22,6 +22,7 @@ from deltacat.compute.compactor_v2.constants import (
     DROP_DUPLICATES,
     TOTAL_MEMORY_BUFFER_PERCENTAGE,
     DEFAULT_DISABLE_COPY_BY_REFERENCE,
+    DEFAULT_NUM_ROUNDS,
 )
 from deltacat.constants import PYARROW_INFLATION_MULTIPLIER
 from deltacat.compute.compactor.utils.sort_key import validate_sort_keys
@@ -102,6 +103,8 @@ class CompactPartitionParams(dict):
 
         result.metrics_config = params.get("metrics_config")
 
+        result.num_rounds = params.get("num_rounds", DEFAULT_NUM_ROUNDS)
+
         if not importlib.util.find_spec("memray"):
             result.enable_profiler = False
 
@@ -189,7 +192,6 @@ class CompactPartitionParams(dict):
             cluster_resources = self.pg_config.resource
             cluster_cpus = cluster_resources["CPU"]
             self.task_max_parallelism = cluster_cpus
-            self["task_max_parallelism"] = self.task_max_parallelism
         return self["task_max_parallelism"]
 
     @task_max_parallelism.setter
@@ -403,6 +405,14 @@ class CompactPartitionParams(dict):
     def metrics_config(self, config: MetricsConfig) -> None:
         self["metrics_config"] = config
 
+    @property
+    def num_rounds(self) -> int:
+        return self["num_rounds"]
+
+    @num_rounds.setter
+    def num_rounds(self, num_rounds: int) -> None:
+        self["num_rounds"] = num_rounds
+
     @staticmethod
     def json_handler_for_compact_partition_params(obj):
         """

deltacat/compute/compactor/model/compaction_session_audit_info.py

@@ -818,7 +818,8 @@ class CompactionSessionAuditInfo(dict):
         return cluster_util_after_task_latency + telemetry_time
 
     def save_round_completion_stats(
-        self, mat_results: List[MaterializeResult], total_telemetry_time: float
+        self,
+        mat_results: List[MaterializeResult],
     ) -> None:
         """
         This method saves all the relevant stats after all the steps are completed.
@@ -888,4 +889,3 @@ class CompactionSessionAuditInfo(dict):
         )
 
         self.set_pyarrow_version(pa.__version__)
-        self.set_telemetry_time_in_seconds(total_telemetry_time)

deltacat/compute/compactor/model/delta_annotated.py

@@ -97,8 +97,7 @@ class DeltaAnnotated(Delta):
         for src_da in split_annotated_deltas:
             src_da_annotations = src_da.annotations
             src_da_entries = src_da.manifest.entries
-            assert (
-                len(src_da_annotations) == len(src_da_entries),
+            assert len(src_da_annotations) == len(src_da_entries), (
                 f"Unexpected Error: Length of delta annotations "
                 f"({len(src_da_annotations)}) doesn't mach the length of "
                 f"delta manifest entries ({len(src_da_entries)}).",
@@ -152,8 +151,7 @@ class DeltaAnnotated(Delta):
         da_group_entry_count = 0
         src_da_annotations = src_da.annotations
         src_da_entries = src_da.manifest.entries
-        assert (
-            len(src_da_annotations) == len(src_da_entries),
+        assert len(src_da_annotations) == len(src_da_entries), (
             f"Unexpected Error: Length of delta annotations "
             f"({len(src_da_annotations)}) doesn't mach the length of "
             f"delta manifest entries ({len(src_da_entries)}).",

deltacat/compute/compactor/steps/hash_bucket.py

@@ -157,11 +157,10 @@ def _read_delta_file_envelopes(
         **deltacat_storage_kwargs,
     )
     annotations = annotated_delta.annotations
-    assert (
-        len(tables) == len(annotations),
+    assert len(tables) == len(annotations), (
         f"Unexpected Error: Length of downloaded delta manifest tables "
         f"({len(tables)}) doesn't match the length of delta manifest "
-        f"annotations ({len(annotations)}).",
+        f"annotations ({len(annotations)})."
     )
     if not tables:
         return None, 0

deltacat/compute/compactor_v2/compaction_session.py

@@ -1,4 +1,3 @@
-import numpy as np
 import importlib
 from contextlib import nullcontext
 import logging
@@ -40,9 +39,10 @@ from deltacat.utils.resources import (
 from deltacat.compute.compactor_v2.private.compaction_utils import (
     _fetch_compaction_metadata,
     _build_uniform_deltas,
+    _group_uniform_deltas,
+    _stage_new_partition,
     _run_hash_and_merge,
     _process_merge_results,
-    _upload_compaction_audit,
     _write_new_round_completion_file,
     _commit_compaction_result,
 )
@@ -69,6 +69,10 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
     assert (
         params.hash_bucket_count is not None and params.hash_bucket_count >= 1
     ), "hash_bucket_count is a required arg for compactor v2"
+    if params.num_rounds > 1:
+        assert (
+            not params.drop_duplicates
+        ), "num_rounds > 1, drop_duplicates must be False but is True"
 
     with memray.Tracker(
         "compaction_partition.bin"
@@ -144,32 +148,28 @@ def _execute_compaction(
         delete_strategy,
         delete_file_envelopes,
     ) = build_uniform_deltas_result
-
-    # run merge
-    _run_hash_and_merge_result: tuple[
-        Optional[List[MergeResult]],
-        np.float64,
-        np.float64,
-        Partition,
-    ] = _run_hash_and_merge(
-        params,
-        uniform_deltas,
-        round_completion_info,
-        delete_strategy,
-        delete_file_envelopes,
-        compaction_audit,
-        previous_compacted_delta_manifest,
-    )
-    (
-        merge_results,
-        telemetry_time_hb,
-        telemetry_time_merge,
-        compacted_partition,
-    ) = _run_hash_and_merge_result
+    logger.info(f"Number of rounds parameter is set to: {params.num_rounds}")
+    uniform_deltas_grouped = _group_uniform_deltas(params, uniform_deltas)
+    logger.info(f"Length of grouped uniform deltas is: {len(uniform_deltas_grouped)}")
+    merge_result_list: List[MergeResult] = []
+    compacted_partition = _stage_new_partition(params)
+    for uniform_deltas in uniform_deltas_grouped:
+        # run hash and merge
+        _run_hash_and_merge_result: List[MergeResult] = _run_hash_and_merge(
+            params,
+            uniform_deltas,
+            round_completion_info,
+            delete_strategy,
+            delete_file_envelopes,
+            compaction_audit,
+            previous_compacted_delta_manifest,
+            compacted_partition,
+        )
+        merge_result_list.extend(_run_hash_and_merge_result)
     # process merge results
     process_merge_results: tuple[
         Delta, list[MaterializeResult], dict
-    ] = _process_merge_results(params, merge_results, compaction_audit)
+    ] = _process_merge_results(params, merge_result_list, compaction_audit)
     merged_delta, mat_results, hb_id_to_entry_indices_range = process_merge_results
     # Record information, logging, and return ExecutionCompactionResult
     record_info_msg: str = f" Materialized records: {merged_delta.meta.record_count}"
@@ -198,15 +198,8 @@ def _execute_compaction(
         session_peak_memory
     )
 
-    compaction_audit.save_round_completion_stats(
-        mat_results, telemetry_time_hb + telemetry_time_merge
-    )
+    compaction_audit.save_round_completion_stats(mat_results)
 
-    _upload_compaction_audit(
-        params,
-        compaction_audit,
-        round_completion_info,
-    )
     compaction_result: ExecutionCompactionResult = _write_new_round_completion_file(
         params,
         compaction_audit,
@@ -216,5 +209,6 @@ def _execute_compaction(
         rcf_source_partition_locator,
         new_compacted_delta_locator,
         pyarrow_write_result,
+        round_completion_info,
     )
     return compaction_result

deltacat/compute/compactor_v2/constants.py

@@ -71,3 +71,7 @@ PREPARE_DELETES_METRIC_PREFIX = "prepare_deletes"
 
 # Metric prefix for compact partition method
 COMPACT_PARTITION_METRIC_PREFIX = "compact_partition"
+
+# Number of rounds to run hash/merge for a single
+# partition. (For large table support)
+DEFAULT_NUM_ROUNDS = 1

deltacat/compute/compactor_v2/private/compaction_utils.py

@@ -4,8 +4,10 @@ import logging
 import ray
 import time
 import json
+from math import ceil
 
 from deltacat.compute.compactor import (
+    PyArrowWriteResult,
     HighWatermark,
     RoundCompletionInfo,
 )
@@ -44,10 +46,11 @@ from deltacat.compute.compactor_v2.deletes.utils import prepare_deletes
 from deltacat.storage import (
     Delta,
     DeltaType,
-    Stream,
-    StreamLocator,
+    DeltaLocator,
     Partition,
     Manifest,
+    Stream,
+    StreamLocator,
 )
 from deltacat.compute.compactor.model.compact_partition_params import (
     CompactPartitionParams,
@@ -60,7 +63,7 @@ from deltacat.compute.compactor_v2.steps import merge as mg
 from deltacat.compute.compactor_v2.steps import hash_bucket as hb
 from deltacat.compute.compactor_v2.utils import io
 
-from typing import Any, List, Optional
+from typing import List, Optional
 from collections import defaultdict
 from deltacat.compute.compactor.model.compaction_session_audit_info import (
     CompactionSessionAuditInfo,
@@ -123,9 +126,9 @@ def _fetch_compaction_metadata(
 
 def _build_uniform_deltas(
     params: CompactPartitionParams,
-    mutable_compaction_audit,
-    input_deltas,
-    delta_discovery_start,
+    mutable_compaction_audit: CompactionSessionAuditInfo,
+    input_deltas: List[Delta],
+    delta_discovery_start: float,
 ) -> tuple[List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope], Partition]:
 
     delete_strategy: Optional[DeleteStrategy] = None
@@ -173,18 +176,34 @@ def _build_uniform_deltas(
     )
 
 
-def _run_hash_and_merge(
-    params: CompactPartitionParams,
-    uniform_deltas,
-    round_completion_info,
-    delete_strategy,
-    delete_file_envelopes,
-    mutable_compaction_audit,
-    previous_compacted_delta_manifest,
-) -> tuple[
-    list[MergeResult], np.int64, np.float64, np.int64, np.int64, np.float64, Partition
-]:
-    # create a new stream for this round
+def _group_uniform_deltas(
+    params: CompactPartitionParams, uniform_deltas: List[DeltaAnnotated]
+) -> List[List[DeltaAnnotated]]:
+    num_deltas = len(uniform_deltas)
+    num_rounds = params.num_rounds
+    if num_rounds == 1:
+        return [uniform_deltas]
+    assert (
+        num_rounds > 0
+    ), f"num_rounds parameter should be greater than zero but is {params.num_rounds}"
+    assert (
+        num_rounds <= num_deltas
+    ), f"{params.num_rounds} rounds should be less than the number of uniform deltas, which is {len(uniform_deltas)}"
+    size = ceil(num_deltas / num_rounds)
+    uniform_deltas_grouped = list(
+        map(
+            lambda x: uniform_deltas[x * size : x * size + size],
+            list(range(num_rounds)),
+        )
+    )
+    num_deltas_after_grouping = sum(len(sublist) for sublist in uniform_deltas_grouped)
+    assert (
+        num_deltas_after_grouping == num_deltas
+    ), f"uniform_deltas_grouped expected to have {num_deltas} deltas, but has {num_deltas_after_grouping}"
+    return uniform_deltas_grouped
+
+
+def _stage_new_partition(params: CompactPartitionParams) -> Partition:
     compacted_stream_locator: Optional[
         StreamLocator
     ] = params.destination_partition_locator.stream_locator
@@ -199,7 +218,19 @@ def _run_hash_and_merge(
         params.destination_partition_locator.partition_values,
         **params.deltacat_storage_kwargs,
     )
+    return compacted_partition
+
 
+def _run_hash_and_merge(
+    params: CompactPartitionParams,
+    uniform_deltas: List[DeltaAnnotated],
+    round_completion_info: RoundCompletionInfo,
+    delete_strategy: Optional[DeleteStrategy],
+    delete_file_envelopes: Optional[DeleteFileEnvelope],
+    mutable_compaction_audit: CompactionSessionAuditInfo,
+    previous_compacted_delta_manifest: Optional[Manifest],
+    compacted_partition: Partition,
+) -> List[MergeResult]:
     telemetry_time_hb = 0
     total_input_records_count = np.int64(0)
     total_hb_record_count = np.int64(0)
@@ -257,7 +288,6 @@ def _run_hash_and_merge(
         for hb_result in hb_results:
             hb_data_processed_size_bytes += hb_result.hb_size_bytes
             total_input_records_count += hb_result.hb_record_count
-
             for hash_group_index, object_id_size_tuple in enumerate(
                 hb_result.hash_bucket_group_to_obj_id_tuple
             ):
@@ -271,7 +301,6 @@ def _run_hash_and_merge(
                     all_hash_group_idx_to_num_rows[
                         hash_group_index
                     ] += object_id_size_tuple[2].item()
-
         logger.info(
             f"Got {total_input_records_count} hash bucket records from hash bucketing step..."
         )
@@ -330,26 +359,31 @@ def _run_hash_and_merge(
         f" Deleted records: {total_deleted_record_count}, "
     )
     logger.info(record_info_msg)
-    return (
-        merge_results,
-        telemetry_time_hb,
-        telemetry_time_merge,
-        compacted_partition,
+    telemetry_this_round = telemetry_time_hb + telemetry_time_merge
+    previous_telemetry = (
+        mutable_compaction_audit.telemetry_time_in_seconds
+        if mutable_compaction_audit.telemetry_time_in_seconds
+        else 0.0
     )
 
+    mutable_compaction_audit.set_telemetry_time_in_seconds(
+        telemetry_this_round + previous_telemetry
+    )
+    return merge_results
+
 
 def _merge(
     params: CompactPartitionParams,
-    task_resource_options_provider,
-    merge_resource_options_provider,
-    all_hash_group_idx_to_size_bytes,
-    all_hash_group_idx_to_num_rows,
-    round_completion_info,
-    previous_compacted_delta_manifest,
-    all_hash_group_idx_to_obj_id,
-    compacted_partition,
-    delete_strategy,
-    delete_file_envelopes,
+    task_resource_options_provider: callable,
+    merge_resource_options_provider: callable,
+    all_hash_group_idx_to_size_bytes: dict,
+    all_hash_group_idx_to_num_rows: dict,
+    round_completion_info: RoundCompletionInfo,
+    previous_compacted_delta_manifest: Manifest,
+    all_hash_group_idx_to_obj_id: dict,
+    compacted_partition: Partition,
+    delete_strategy: DeleteStrategy,
+    delete_file_envelopes: DeleteFileEnvelope,
 ) -> tuple[List[MergeResult], float]:
     merge_options_provider = functools.partial(
         task_resource_options_provider,
@@ -416,8 +450,9 @@ def _merge(
 
 def _hash_bucket(
     params: CompactPartitionParams,
-    uniform_deltas,
-):
+    uniform_deltas: List[DeltaAnnotated],
+) -> tuple[List[HashBucketResult], float]:
+
     hb_options_provider = functools.partial(
         task_resource_options_provider,
         pg_config=params.pg_config,
@@ -455,7 +490,6 @@ def _hash_bucket(
         options_provider=hb_options_provider,
         kwargs_provider=hash_bucket_input_provider,
     )
-
     hb_invoke_end = time.monotonic()
 
     logger.info(f"Getting {len(hb_tasks_pending)} hash bucket results...")
@@ -467,15 +501,15 @@ def _hash_bucket(
 
 def _run_local_merge(
     params: CompactPartitionParams,
-    uniform_deltas,
-    compacted_partition,
-    round_completion_info,
-    delete_strategy,
-    delete_file_envelopes,
-    mutable_compaction_audit,
-    previous_compacted_delta_manifest,
-    total_input_records_count,
-) -> tuple[list[Any], Any]:
+    uniform_deltas: List[DeltaAnnotated],
+    compacted_partition: Partition,
+    round_completion_info: RoundCompletionInfo,
+    delete_strategy: Optional[DeleteStrategy],
+    delete_file_envelopes: Optional[DeleteFileEnvelope],
+    mutable_compaction_audit: CompactionSessionAuditInfo,
+    previous_compacted_delta_manifest: Optional[Manifest],
+    total_input_records_count: np.int64,
+) -> tuple[List[MergeResult], np.int64]:
     local_merge_input: MergeInput = generate_local_merge_input(
         params,
         uniform_deltas,
@@ -513,8 +547,10 @@ def _run_local_merge(
 
 
 def _process_merge_results(
-    params: CompactPartitionParams, merge_results, mutable_compaction_audit
-) -> tuple[Delta, list[MaterializeResult], dict]:
+    params: CompactPartitionParams,
+    merge_results: List[MergeResult],
+    mutable_compaction_audit: CompactionSessionAuditInfo,
+) -> tuple[Delta, List[MaterializeResult], dict]:
     mat_results = []
     for merge_result in merge_results:
         mat_results.extend(merge_result.materialize_results)
@@ -522,19 +558,23 @@ def _process_merge_results(
     mat_results: List[MaterializeResult] = sorted(
         mat_results, key=lambda m: m.task_index
     )
-
     hb_id_to_entry_indices_range = {}
     file_index = 0
     previous_task_index = -1
 
+    duplicate_hash_bucket_mat_results = 0
     for mat_result in mat_results:
         assert (
             mat_result.pyarrow_write_result.files >= 1
-        ), "Atleast one file must be materialized"
-        assert (
-            mat_result.task_index != previous_task_index
-        ), f"Multiple materialize results found for a hash bucket: {mat_result.task_index}"
-
+        ), "At least one file must be materialized"
+        if mat_result.task_index == previous_task_index:
+            duplicate_hash_bucket_mat_results += 1
+        else:
+            duplicate_hash_bucket_mat_results = 0
+        assert duplicate_hash_bucket_mat_results < params.num_rounds, (
+            f"Duplicate record count ({duplicate_hash_bucket_mat_results}) is as large "
+            f"as or greater than params.num_rounds, which is {params.num_rounds}"
+        )
         hb_id_to_entry_indices_range[str(mat_result.task_index)] = (
             file_index,
             file_index + mat_result.pyarrow_write_result.files,
@@ -548,9 +588,7 @@ def _process_merge_results(
         str(json.dumps(mutable_compaction_audit)),
         **params.s3_client_kwargs,
     )
-
     deltas: List[Delta] = [m.delta for m in mat_results]
-
     # Note: An appropriate last stream position must be set
     # to avoid correctness issue.
     merged_delta: Delta = Delta.merge_deltas(
@@ -561,10 +599,10 @@ def _process_merge_results(
     return merged_delta, mat_results, hb_id_to_entry_indices_range
 
 
-def _upload_compaction_audit(
+def _update_and_upload_compaction_audit(
     params: CompactPartitionParams,
-    mutable_compaction_audit,
-    round_completion_info,
+    mutable_compaction_audit: CompactionSessionAuditInfo,
+    round_completion_info: Optional[RoundCompletionInfo] = None,
 ) -> None:
 
     # After all incremental delta related calculations, we update
@@ -593,13 +631,14 @@ def _upload_compaction_audit(
 
 def _write_new_round_completion_file(
     params: CompactPartitionParams,
-    mutable_compaction_audit,
-    compacted_partition,
-    audit_url,
-    hb_id_to_entry_indices_range,
-    rcf_source_partition_locator,
-    new_compacted_delta_locator,
-    pyarrow_write_result,
+    mutable_compaction_audit: CompactionSessionAuditInfo,
+    compacted_partition: Partition,
+    audit_url: str,
+    hb_id_to_entry_indices_range: dict,
+    rcf_source_partition_locator: rcf.PartitionLocator,
+    new_compacted_delta_locator: DeltaLocator,
+    pyarrow_write_result: PyArrowWriteResult,
+    prev_round_completion_info: Optional[RoundCompletionInfo] = None,
 ) -> ExecutionCompactionResult:
     input_inflation = None
     input_average_record_size_bytes = None
@@ -627,6 +666,12 @@ def _write_new_round_completion_file(
         f" and average record size={input_average_record_size_bytes}"
     )
 
+    _update_and_upload_compaction_audit(
+        params,
+        mutable_compaction_audit,
+        prev_round_completion_info,
+    )
+
     new_round_completion_info = RoundCompletionInfo.of(
         high_watermark=params.last_stream_position_to_compact,
         compacted_delta_locator=new_compacted_delta_locator,

deltacat/compute/compactor_v2/steps/merge.py

@@ -177,13 +177,10 @@ def _download_compacted_table(
 
     if str(hb_index) not in hb_index_to_indices:
         return None
-
     indices = hb_index_to_indices[str(hb_index)]
-
     assert (
         indices is not None and len(indices) == 2
     ), "indices should not be none and contains exactly two elements"
-
     for offset in range(indices[1] - indices[0]):
         table = deltacat_storage.download_delta_manifest_entry(
             rcf.compacted_delta_locator,

deltacat/compute/compactor_v2/utils/delta.py

@@ -42,11 +42,10 @@ def read_delta_file_envelopes(
         **deltacat_storage_kwargs,
     )
     annotations = annotated_delta.annotations
-    assert (
-        len(tables) == len(annotations),
+    assert len(tables) == len(annotations), (
         f"Unexpected Error: Length of downloaded delta manifest tables "
         f"({len(tables)}) doesn't match the length of delta manifest "
-        f"annotations ({len(annotations)}).",
+        f"annotations ({len(annotations)})."
     )
     if not tables:
         return None, 0, 0

deltacat/compute/compactor_v2/utils/io.py

@@ -61,7 +61,6 @@ def discover_deltas(
     )
 
     result.extend(delta_source_incremental_deltas)
-
     logger.info(
         f"Length of input deltas from delta source table is {len(delta_source_incremental_deltas)}"
         f" from ({previous_compacted_high_watermark}, {last_stream_position_to_compact}]"
@@ -132,7 +131,6 @@ def create_uniform_input_deltas(
     size_estimation_function = functools.partial(
         estimate_manifest_entry_size_bytes, previous_inflation=previous_inflation
     )
-
     rebatched_da_list = DeltaAnnotated.rebatch(
         input_da_list,
         min_delta_bytes=min_delta_bytes,

deltacat/compute/compactor_v2/utils/merge.py

@@ -109,7 +109,6 @@ def generate_local_merge_input(
         A MergeInput object
 
     """
-
     return MergeInput.of(
         merge_file_groups_provider=LocalMergeFileGroupsProvider(
             annotated_deltas,