deltacat 1.1.35__py3-none-any.whl → 2.0__py3-none-any.whl

deltacat/compute/compactor/steps/hash_bucket.py

@@ -201,7 +201,7 @@ def _timed_hash_bucket(
     with memray.Tracker(
         f"hash_bucket_{worker_id}_{task_id}.bin"
     ) if enable_profiler else nullcontext():
-        sort_key_names = [key.key_name for key in sort_keys]
+        sort_key_names = list(chain.from_iterable([key.key for key in sort_keys]))
         if not round_completion_info:
             is_src_delta = True
         else:

deltacat/compute/compactor/steps/materialize.py

@@ -25,9 +25,10 @@ from deltacat.storage import (
     DeltaType,
     Partition,
     PartitionLocator,
-    Manifest,
     ManifestEntry,
+    ManifestEntryList,
 )
+from deltacat.storage.model.manifest import Manifest
 from deltacat.storage import interface as unimplemented_deltacat_storage
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.types.media import DELIMITED_TEXT_CONTENT_TYPES, ContentType
@@ -82,7 +83,10 @@ def materialize(
         assert (
             delta_type == DeltaType.UPSERT
         ), "Stage delta with existing manifest entries only supports UPSERT delta type!"
-        manifest = Manifest.of(entries=manifest_entry_list_reference, uuid=str(uuid4()))
+        manifest = Manifest.of(
+            entries=ManifestEntryList.of(manifest_entry_list_reference),
+            uuid=str(uuid4()),
+        )
         delta = Delta.of(
             locator=DeltaLocator.of(partition.locator),
             delta_type=delta_type,

deltacat/compute/compactor/utils/io.py

@@ -358,7 +358,7 @@ def fit_input_deltas(
 def _discover_deltas(
     source_partition_locator: PartitionLocator,
     start_position_exclusive: Optional[int],
-    end_position_inclusive: int,
+    end_position_inclusive: Optional[int],
     deltacat_storage=unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
     list_deltas_kwargs: Optional[Dict[str, Any]] = {},

deltacat/compute/compactor/utils/sort_key.py

@@ -1,6 +1,7 @@
 import pyarrow as pa
 from typing import List
-from deltacat.storage import PartitionLocator, SortKey
+from itertools import chain
+from deltacat.storage import PartitionLocator, SortKey, TransformName
 
 MAX_SORT_KEYS_BIT_WIDTH = 256
 
@@ -22,7 +23,13 @@ def validate_sort_keys(
         deltacat_storage_kwargs = {}
     total_sort_keys_bit_width = 0
     if sort_keys:
-        sort_key_names = [key.key_name for key in sort_keys]
+        sort_key_names = list(chain.from_iterable([key.key for key in sort_keys]))
+        assert all(
+            [
+                key.transform is None or key.transform.name == TransformName.IDENTITY
+                for key in sort_keys
+            ]
+        ), f"Sort key transforms are not supported: {sort_keys}"
         assert len(sort_key_names) == len(
             set(sort_key_names)
         ), f"Sort key names must be unique: {sort_key_names}"

deltacat/compute/compactor_v2/compaction_session.py

@@ -27,9 +27,8 @@ from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
 from deltacat.storage import (
     Delta,
     DeltaLocator,
-    Manifest,
-    Partition,
 )
+from deltacat.storage.model.manifest import Manifest
 from deltacat.compute.compactor.model.compact_partition_params import (
     CompactPartitionParams,
 )
@@ -139,7 +138,7 @@ def _execute_compaction(
         logger.info("No input deltas found to compact.")
         return ExecutionCompactionResult(None, None, None, False)
     build_uniform_deltas_result: tuple[
-        List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope], Partition
+        List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope]
     ] = _build_uniform_deltas(
         params, compaction_audit, input_deltas, delta_discovery_start
     )

deltacat/compute/compactor_v2/constants.py

@@ -1,5 +1,3 @@
-from deltacat.utils.common import env_bool, env_integer, env_string
-
 TOTAL_BYTES_IN_SHA1_HASH = 20
 
 PK_DELIMITER = "L6kl7u5f"
@@ -33,9 +31,7 @@ TOTAL_MEMORY_BUFFER_PERCENTAGE = 30
 # The total size of records that will be hash bucketed at once
 # Since, sorting is nlogn, we ensure that is not performed
 # on a very large dataset for best performance.
-MAX_SIZE_OF_RECORD_BATCH_IN_GIB = env_integer(
-    "MAX_SIZE_OF_RECORD_BATCH_IN_GIB", 2 * 1024 * 1024 * 1024
-)
+MAX_SIZE_OF_RECORD_BATCH_IN_GIB = 2 * 1024 * 1024 * 1024
 
 # Whether to drop duplicates during merge.
 DROP_DUPLICATES = True
@@ -82,28 +78,3 @@ COMPACT_PARTITION_METRIC_PREFIX = "compact_partition"
 # Number of rounds to run hash/merge for a single
 # partition. (For large table support)
 DEFAULT_NUM_ROUNDS = 1
-
-# Whether to perform sha1 hashing when required to
-# optimize memory. For example, hashing is always
-# required for bucketing where it's not mandatory
-# when dropping duplicates. Setting this to True
-# will disable sha1 hashing in cases where it isn't
-# mandatory. This flag is False by default.
-SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED = env_bool(
-    "SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED", False
-)
-
-# This env variable specifies whether to check bucketing spec
-# compliance of the existing compacted table.
-# PRINT_LOG: Enable logging if any partition is found
-# to be non-compliant with the bucketing spec.
-# ASSERT: Fail the job with ValidationError if the
-# current compacted partition is found to be non-compliant
-# with bucketing spec. Note, logging is implicitly enabled
-# in this case.
-BUCKETING_SPEC_COMPLIANCE_PROFILE = env_string(
-    "BUCKETING_SPEC_COMPLIANCE_PROFILE", None
-)
-
-BUCKETING_SPEC_COMPLIANCE_PRINT_LOG = "PRINT_LOG"
-BUCKETING_SPEC_COMPLIANCE_ASSERT = "ASSERT"

deltacat/compute/compactor_v2/deletes/utils.py

@@ -49,7 +49,7 @@ def _aggregate_delete_deltas(input_deltas: List[Delta]) -> Dict[int, List[Delta]
     ] = [
         (is_delete, list(delete_delta_group))
         for (is_delete, _), delete_delta_group in itertools.groupby(
-            input_deltas, lambda d: (d.type is DeltaType.DELETE, d.delete_parameters)
+            input_deltas, lambda d: (d.type is DeltaType.DELETE, d.meta.entry_params)
         )
     ]
     for (
@@ -89,11 +89,11 @@ def _get_delete_file_envelopes(
         consecutive_delete_tables: List[pa.Table] = []
         for delete_delta in delete_delta_sequence:
             assert (
-                delete_delta.delete_parameters is not None
+                delete_delta.meta.entry_params is not None
             ), "Delete type deltas are required to have delete parameters defined"
             delete_columns: Optional[
                 List[str]
-            ] = delete_delta.delete_parameters.equality_column_names
+            ] = delete_delta.meta.entry_params.equality_field_locators
             assert len(delete_columns) > 0, "At least 1 delete column is required"
             # delete columns should exist in underlying table
             delete_dataset = params.deltacat_storage.download_delta(

deltacat/compute/compactor_v2/model/merge_input.py

@@ -43,7 +43,7 @@ class MergeInput(Dict):
         round_completion_info: Optional[RoundCompletionInfo] = None,
         object_store: Optional[IObjectStore] = None,
         delete_strategy: Optional[DeleteStrategy] = None,
-        delete_file_envelopes: Optional[List] = None,
+        delete_file_envelopes: Optional[List[DeleteFileEnvelope]] = None,
         deltacat_storage=unimplemented_deltacat_storage,
         deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
         memory_logs_enabled: Optional[bool] = None,

deltacat/compute/compactor_v2/private/compaction_utils.py

@@ -63,7 +63,7 @@ from deltacat.compute.compactor_v2.steps import merge as mg
 from deltacat.compute.compactor_v2.steps import hash_bucket as hb
 from deltacat.compute.compactor_v2.utils import io
 
-from typing import List, Optional
+from typing import List, Optional, Union
 from collections import defaultdict
 from deltacat.compute.compactor.model.compaction_session_audit_info import (
     CompactionSessionAuditInfo,
@@ -83,7 +83,7 @@ def _fetch_compaction_metadata(
 
     # read the results from any previously completed compaction round
     round_completion_info: Optional[RoundCompletionInfo] = None
-    high_watermark: Optional[HighWatermark] = None
+    high_watermark: Optional[Union[HighWatermark, int]] = None
     previous_compacted_delta_manifest: Optional[Manifest] = None
 
     if not params.rebase_source_partition_locator:
@@ -129,7 +129,7 @@ def _build_uniform_deltas(
     mutable_compaction_audit: CompactionSessionAuditInfo,
     input_deltas: List[Delta],
     delta_discovery_start: float,
-) -> tuple[List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope], Partition]:
+) -> tuple[List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope]]:
 
     delete_strategy: Optional[DeleteStrategy] = None
     delete_file_envelopes: Optional[List[DeleteFileEnvelope]] = None
@@ -222,7 +222,7 @@ def _run_hash_and_merge(
     uniform_deltas: List[DeltaAnnotated],
     round_completion_info: RoundCompletionInfo,
     delete_strategy: Optional[DeleteStrategy],
-    delete_file_envelopes: Optional[DeleteFileEnvelope],
+    delete_file_envelopes: Optional[List[DeleteFileEnvelope]],
     mutable_compaction_audit: CompactionSessionAuditInfo,
     previous_compacted_delta_manifest: Optional[Manifest],
     compacted_partition: Partition,
@@ -389,7 +389,7 @@ def _merge(
     all_hash_group_idx_to_obj_id: dict,
     compacted_partition: Partition,
     delete_strategy: DeleteStrategy,
-    delete_file_envelopes: DeleteFileEnvelope,
+    delete_file_envelopes: List[DeleteFileEnvelope],
 ) -> tuple[List[MergeResult], float]:
     merge_options_provider = functools.partial(
         task_resource_options_provider,

deltacat/compute/compactor_v2/steps/merge.py

@@ -7,7 +7,6 @@ import ray
 import itertools
 import time
 import pyarrow.compute as pc
-from deltacat.utils.pyarrow import MAX_INT_BYTES
 import deltacat.compute.compactor_v2.utils.merge as merge_utils
 from uuid import uuid4
 from deltacat import logs
@@ -32,25 +31,21 @@ from deltacat.utils.resources import (
 )
 from deltacat.compute.compactor_v2.utils.primary_key_index import (
     generate_pk_hash_column,
-    pk_digest_to_hash_bucket_index,
 )
 from deltacat.storage import (
     Delta,
     DeltaLocator,
     DeltaType,
-    Manifest,
     Partition,
     interface as unimplemented_deltacat_storage,
 )
+from deltacat.storage.model.manifest import Manifest
 from deltacat.compute.compactor_v2.utils.dedupe import drop_duplicates
 from deltacat.constants import BYTES_PER_GIBIBYTE
 from deltacat.compute.compactor_v2.constants import (
     MERGE_TIME_IN_SECONDS,
     MERGE_SUCCESS_COUNT,
     MERGE_FAILURE_COUNT,
-    BUCKETING_SPEC_COMPLIANCE_PROFILE,
-    BUCKETING_SPEC_COMPLIANCE_ASSERT,
-    BUCKETING_SPEC_COMPLIANCE_PRINT_LOG,
 )
 from deltacat.exceptions import (
     categorize_errors,
@@ -152,32 +147,10 @@ def _merge_tables(
     if compacted_table:
         compacted_table = all_tables[0]
 
-        compacted_pk_hash_str = compacted_table[sc._PK_HASH_STRING_COLUMN_NAME]
-        incremental_pk_hash_str = incremental_table[sc._PK_HASH_STRING_COLUMN_NAME]
-
-        logger.info(
-            f"Size of compacted pk hash={compacted_pk_hash_str.nbytes} "
-            f"and incremental pk hash={incremental_pk_hash_str.nbytes}."
-        )
-
-        if (
-            compacted_table[sc._PK_HASH_STRING_COLUMN_NAME].nbytes >= MAX_INT_BYTES
-            or incremental_table[sc._PK_HASH_STRING_COLUMN_NAME].nbytes >= MAX_INT_BYTES
-        ):
-            logger.info("Casting compacted and incremental pk hash to large_string...")
-            # is_in combines the chunks of the chunked array passed which can cause
-            # ArrowCapacityError if the total size of string array is over 2GB.
-            # Using a large_string would resolve this issue.
-            # The cast here should be zero-copy in most cases.
-            compacted_pk_hash_str = pc.cast(compacted_pk_hash_str, pa.large_string())
-            incremental_pk_hash_str = pc.cast(
-                incremental_pk_hash_str, pa.large_string()
-            )
-
         records_to_keep = pc.invert(
             pc.is_in(
-                compacted_pk_hash_str,
-                incremental_pk_hash_str,
+                compacted_table[sc._PK_HASH_STRING_COLUMN_NAME],
+                incremental_table[sc._PK_HASH_STRING_COLUMN_NAME],
             )
         )
 
@@ -192,34 +165,9 @@ def _merge_tables(
     return final_table
 
 
-def _validate_bucketing_spec_compliance(
-    table: pa.Table, rcf: RoundCompletionInfo, hb_index: int, primary_keys: List[str]
-) -> None:
-    pki_table = generate_pk_hash_column(
-        [table], primary_keys=primary_keys, requires_hash=True
-    )[0]
-    for index, hash_value in enumerate(sc.pk_hash_string_column_np(pki_table)):
-        hash_bucket = pk_digest_to_hash_bucket_index(hash_value, rcf.hash_bucket_count)
-        if hash_bucket != hb_index:
-            logger.info(
-                f"{rcf.compacted_delta_locator.namespace}.{rcf.compacted_delta_locator.table_name}"
-                f".{rcf.compacted_delta_locator.table_version}.{rcf.compacted_delta_locator.partition_id}"
-                f".{rcf.compacted_delta_locator.partition_values} has non-compliant bucketing spec. "
-                f"Expected hash bucket is {hb_index} but found {hash_bucket}."
-            )
-            if BUCKETING_SPEC_COMPLIANCE_PROFILE == BUCKETING_SPEC_COMPLIANCE_ASSERT:
-                raise AssertionError(
-                    "Hash bucket drift detected. Expected hash bucket index"
-                    f" to be {hb_index} but found {hash_bucket}"
-                )
-            # No further checks necessary
-            break
-
-
 def _download_compacted_table(
     hb_index: int,
     rcf: RoundCompletionInfo,
-    primary_keys: List[str],
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
     deltacat_storage=unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[dict] = None,
@@ -243,23 +191,7 @@ def _download_compacted_table(
 
         tables.append(table)
 
-    compacted_table = pa.concat_tables(tables)
-    check_bucketing_spec = BUCKETING_SPEC_COMPLIANCE_PROFILE in [
-        BUCKETING_SPEC_COMPLIANCE_PRINT_LOG,
-        BUCKETING_SPEC_COMPLIANCE_ASSERT,
-    ]
-
-    logger.debug(
-        f"Value of BUCKETING_SPEC_COMPLIANCE_PROFILE, check_bucketing_spec:"
-        f" {BUCKETING_SPEC_COMPLIANCE_PROFILE}, {check_bucketing_spec}"
-    )
-
-    # Bucketing spec compliance isn't required without primary keys
-    if primary_keys and check_bucketing_spec:
-        _validate_bucketing_spec_compliance(
-            compacted_table, rcf, hb_index, primary_keys
-        )
-    return compacted_table
+    return pa.concat_tables(tables)
 
 
 def _copy_all_manifest_files_from_old_hash_buckets(
@@ -518,7 +450,9 @@ def _apply_upserts(
         # on non event based sort key does not produce consistent
         # compaction results. E.g., compaction(delta1, delta2, delta3)
         # will not be equal to compaction(compaction(delta1, delta2), delta3).
-        table = table.sort_by(input.sort_keys)
+        table = table.sort_by(
+            [pa_key for key in input.sort_keys for pa_key in key.arrow]
+        )
     hb_table_record_count = len(table) + (len(prev_table) if prev_table else 0)
     table, merge_time = timed_invocation(
         func=_merge_tables,
@@ -560,11 +494,9 @@ def _copy_manifests_from_hash_bucketing(
 def _timed_merge(input: MergeInput) -> MergeResult:
     task_id = get_current_ray_task_id()
     worker_id = get_current_ray_worker_id()
-    with (
-        memray.Tracker(f"merge_{worker_id}_{task_id}.bin")
-        if input.enable_profiler
-        else nullcontext()
-    ):
+    with memray.Tracker(
+        f"merge_{worker_id}_{task_id}.bin"
+    ) if input.enable_profiler else nullcontext():
         total_input_records, total_deduped_records = 0, 0
         total_dropped_records = 0
         materialized_results: List[MaterializeResult] = []
@@ -588,7 +520,6 @@ def _timed_merge(input: MergeInput) -> MergeResult:
                 compacted_table = _download_compacted_table(
                     hb_index=merge_file_group.hb_index,
                     rcf=input.round_completion_info,
-                    primary_keys=input.primary_keys,
                     read_kwargs_provider=input.read_kwargs_provider,
                     deltacat_storage=input.deltacat_storage,
                     deltacat_storage_kwargs=input.deltacat_storage_kwargs,
@@ -673,5 +604,5 @@ def merge(input: MergeInput) -> MergeResult:
             merge_result[3],
             merge_result[4],
             np.double(emit_metrics_time),
-            merge_result[6],
+            merge_result[4],
         )

deltacat/compute/compactor_v2/utils/content_type_params.py

@@ -5,7 +5,6 @@ from deltacat.compute.compactor_v2.constants import (
     TASK_MAX_PARALLELISM,
     MAX_PARQUET_METADATA_SIZE,
 )
-from deltacat.utils.common import ReadKwargsProvider
 from deltacat.utils.ray_utils.concurrency import invoke_parallel
 from deltacat import logs
 from deltacat.storage import (
@@ -76,21 +75,11 @@ def _download_parquet_metadata_for_manifest_entry(
     entry_index: int,
     deltacat_storage: unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[Dict[Any, Any]] = {},
-    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
 ) -> Dict[str, Any]:
-    logger.info(
-        f"Downloading the parquet metadata for Delta with locator {delta.locator} and entry_index: {entry_index}"
-    )
-    if "file_reader_kwargs_provider" in deltacat_storage_kwargs:
-        logger.info(
-            "'file_reader_kwargs_provider' is also present in deltacat_storage_kwargs. Removing to prevent multiple values for keyword argument"
-        )
-        deltacat_storage_kwargs.pop("file_reader_kwargs_provider")
     pq_file = deltacat_storage.download_delta_manifest_entry(
         delta,
         entry_index=entry_index,
         table_type=TableType.PYARROW_PARQUET,
-        file_reader_kwargs_provider=file_reader_kwargs_provider,
         **deltacat_storage_kwargs,
     )
 
@@ -108,15 +97,11 @@ def append_content_type_params(
     max_parquet_meta_size_bytes: Optional[int] = MAX_PARQUET_METADATA_SIZE,
     deltacat_storage=unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
-    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
 ) -> bool:
     """
     This operation appends content type params into the delta entry. Note
     that this operation can be time consuming, hence we cache it in a Ray actor.
     """
-    logger.info(
-        f"Appending the content type params for Delta with locator {delta.locator}..."
-    )
 
     if not delta.meta:
         logger.warning(f"Delta with locator {delta.locator} doesn't contain meta.")
@@ -174,7 +159,6 @@ def append_content_type_params(
 
     def input_provider(index, item) -> Dict:
         return {
-            "file_reader_kwargs_provider": file_reader_kwargs_provider,
             "deltacat_storage_kwargs": deltacat_storage_kwargs,
             "deltacat_storage": deltacat_storage,
             "delta": delta,
@@ -184,7 +168,6 @@ def append_content_type_params(
     logger.info(
         f"Downloading parquet meta for {len(entry_indices_to_download)} manifest entries..."
     )
-
     pq_files_promise = invoke_parallel(
         entry_indices_to_download,
         ray_task=_download_parquet_metadata_for_manifest_entry,

deltacat/compute/compactor_v2/utils/dedupe.py

@@ -25,7 +25,7 @@ def _create_chunked_index_array(array: pa.Array) -> pa.Array:
         result[index] = np.arange(cl, dtype="int32")
 
     chunk_lengths = ([0] + chunk_lengths)[:-1]
-    result = pa.chunked_array(result + np.cumsum(chunk_lengths), type=pa.int32())
+    result = pa.chunked_array(result + np.cumsum(chunk_lengths))
     return result
 
 

deltacat/compute/compactor_v2/utils/io.py

@@ -101,6 +101,7 @@ def create_uniform_input_deltas(
     delta_manifest_entries_count = 0
     estimated_da_bytes = 0
     input_da_list = []
+
     for delta in input_deltas:
         if (
             compact_partition_params.enable_input_split
@@ -117,7 +118,6 @@ def create_uniform_input_deltas(
                 deltacat_storage_kwargs=deltacat_storage_kwargs,
                 task_max_parallelism=compact_partition_params.task_max_parallelism,
                 max_parquet_meta_size_bytes=compact_partition_params.max_parquet_meta_size_bytes,
-                file_reader_kwargs_provider=compact_partition_params.read_kwargs_provider,
             )
 
         manifest_entries = delta.manifest.entries

deltacat/compute/compactor_v2/utils/primary_key_index.py

@@ -10,7 +10,6 @@ from deltacat.compute.compactor_v2.constants import (
     TOTAL_BYTES_IN_SHA1_HASH,
     PK_DELIMITER,
     MAX_SIZE_OF_RECORD_BATCH_IN_GIB,
-    SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED,
 )
 import time
 from deltacat.compute.compactor.model.delta_file_envelope import DeltaFileEnvelope
@@ -49,13 +48,6 @@ def _is_sha1_desired(hash_columns: List[pa.Array]) -> bool:
         f"Found total length of hash column={total_len} and total_size={total_size}"
     )
 
-    if SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED:
-        logger.info(
-            f"SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED is True. "
-            f"Returning False for is_sha1_desired"
-        )
-        return False
-
     return total_size > TOTAL_BYTES_IN_SHA1_HASH * total_len
 
 
@@ -116,10 +108,9 @@ def _optimized_group_record_batches_by_hash_bucket(
     record_batches = []
     result_len = 0
     for record_batch in table_batches:
-        if (
-            record_batches
-            and current_bytes + record_batch.nbytes >= MAX_SIZE_OF_RECORD_BATCH_IN_GIB
-        ):
+        current_bytes += record_batch.nbytes
+        record_batches.append(record_batch)
+        if current_bytes >= MAX_SIZE_OF_RECORD_BATCH_IN_GIB:
             logger.info(
                 f"Total number of record batches without exceeding {MAX_SIZE_OF_RECORD_BATCH_IN_GIB} "
                 f"is {len(record_batches)} and size {current_bytes}"
@@ -137,9 +128,6 @@ def _optimized_group_record_batches_by_hash_bucket(
             current_bytes = 0
             record_batches.clear()
 
-        current_bytes += record_batch.nbytes
-        record_batches.append(record_batch)
-
     if record_batches:
         appended_len, append_latency = timed_invocation(
             _append_table_by_hash_bucket,

deltacat/compute/compactor_v2/utils/task_options.py

@@ -1,16 +1,11 @@
 import logging
 from typing import Dict, Optional, List, Tuple, Any
 from deltacat import logs
-from deltacat.constants import PYARROW_INFLATION_MULTIPLIER
-from deltacat.compute.compactor_v2.constants import (
-    AVERAGE_RECORD_SIZE_BYTES as DEFAULT_AVERAGE_RECORD_SIZE_BYTES,
-)
 from deltacat.compute.compactor_v2.model.merge_file_group import (
     LocalMergeFileGroupsProvider,
 )
 from deltacat.storage import (
     Manifest,
-    ManifestEntry,
     interface as unimplemented_deltacat_storage,
 )
 from deltacat.compute.compactor.model.delta_annotated import DeltaAnnotated
@@ -86,27 +81,16 @@ def _get_merge_task_options(
         and compacted_delta_manifest
         and round_completion_info.hb_index_to_entry_range
     ):
-        logger.debug_conditional(
-            f"[Merge task {index}]: Using previous compaction rounds to calculate merge memory: {round_completion_info.compacted_pyarrow_write_result}",
-            memory_logs_enabled,
-        )
-        previous_inflation: float = (
-            (
-                round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
-                / round_completion_info.compacted_pyarrow_write_result.file_bytes
-            )
-            if round_completion_info.compacted_pyarrow_write_result.file_bytes
-            else PYARROW_INFLATION_MULTIPLIER
+
+        previous_inflation = (
+            round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
+            / round_completion_info.compacted_pyarrow_write_result.file_bytes
         )
         debug_memory_params["previous_inflation"] = previous_inflation
 
-        average_record_size: float = (
-            (
-                round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
-                / round_completion_info.compacted_pyarrow_write_result.records
-            )
-            if round_completion_info.compacted_pyarrow_write_result.records
-            else DEFAULT_AVERAGE_RECORD_SIZE_BYTES
+        average_record_size = (
+            round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
+            / round_completion_info.compacted_pyarrow_write_result.records
         )
         debug_memory_params["average_record_size"] = average_record_size
 
@@ -122,36 +106,31 @@ def _get_merge_task_options(
                 str(hb_idx)
             ]
             for entry_index in range(entry_start, entry_end):
-                entry: ManifestEntry = compacted_delta_manifest.entries[entry_index]
-                current_entry_size: float = (
-                    estimate_manifest_entry_size_bytes(
-                        entry=entry,
-                        operation_type=OperationType.PYARROW_DOWNLOAD,
-                        estimate_resources_params=estimate_resources_params,
-                    )
-                    or 0.0
+                entry = compacted_delta_manifest.entries[entry_index]
+
+                current_entry_size = estimate_manifest_entry_size_bytes(
+                    entry=entry,
+                    operation_type=OperationType.PYARROW_DOWNLOAD,
+                    estimate_resources_params=estimate_resources_params,
                 )
-                current_entry_rows: int = (
-                    estimate_manifest_entry_num_rows(
-                        entry=entry,
-                        operation_type=OperationType.PYARROW_DOWNLOAD,
-                        estimate_resources_params=estimate_resources_params,
-                    )
-                    or 0
+                current_entry_rows = estimate_manifest_entry_num_rows(
+                    entry=entry,
+                    operation_type=OperationType.PYARROW_DOWNLOAD,
+                    estimate_resources_params=estimate_resources_params,
                 )
-                # NOTE: We can treat the current_entry_size and current_entry_rows as 0 as a None estimated entry size implies a 0 value
+
                 data_size += current_entry_size
                 num_rows += current_entry_rows
+
                 if primary_keys:
-                    pk_size: Optional[
-                        float
-                    ] = estimate_manifest_entry_column_size_bytes(
+                    pk_size = estimate_manifest_entry_column_size_bytes(
                         entry=entry,
                         columns=primary_keys,
                         operation_type=OperationType.PYARROW_DOWNLOAD,
                         estimate_resources_params=estimate_resources_params,
                     )
-                    if not pk_size:
+
+                    if pk_size is None:
                         pk_size_bytes += current_entry_size
                     else:
                         pk_size_bytes += pk_size
@@ -180,6 +159,7 @@ def _get_merge_task_options(
         f"[Merge task {index}]: Params used for calculating merge memory: {debug_memory_params}",
         memory_logs_enabled,
     )
+
     return _get_task_options(0.01, total_memory, ray_custom_resources)
 
 

deltacat/compute/converter/constants.py

@@ -0,0 +1,4 @@
+DEFAULT_CONVERTER_TASK_MAX_PARALLELISM = 4096
+
+# Safe limit ONLY considering CPU limit, typically 32 for a 8x-large worker
+DEFAULT_MAX_PARALLEL_DATA_FILE_DOWNLOAD = 30