deltacat 1.1.36__py3-none-any.whl → 2.0__py3-none-any.whl

deltacat/compute/compactor/steps/hash_bucket.py

@@ -201,7 +201,7 @@ def _timed_hash_bucket(
     with memray.Tracker(
         f"hash_bucket_{worker_id}_{task_id}.bin"
     ) if enable_profiler else nullcontext():
-        sort_key_names = [key.key_name for key in sort_keys]
+        sort_key_names = list(chain.from_iterable([key.key for key in sort_keys]))
         if not round_completion_info:
             is_src_delta = True
         else:

deltacat/compute/compactor/steps/materialize.py

@@ -25,9 +25,10 @@ from deltacat.storage import (
     DeltaType,
     Partition,
     PartitionLocator,
-    Manifest,
     ManifestEntry,
+    ManifestEntryList,
 )
+from deltacat.storage.model.manifest import Manifest
 from deltacat.storage import interface as unimplemented_deltacat_storage
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.types.media import DELIMITED_TEXT_CONTENT_TYPES, ContentType
@@ -82,7 +83,10 @@ def materialize(
         assert (
             delta_type == DeltaType.UPSERT
         ), "Stage delta with existing manifest entries only supports UPSERT delta type!"
-        manifest = Manifest.of(entries=manifest_entry_list_reference, uuid=str(uuid4()))
+        manifest = Manifest.of(
+            entries=ManifestEntryList.of(manifest_entry_list_reference),
+            uuid=str(uuid4()),
+        )
         delta = Delta.of(
             locator=DeltaLocator.of(partition.locator),
             delta_type=delta_type,

deltacat/compute/compactor/utils/io.py

@@ -358,7 +358,7 @@ def fit_input_deltas(
 def _discover_deltas(
     source_partition_locator: PartitionLocator,
     start_position_exclusive: Optional[int],
-    end_position_inclusive: int,
+    end_position_inclusive: Optional[int],
     deltacat_storage=unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
     list_deltas_kwargs: Optional[Dict[str, Any]] = {},

deltacat/compute/compactor/utils/sort_key.py

@@ -1,6 +1,7 @@
 import pyarrow as pa
 from typing import List
-from deltacat.storage import PartitionLocator, SortKey
+from itertools import chain
+from deltacat.storage import PartitionLocator, SortKey, TransformName
 
 MAX_SORT_KEYS_BIT_WIDTH = 256
 
@@ -22,7 +23,13 @@ def validate_sort_keys(
         deltacat_storage_kwargs = {}
     total_sort_keys_bit_width = 0
     if sort_keys:
-        sort_key_names = [key.key_name for key in sort_keys]
+        sort_key_names = list(chain.from_iterable([key.key for key in sort_keys]))
+        assert all(
+            [
+                key.transform is None or key.transform.name == TransformName.IDENTITY
+                for key in sort_keys
+            ]
+        ), f"Sort key transforms are not supported: {sort_keys}"
         assert len(sort_key_names) == len(
             set(sort_key_names)
         ), f"Sort key names must be unique: {sort_key_names}"

deltacat/compute/compactor_v2/compaction_session.py

@@ -27,9 +27,8 @@ from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
 from deltacat.storage import (
     Delta,
     DeltaLocator,
-    Manifest,
-    Partition,
 )
+from deltacat.storage.model.manifest import Manifest
 from deltacat.compute.compactor.model.compact_partition_params import (
     CompactPartitionParams,
 )
@@ -69,17 +68,14 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
     assert (
         params.hash_bucket_count is not None and params.hash_bucket_count >= 1
     ), "hash_bucket_count is a required arg for compactor v2"
-    assert type(params.hash_bucket_count) is int, "Hash bucket count must be an integer"
     if params.num_rounds > 1:
         assert (
             not params.drop_duplicates
         ), "num_rounds > 1, drop_duplicates must be False but is True"
 
-    with (
-        memray.Tracker("compaction_partition.bin")
-        if params.enable_profiler
-        else nullcontext()
-    ):
+    with memray.Tracker(
+        "compaction_partition.bin"
+    ) if params.enable_profiler else nullcontext():
         execute_compaction_result: ExecutionCompactionResult = _execute_compaction(
             params,
             **kwargs,
@@ -142,7 +138,7 @@ def _execute_compaction(
         logger.info("No input deltas found to compact.")
         return ExecutionCompactionResult(None, None, None, False)
     build_uniform_deltas_result: tuple[
-        List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope], Partition
+        List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope]
     ] = _build_uniform_deltas(
         params, compaction_audit, input_deltas, delta_discovery_start
     )

deltacat/compute/compactor_v2/constants.py

@@ -1,5 +1,3 @@
-from deltacat.utils.common import env_bool, env_integer, env_string
-
 TOTAL_BYTES_IN_SHA1_HASH = 20
 
 PK_DELIMITER = "L6kl7u5f"
@@ -33,9 +31,7 @@ TOTAL_MEMORY_BUFFER_PERCENTAGE = 30
 # The total size of records that will be hash bucketed at once
 # Since, sorting is nlogn, we ensure that is not performed
 # on a very large dataset for best performance.
-MAX_SIZE_OF_RECORD_BATCH_IN_GIB = env_integer(
-    "MAX_SIZE_OF_RECORD_BATCH_IN_GIB", 2 * 1024 * 1024 * 1024
-)
+MAX_SIZE_OF_RECORD_BATCH_IN_GIB = 2 * 1024 * 1024 * 1024
 
 # Whether to drop duplicates during merge.
 DROP_DUPLICATES = True
@@ -82,28 +78,3 @@ COMPACT_PARTITION_METRIC_PREFIX = "compact_partition"
 # Number of rounds to run hash/merge for a single
 # partition. (For large table support)
 DEFAULT_NUM_ROUNDS = 1
-
-# Whether to perform sha1 hashing when required to
-# optimize memory. For example, hashing is always
-# required for bucketing where it's not mandatory
-# when dropping duplicates. Setting this to True
-# will disable sha1 hashing in cases where it isn't
-# mandatory. This flag is False by default.
-SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED = env_bool(
-    "SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED", False
-)
-
-# This env variable specifies whether to check bucketing spec
-# compliance of the existing compacted table.
-# PRINT_LOG: Enable logging if any partition is found
-# to be non-compliant with the bucketing spec.
-# ASSERT: Fail the job with ValidationError if the
-# current compacted partition is found to be non-compliant
-# with bucketing spec. Note, logging is implicitly enabled
-# in this case.
-BUCKETING_SPEC_COMPLIANCE_PROFILE = env_string(
-    "BUCKETING_SPEC_COMPLIANCE_PROFILE", None
-)
-
-BUCKETING_SPEC_COMPLIANCE_PRINT_LOG = "PRINT_LOG"
-BUCKETING_SPEC_COMPLIANCE_ASSERT = "ASSERT"

deltacat/compute/compactor_v2/deletes/utils.py

@@ -49,7 +49,7 @@ def _aggregate_delete_deltas(input_deltas: List[Delta]) -> Dict[int, List[Delta]
     ] = [
         (is_delete, list(delete_delta_group))
         for (is_delete, _), delete_delta_group in itertools.groupby(
-            input_deltas, lambda d: (d.type is DeltaType.DELETE, d.delete_parameters)
+            input_deltas, lambda d: (d.type is DeltaType.DELETE, d.meta.entry_params)
         )
     ]
     for (
@@ -89,11 +89,11 @@ def _get_delete_file_envelopes(
         consecutive_delete_tables: List[pa.Table] = []
         for delete_delta in delete_delta_sequence:
             assert (
-                delete_delta.delete_parameters is not None
+                delete_delta.meta.entry_params is not None
             ), "Delete type deltas are required to have delete parameters defined"
             delete_columns: Optional[
                 List[str]
-            ] = delete_delta.delete_parameters.equality_column_names
+            ] = delete_delta.meta.entry_params.equality_field_locators
             assert len(delete_columns) > 0, "At least 1 delete column is required"
             # delete columns should exist in underlying table
             delete_dataset = params.deltacat_storage.download_delta(

deltacat/compute/compactor_v2/model/merge_input.py

@@ -43,12 +43,11 @@ class MergeInput(Dict):
         round_completion_info: Optional[RoundCompletionInfo] = None,
         object_store: Optional[IObjectStore] = None,
         delete_strategy: Optional[DeleteStrategy] = None,
-        delete_file_envelopes: Optional[List] = None,
+        delete_file_envelopes: Optional[List[DeleteFileEnvelope]] = None,
         deltacat_storage=unimplemented_deltacat_storage,
         deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
         memory_logs_enabled: Optional[bool] = None,
         disable_copy_by_reference: Optional[bool] = None,
-        hash_bucket_count: Optional[int] = None,
     ) -> MergeInput:
 
         result = MergeInput()
@@ -72,7 +71,6 @@ class MergeInput(Dict):
         result["deltacat_storage_kwargs"] = deltacat_storage_kwargs or {}
         result["memory_logs_enabled"] = memory_logs_enabled
         result["disable_copy_by_reference"] = disable_copy_by_reference
-        result["hash_bucket_count"] = hash_bucket_count
         return result
 
     @property
@@ -156,7 +154,3 @@ class MergeInput(Dict):
     @property
     def disable_copy_by_reference(self) -> bool:
         return self["disable_copy_by_reference"]
-
-    @property
-    def hash_bucket_count(self) -> int:
-        return self["hash_bucket_count"]

deltacat/compute/compactor_v2/private/compaction_utils.py

@@ -63,7 +63,7 @@ from deltacat.compute.compactor_v2.steps import merge as mg
 from deltacat.compute.compactor_v2.steps import hash_bucket as hb
 from deltacat.compute.compactor_v2.utils import io
 
-from typing import List, Optional
+from typing import List, Optional, Union
 from collections import defaultdict
 from deltacat.compute.compactor.model.compaction_session_audit_info import (
     CompactionSessionAuditInfo,
@@ -83,7 +83,7 @@ def _fetch_compaction_metadata(
 
     # read the results from any previously completed compaction round
     round_completion_info: Optional[RoundCompletionInfo] = None
-    high_watermark: Optional[HighWatermark] = None
+    high_watermark: Optional[Union[HighWatermark, int]] = None
     previous_compacted_delta_manifest: Optional[Manifest] = None
 
     if not params.rebase_source_partition_locator:
@@ -129,7 +129,7 @@ def _build_uniform_deltas(
     mutable_compaction_audit: CompactionSessionAuditInfo,
     input_deltas: List[Delta],
     delta_discovery_start: float,
-) -> tuple[List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope], Partition]:
+) -> tuple[List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope]]:
 
     delete_strategy: Optional[DeleteStrategy] = None
     delete_file_envelopes: Optional[List[DeleteFileEnvelope]] = None
@@ -222,7 +222,7 @@ def _run_hash_and_merge(
     uniform_deltas: List[DeltaAnnotated],
     round_completion_info: RoundCompletionInfo,
     delete_strategy: Optional[DeleteStrategy],
-    delete_file_envelopes: Optional[DeleteFileEnvelope],
+    delete_file_envelopes: Optional[List[DeleteFileEnvelope]],
     mutable_compaction_audit: CompactionSessionAuditInfo,
     previous_compacted_delta_manifest: Optional[Manifest],
     compacted_partition: Partition,
@@ -389,7 +389,7 @@ def _merge(
     all_hash_group_idx_to_obj_id: dict,
     compacted_partition: Partition,
     delete_strategy: DeleteStrategy,
-    delete_file_envelopes: DeleteFileEnvelope,
+    delete_file_envelopes: List[DeleteFileEnvelope],
 ) -> tuple[List[MergeResult], float]:
     merge_options_provider = functools.partial(
         task_resource_options_provider,
@@ -438,7 +438,6 @@ def _merge(
                 delete_file_envelopes=delete_file_envelopes,
                 memory_logs_enabled=params.memory_logs_enabled,
                 disable_copy_by_reference=params.disable_copy_by_reference,
-                hash_bucket_count=params.hash_bucket_count,
             )
         }
 

deltacat/compute/compactor_v2/steps/merge.py

@@ -7,7 +7,6 @@ import ray
 import itertools
 import time
 import pyarrow.compute as pc
-from deltacat.utils.pyarrow import MAX_INT_BYTES
 import deltacat.compute.compactor_v2.utils.merge as merge_utils
 from uuid import uuid4
 from deltacat import logs
@@ -32,25 +31,21 @@ from deltacat.utils.resources import (
 )
 from deltacat.compute.compactor_v2.utils.primary_key_index import (
     generate_pk_hash_column,
-    pk_digest_to_hash_bucket_index,
 )
 from deltacat.storage import (
     Delta,
     DeltaLocator,
     DeltaType,
-    Manifest,
     Partition,
     interface as unimplemented_deltacat_storage,
 )
+from deltacat.storage.model.manifest import Manifest
 from deltacat.compute.compactor_v2.utils.dedupe import drop_duplicates
 from deltacat.constants import BYTES_PER_GIBIBYTE
 from deltacat.compute.compactor_v2.constants import (
     MERGE_TIME_IN_SECONDS,
     MERGE_SUCCESS_COUNT,
     MERGE_FAILURE_COUNT,
-    BUCKETING_SPEC_COMPLIANCE_PROFILE,
-    BUCKETING_SPEC_COMPLIANCE_ASSERT,
-    BUCKETING_SPEC_COMPLIANCE_PRINT_LOG,
 )
 from deltacat.exceptions import (
     categorize_errors,
@@ -62,10 +57,6 @@ if importlib.util.find_spec("memray"):
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
-_EXISTING_VARIANT_LOG_PREFIX = "Existing variant "
-_INCREMENTAL_TABLE_LOG_PREFIX = "Incremental table "
-
-
 def _append_delta_type_column(table: pa.Table, value: np.bool_):
     return table.append_column(
         sc._DELTA_TYPE_COLUMN_FIELD,
@@ -116,8 +107,6 @@ def _merge_tables(
     table: pa.Table,
     primary_keys: List[str],
     can_drop_duplicates: bool,
-    hb_index: int,
-    num_buckets: int,
     compacted_table: Optional[pa.Table] = None,
 ) -> pa.Table:
     """
@@ -136,20 +125,6 @@ def _merge_tables(
 
     all_tables.append(table)
 
-    check_bucketing_spec = BUCKETING_SPEC_COMPLIANCE_PROFILE in [
-        BUCKETING_SPEC_COMPLIANCE_PRINT_LOG,
-        BUCKETING_SPEC_COMPLIANCE_ASSERT,
-    ]
-
-    if primary_keys and check_bucketing_spec:
-        _validate_bucketing_spec_compliance(
-            table=all_tables[incremental_idx],
-            num_buckets=num_buckets,
-            primary_keys=primary_keys,
-            hb_index=hb_index,
-            log_prefix=_INCREMENTAL_TABLE_LOG_PREFIX,
-        )
-
     if not primary_keys or not can_drop_duplicates:
         logger.info(
             f"Not dropping duplicates for primary keys={primary_keys} "
@@ -172,32 +147,10 @@ def _merge_tables(
     if compacted_table:
         compacted_table = all_tables[0]
 
-        compacted_pk_hash_str = compacted_table[sc._PK_HASH_STRING_COLUMN_NAME]
-        incremental_pk_hash_str = incremental_table[sc._PK_HASH_STRING_COLUMN_NAME]
-
-        logger.info(
-            f"Size of compacted pk hash={compacted_pk_hash_str.nbytes} "
-            f"and incremental pk hash={incremental_pk_hash_str.nbytes}."
-        )
-
-        if (
-            compacted_table[sc._PK_HASH_STRING_COLUMN_NAME].nbytes >= MAX_INT_BYTES
-            or incremental_table[sc._PK_HASH_STRING_COLUMN_NAME].nbytes >= MAX_INT_BYTES
-        ):
-            logger.info("Casting compacted and incremental pk hash to large_string...")
-            # is_in combines the chunks of the chunked array passed which can cause
-            # ArrowCapacityError if the total size of string array is over 2GB.
-            # Using a large_string would resolve this issue.
-            # The cast here should be zero-copy in most cases.
-            compacted_pk_hash_str = pc.cast(compacted_pk_hash_str, pa.large_string())
-            incremental_pk_hash_str = pc.cast(
-                incremental_pk_hash_str, pa.large_string()
-            )
-
         records_to_keep = pc.invert(
             pc.is_in(
-                compacted_pk_hash_str,
-                incremental_pk_hash_str,
+                compacted_table[sc._PK_HASH_STRING_COLUMN_NAME],
+                incremental_table[sc._PK_HASH_STRING_COLUMN_NAME],
             )
         )
 
@@ -212,47 +165,9 @@ def _merge_tables(
     return final_table
 
 
-def _validate_bucketing_spec_compliance(
-    table: pa.Table,
-    num_buckets: int,
-    hb_index: int,
-    primary_keys: List[str],
-    rcf: RoundCompletionInfo = None,
-    log_prefix=None,
-) -> None:
-    if rcf is not None:
-        message_prefix = f"{log_prefix}{rcf.compacted_delta_locator.namespace}.{rcf.compacted_delta_locator.table_name}.{rcf.compacted_delta_locator.table_version}.{rcf.compacted_delta_locator.partition_id}.{rcf.compacted_delta_locator.partition_values}"
-    else:
-        message_prefix = f"{log_prefix}"
-    pki_table = generate_pk_hash_column(
-        [table], primary_keys=primary_keys, requires_hash=True
-    )[0]
-    is_not_compliant: bool = False
-    for index, hash_value in enumerate(sc.pk_hash_string_column_np(pki_table)):
-        hash_bucket: int = pk_digest_to_hash_bucket_index(hash_value, num_buckets)
-        if hash_bucket != hb_index:
-            is_not_compliant = True
-            logger.info(
-                f"{message_prefix} has non-compliant bucketing spec at index: {index} "
-                f"Expected hash bucket is {hb_index} but found {hash_bucket}."
-            )
-            if BUCKETING_SPEC_COMPLIANCE_PROFILE == BUCKETING_SPEC_COMPLIANCE_ASSERT:
-                raise AssertionError(
-                    f"Hash bucket drift detected at index: {index}. Expected hash bucket index"
-                    f" to be {hb_index} but found {hash_bucket}"
-                )
-            # No further checks necessary
-            break
-    if not is_not_compliant:
-        logger.debug(
-            f"{message_prefix} has compliant bucketing spec for hb_index: {hb_index}"
-        )
-
-
 def _download_compacted_table(
     hb_index: int,
     rcf: RoundCompletionInfo,
-    primary_keys: List[str],
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
     deltacat_storage=unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[dict] = None,
@@ -276,28 +191,7 @@ def _download_compacted_table(
 
         tables.append(table)
 
-    compacted_table = pa.concat_tables(tables)
-    check_bucketing_spec = BUCKETING_SPEC_COMPLIANCE_PROFILE in [
-        BUCKETING_SPEC_COMPLIANCE_PRINT_LOG,
-        BUCKETING_SPEC_COMPLIANCE_ASSERT,
-    ]
-
-    logger.debug(
-        f"Value of BUCKETING_SPEC_COMPLIANCE_PROFILE, check_bucketing_spec:"
-        f" {BUCKETING_SPEC_COMPLIANCE_PROFILE}, {check_bucketing_spec}"
-    )
-
-    # Bucketing spec compliance isn't required without primary keys
-    if primary_keys and check_bucketing_spec:
-        _validate_bucketing_spec_compliance(
-            compacted_table,
-            rcf.hash_bucket_count,
-            hb_index,
-            primary_keys,
-            rcf=rcf,
-            log_prefix=_EXISTING_VARIANT_LOG_PREFIX,
-        )
-    return compacted_table
+    return pa.concat_tables(tables)
 
 
 def _copy_all_manifest_files_from_old_hash_buckets(
@@ -500,12 +394,12 @@ def _compact_tables(
         _group_sequence_by_delta_type(reordered_all_dfes)
     ):
         if delta_type is DeltaType.UPSERT:
-            (table, incremental_len, deduped_records, merge_time,) = _apply_upserts(
-                input=input,
-                dfe_list=delta_type_sequence,
-                hb_idx=hb_idx,
-                prev_table=table,
-            )
+            (
+                table,
+                incremental_len,
+                deduped_records,
+                merge_time,
+            ) = _apply_upserts(input, delta_type_sequence, hb_idx, table)
             logger.info(
                 f" [Merge task index {input.merge_task_index}] Merged"
                 f" record count: {len(table)}, size={table.nbytes} took: {merge_time}s"
@@ -556,7 +450,9 @@ def _apply_upserts(
         # on non event based sort key does not produce consistent
         # compaction results. E.g., compaction(delta1, delta2, delta3)
         # will not be equal to compaction(compaction(delta1, delta2), delta3).
-        table = table.sort_by(input.sort_keys)
+        table = table.sort_by(
+            [pa_key for key in input.sort_keys for pa_key in key.arrow]
+        )
     hb_table_record_count = len(table) + (len(prev_table) if prev_table else 0)
     table, merge_time = timed_invocation(
         func=_merge_tables,
@@ -564,8 +460,6 @@ def _apply_upserts(
         primary_keys=input.primary_keys,
         can_drop_duplicates=input.drop_duplicates,
         compacted_table=prev_table,
-        hb_index=hb_idx,
-        num_buckets=input.hash_bucket_count,
     )
     deduped_records = hb_table_record_count - len(table)
     return table, incremental_len, deduped_records, merge_time
@@ -600,11 +494,9 @@ def _copy_manifests_from_hash_bucketing(
 def _timed_merge(input: MergeInput) -> MergeResult:
     task_id = get_current_ray_task_id()
     worker_id = get_current_ray_worker_id()
-    with (
-        memray.Tracker(f"merge_{worker_id}_{task_id}.bin")
-        if input.enable_profiler
-        else nullcontext()
-    ):
+    with memray.Tracker(
+        f"merge_{worker_id}_{task_id}.bin"
+    ) if input.enable_profiler else nullcontext():
         total_input_records, total_deduped_records = 0, 0
         total_dropped_records = 0
         materialized_results: List[MaterializeResult] = []
@@ -628,7 +520,6 @@ def _timed_merge(input: MergeInput) -> MergeResult:
                 compacted_table = _download_compacted_table(
                     hb_index=merge_file_group.hb_index,
                     rcf=input.round_completion_info,
-                    primary_keys=input.primary_keys,
                     read_kwargs_provider=input.read_kwargs_provider,
                     deltacat_storage=input.deltacat_storage,
                     deltacat_storage_kwargs=input.deltacat_storage_kwargs,
@@ -713,5 +604,5 @@ def merge(input: MergeInput) -> MergeResult:
             merge_result[3],
             merge_result[4],
             np.double(emit_metrics_time),
-            merge_result[6],
+            merge_result[4],
         )

deltacat/compute/compactor_v2/utils/content_type_params.py

@@ -5,7 +5,6 @@ from deltacat.compute.compactor_v2.constants import (
     TASK_MAX_PARALLELISM,
     MAX_PARQUET_METADATA_SIZE,
 )
-from deltacat.utils.common import ReadKwargsProvider
 from deltacat.utils.ray_utils.concurrency import invoke_parallel
 from deltacat import logs
 from deltacat.storage import (
@@ -76,21 +75,11 @@ def _download_parquet_metadata_for_manifest_entry(
     entry_index: int,
     deltacat_storage: unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[Dict[Any, Any]] = {},
-    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
 ) -> Dict[str, Any]:
-    logger.info(
-        f"Downloading the parquet metadata for Delta with locator {delta.locator} and entry_index: {entry_index}"
-    )
-    if "file_reader_kwargs_provider" in deltacat_storage_kwargs:
-        logger.info(
-            "'file_reader_kwargs_provider' is also present in deltacat_storage_kwargs. Removing to prevent multiple values for keyword argument"
-        )
-        deltacat_storage_kwargs.pop("file_reader_kwargs_provider")
     pq_file = deltacat_storage.download_delta_manifest_entry(
         delta,
         entry_index=entry_index,
         table_type=TableType.PYARROW_PARQUET,
-        file_reader_kwargs_provider=file_reader_kwargs_provider,
         **deltacat_storage_kwargs,
     )
 
@@ -108,15 +97,11 @@ def append_content_type_params(
     max_parquet_meta_size_bytes: Optional[int] = MAX_PARQUET_METADATA_SIZE,
     deltacat_storage=unimplemented_deltacat_storage,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
-    file_reader_kwargs_provider: Optional[ReadKwargsProvider] = None,
 ) -> bool:
     """
     This operation appends content type params into the delta entry. Note
     that this operation can be time consuming, hence we cache it in a Ray actor.
     """
-    logger.info(
-        f"Appending the content type params for Delta with locator {delta.locator}..."
-    )
 
     if not delta.meta:
         logger.warning(f"Delta with locator {delta.locator} doesn't contain meta.")
@@ -174,7 +159,6 @@ def append_content_type_params(
 
     def input_provider(index, item) -> Dict:
         return {
-            "file_reader_kwargs_provider": file_reader_kwargs_provider,
             "deltacat_storage_kwargs": deltacat_storage_kwargs,
             "deltacat_storage": deltacat_storage,
             "delta": delta,
@@ -184,7 +168,6 @@ def append_content_type_params(
     logger.info(
         f"Downloading parquet meta for {len(entry_indices_to_download)} manifest entries..."
     )
-
     pq_files_promise = invoke_parallel(
         entry_indices_to_download,
         ray_task=_download_parquet_metadata_for_manifest_entry,

deltacat/compute/compactor_v2/utils/dedupe.py

@@ -25,7 +25,7 @@ def _create_chunked_index_array(array: pa.Array) -> pa.Array:
         result[index] = np.arange(cl, dtype="int32")
 
     chunk_lengths = ([0] + chunk_lengths)[:-1]
-    result = pa.chunked_array(result + np.cumsum(chunk_lengths), type=pa.int32())
+    result = pa.chunked_array(result + np.cumsum(chunk_lengths))
     return result
 
 

deltacat/compute/compactor_v2/utils/io.py

@@ -101,6 +101,7 @@ def create_uniform_input_deltas(
     delta_manifest_entries_count = 0
     estimated_da_bytes = 0
     input_da_list = []
+
     for delta in input_deltas:
         if (
             compact_partition_params.enable_input_split
@@ -117,7 +118,6 @@ def create_uniform_input_deltas(
                 deltacat_storage_kwargs=deltacat_storage_kwargs,
                 task_max_parallelism=compact_partition_params.task_max_parallelism,
                 max_parquet_meta_size_bytes=compact_partition_params.max_parquet_meta_size_bytes,
-                file_reader_kwargs_provider=compact_partition_params.read_kwargs_provider,
             )
 
         manifest_entries = delta.manifest.entries

deltacat/compute/compactor_v2/utils/merge.py

@@ -133,5 +133,4 @@ def generate_local_merge_input(
         delete_strategy=delete_strategy,
         delete_file_envelopes=delete_file_envelopes,
         disable_copy_by_reference=params.disable_copy_by_reference,
-        hash_bucket_count=params.hash_bucket_count,
     )

deltacat/compute/compactor_v2/utils/primary_key_index.py

@@ -10,7 +10,6 @@ from deltacat.compute.compactor_v2.constants import (
     TOTAL_BYTES_IN_SHA1_HASH,
     PK_DELIMITER,
     MAX_SIZE_OF_RECORD_BATCH_IN_GIB,
-    SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED,
 )
 import time
 from deltacat.compute.compactor.model.delta_file_envelope import DeltaFileEnvelope
@@ -49,13 +48,6 @@ def _is_sha1_desired(hash_columns: List[pa.Array]) -> bool:
         f"Found total length of hash column={total_len} and total_size={total_size}"
     )
 
-    if SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED:
-        logger.info(
-            f"SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED is True. "
-            f"Returning False for is_sha1_desired"
-        )
-        return False
-
     return total_size > TOTAL_BYTES_IN_SHA1_HASH * total_len
 
 
@@ -116,10 +108,9 @@ def _optimized_group_record_batches_by_hash_bucket(
     record_batches = []
     result_len = 0
     for record_batch in table_batches:
-        if (
-            record_batches
-            and current_bytes + record_batch.nbytes >= MAX_SIZE_OF_RECORD_BATCH_IN_GIB
-        ):
+        current_bytes += record_batch.nbytes
+        record_batches.append(record_batch)
+        if current_bytes >= MAX_SIZE_OF_RECORD_BATCH_IN_GIB:
             logger.info(
                 f"Total number of record batches without exceeding {MAX_SIZE_OF_RECORD_BATCH_IN_GIB} "
                 f"is {len(record_batches)} and size {current_bytes}"
@@ -137,9 +128,6 @@ def _optimized_group_record_batches_by_hash_bucket(
             current_bytes = 0
             record_batches.clear()
 
-        current_bytes += record_batch.nbytes
-        record_batches.append(record_batch)
-
     if record_batches:
         appended_len, append_latency = timed_invocation(
             _append_table_by_hash_bucket,