deltacat 2.0__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/compute/compactor_v2/utils/io.py

@@ -3,7 +3,7 @@ import functools
 from deltacat.storage import (
     PartitionLocator,
     Delta,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from deltacat import logs
 from deltacat.compute.compactor.utils import io as io_v1
@@ -38,7 +38,7 @@ def discover_deltas(
     rebase_source_partition_locator: Optional[PartitionLocator] = None,
     rebase_source_partition_high_watermark: Optional[int] = None,
     rcf_high_watermark: Optional[int] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
     list_deltas_kwargs: Optional[Dict[str, Any]] = {},
 ) -> List[Delta]:
@@ -67,6 +67,11 @@ def discover_deltas(
         f"Length of input deltas from delta source table is {len(delta_source_incremental_deltas)}"
         f" from ({previous_compacted_high_watermark}, {last_stream_position_to_compact}]"
     )
+    logger.info(f"DEBUG: source_partition_locator = {source_partition_locator}")
+    logger.info(
+        f"DEBUG: source_partition_locator.partition_id = {getattr(source_partition_locator, 'partition_id', 'NO_PARTITION_ID')}"
+    )
+    logger.info(f"DEBUG: total input deltas found = {len(result)}")
 
     if rebase_source_partition_locator:
         previous_compacted_deltas = io_v1._discover_deltas(
@@ -93,7 +98,8 @@ def create_uniform_input_deltas(
     hash_bucket_count: int,
     compaction_audit: CompactionSessionAuditInfo,
     compact_partition_params: CompactPartitionParams,
-    deltacat_storage=unimplemented_deltacat_storage,
+    all_column_names: List[str],
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict[str, Any]] = {},
 ) -> List[DeltaAnnotated]:
 
@@ -101,7 +107,6 @@ def create_uniform_input_deltas(
     delta_manifest_entries_count = 0
     estimated_da_bytes = 0
     input_da_list = []
-
     for delta in input_deltas:
         if (
             compact_partition_params.enable_input_split
@@ -114,10 +119,12 @@ def create_uniform_input_deltas(
             )
             append_content_type_params(
                 delta=delta,
+                all_column_names=all_column_names,
                 deltacat_storage=deltacat_storage,
                 deltacat_storage_kwargs=deltacat_storage_kwargs,
                 task_max_parallelism=compact_partition_params.task_max_parallelism,
                 max_parquet_meta_size_bytes=compact_partition_params.max_parquet_meta_size_bytes,
+                file_reader_kwargs_provider=compact_partition_params.read_kwargs_provider,
             )
 
         manifest_entries = delta.manifest.entries

deltacat/compute/compactor_v2/utils/merge.py

@@ -23,6 +23,7 @@ from deltacat.types.tables import TABLE_CLASS_TO_SIZE_FUNC
 
 from deltacat.utils.performance import timed_invocation
 from deltacat.storage import (
+    DeltaType,
     Partition,
 )
 from deltacat.compute.compactor_v2.deletes.delete_strategy import (
@@ -47,13 +48,21 @@ def materialize(
         # TODO (pdames): compare performance to pandas-native materialize path
         df = compacted_table.to_pandas(split_blocks=True, self_destruct=True)
         compacted_table = df
+    # Extract schema from table_writer_kwargs to pass as direct parameter
+    # This ensures schema_id is properly set in the manifest
+    schema = None
+    if input.table_writer_kwargs and "schema" in input.table_writer_kwargs:
+        schema = input.table_writer_kwargs["schema"]
+
     delta, stage_delta_time = timed_invocation(
         input.deltacat_storage.stage_delta,
         compacted_table,
         input.write_to_partition,
+        delta_type=DeltaType.APPEND,  # Compaction always produces APPEND deltas
         max_records_per_entry=input.max_records_per_output_file,
         content_type=input.compacted_file_content_type,
-        s3_table_writer_kwargs=input.s3_table_writer_kwargs,
+        schema=schema,  # Pass schema as direct parameter for schema_id extraction
+        table_writer_kwargs=input.table_writer_kwargs,
         **input.deltacat_storage_kwargs,
     )
     compacted_table_size = TABLE_CLASS_TO_SIZE_FUNC[type(compacted_table)](
@@ -112,6 +121,7 @@ def generate_local_merge_input(
     return MergeInput.of(
         merge_file_groups_provider=LocalMergeFileGroupsProvider(
             annotated_deltas,
+            all_column_names=params.all_column_names,
             read_kwargs_provider=params.read_kwargs_provider,
             deltacat_storage=params.deltacat_storage,
             deltacat_storage_kwargs=params.deltacat_storage_kwargs,
@@ -119,12 +129,13 @@ def generate_local_merge_input(
         write_to_partition=compacted_partition,
         compacted_file_content_type=params.compacted_file_content_type,
         primary_keys=params.primary_keys,
+        all_column_names=params.all_column_names,
         sort_keys=params.sort_keys,
         drop_duplicates=params.drop_duplicates,
         max_records_per_output_file=params.records_per_compacted_file,
         enable_profiler=params.enable_profiler,
         metrics_config=params.metrics_config,
-        s3_table_writer_kwargs=params.s3_table_writer_kwargs,
+        table_writer_kwargs=params.table_writer_kwargs,
         read_kwargs_provider=params.read_kwargs_provider,
         round_completion_info=round_completion_info,
         object_store=params.object_store,
@@ -133,4 +144,6 @@ def generate_local_merge_input(
         delete_strategy=delete_strategy,
         delete_file_envelopes=delete_file_envelopes,
         disable_copy_by_reference=params.disable_copy_by_reference,
+        hash_bucket_count=params.hash_bucket_count,
+        original_fields=params.original_fields,
     )

deltacat/compute/compactor_v2/utils/primary_key_index.py

@@ -10,6 +10,7 @@ from deltacat.compute.compactor_v2.constants import (
     TOTAL_BYTES_IN_SHA1_HASH,
     PK_DELIMITER,
     MAX_SIZE_OF_RECORD_BATCH_IN_GIB,
+    SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED,
 )
 import time
 from deltacat.compute.compactor.model.delta_file_envelope import DeltaFileEnvelope
@@ -48,6 +49,13 @@ def _is_sha1_desired(hash_columns: List[pa.Array]) -> bool:
         f"Found total length of hash column={total_len} and total_size={total_size}"
     )
 
+    if SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED:
+        logger.info(
+            f"SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED is True. "
+            f"Returning False for is_sha1_desired"
+        )
+        return False
+
     return total_size > TOTAL_BYTES_IN_SHA1_HASH * total_len
 
 
@@ -70,13 +78,25 @@ def _append_table_by_hash_bucket(
         f"Grouping a pki table of length {len(pki_table)} took {groupby_latency}s"
     )
 
+    hb_pk_grouped_by = hb_pk_grouped_by.sort_by(sc._HASH_BUCKET_IDX_COLUMN_NAME)
     group_count_array = hb_pk_grouped_by[f"{sc._HASH_BUCKET_IDX_COLUMN_NAME}_count"]
     hb_group_array = hb_pk_grouped_by[sc._HASH_BUCKET_IDX_COLUMN_NAME]
 
     result_len = 0
     for i, group_count in enumerate(group_count_array):
         hb_idx = hb_group_array[i].as_py()
-        pyarrow_table = hb_pk_table.slice(offset=result_len, length=group_count.as_py())
+        group_count_py = group_count.as_py()
+        pyarrow_table = hb_pk_table.slice(offset=result_len, length=group_count_py)
+        assert group_count_py == len(
+            pyarrow_table
+        ), f"Group count {group_count_py} not equal to {len(pyarrow_table)}"
+        all_buckets = pc.unique(pyarrow_table[sc._HASH_BUCKET_IDX_COLUMN_NAME])
+        assert (
+            len(all_buckets) == 1
+        ), f"Only one hash bucket is allowed but found {len(all_buckets)}"
+        assert (
+            all_buckets[0].as_py() == hb_idx
+        ), f"Hash bucket not equal, {all_buckets[0]} and {hb_idx}"
         pyarrow_table = pyarrow_table.drop(
             [sc._HASH_BUCKET_IDX_COLUMN_NAME, sc._PK_HASH_STRING_COLUMN_NAME]
         )
@@ -108,9 +128,10 @@ def _optimized_group_record_batches_by_hash_bucket(
     record_batches = []
     result_len = 0
     for record_batch in table_batches:
-        current_bytes += record_batch.nbytes
-        record_batches.append(record_batch)
-        if current_bytes >= MAX_SIZE_OF_RECORD_BATCH_IN_GIB:
+        if (
+            record_batches
+            and current_bytes + record_batch.nbytes >= MAX_SIZE_OF_RECORD_BATCH_IN_GIB
+        ):
             logger.info(
                 f"Total number of record batches without exceeding {MAX_SIZE_OF_RECORD_BATCH_IN_GIB} "
                 f"is {len(record_batches)} and size {current_bytes}"
@@ -128,6 +149,9 @@ def _optimized_group_record_batches_by_hash_bucket(
             current_bytes = 0
             record_batches.clear()
 
+        current_bytes += record_batch.nbytes
+        record_batches.append(record_batch)
+
     if record_batches:
         appended_len, append_latency = timed_invocation(
             _append_table_by_hash_bucket,

deltacat/compute/compactor_v2/utils/task_options.py

@@ -1,12 +1,17 @@
 import logging
 from typing import Dict, Optional, List, Tuple, Any
 from deltacat import logs
+from deltacat.constants import PYARROW_INFLATION_MULTIPLIER
+from deltacat.compute.compactor_v2.constants import (
+    AVERAGE_RECORD_SIZE_BYTES as DEFAULT_AVERAGE_RECORD_SIZE_BYTES,
+)
 from deltacat.compute.compactor_v2.model.merge_file_group import (
     LocalMergeFileGroupsProvider,
 )
 from deltacat.storage import (
     Manifest,
-    interface as unimplemented_deltacat_storage,
+    ManifestEntry,
+    metastore,
 )
 from deltacat.compute.compactor.model.delta_annotated import DeltaAnnotated
 from deltacat.compute.compactor.model.round_completion_info import RoundCompletionInfo
@@ -72,8 +77,6 @@ def _get_merge_task_options(
     round_completion_info: Optional[RoundCompletionInfo] = None,
     compacted_delta_manifest: Optional[Manifest] = None,
     primary_keys: Optional[List[str]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
-    deltacat_storage_kwargs: Optional[Dict] = {},
     memory_logs_enabled: Optional[bool] = None,
 ) -> Dict[str, Any]:
     if (
@@ -81,16 +84,27 @@ def _get_merge_task_options(
         and compacted_delta_manifest
         and round_completion_info.hb_index_to_entry_range
     ):
-
-        previous_inflation = (
-            round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
-            / round_completion_info.compacted_pyarrow_write_result.file_bytes
+        logger.debug_conditional(
+            f"[Merge task {index}]: Using previous compaction rounds to calculate merge memory: {round_completion_info.compacted_pyarrow_write_result}",
+            memory_logs_enabled,
+        )
+        previous_inflation: float = (
+            (
+                round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
+                / round_completion_info.compacted_pyarrow_write_result.file_bytes
+            )
+            if round_completion_info.compacted_pyarrow_write_result.file_bytes
+            else PYARROW_INFLATION_MULTIPLIER
         )
         debug_memory_params["previous_inflation"] = previous_inflation
 
-        average_record_size = (
-            round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
-            / round_completion_info.compacted_pyarrow_write_result.records
+        average_record_size: float = (
+            (
+                round_completion_info.compacted_pyarrow_write_result.pyarrow_bytes
+                / round_completion_info.compacted_pyarrow_write_result.records
+            )
+            if round_completion_info.compacted_pyarrow_write_result.records
+            else DEFAULT_AVERAGE_RECORD_SIZE_BYTES
         )
         debug_memory_params["average_record_size"] = average_record_size
 
@@ -106,31 +120,36 @@ def _get_merge_task_options(
                 str(hb_idx)
             ]
             for entry_index in range(entry_start, entry_end):
-                entry = compacted_delta_manifest.entries[entry_index]
-
-                current_entry_size = estimate_manifest_entry_size_bytes(
-                    entry=entry,
-                    operation_type=OperationType.PYARROW_DOWNLOAD,
-                    estimate_resources_params=estimate_resources_params,
+                entry: ManifestEntry = compacted_delta_manifest.entries[entry_index]
+                current_entry_size: float = (
+                    estimate_manifest_entry_size_bytes(
+                        entry=entry,
+                        operation_type=OperationType.PYARROW_DOWNLOAD,
+                        estimate_resources_params=estimate_resources_params,
+                    )
+                    or 0.0
                 )
-                current_entry_rows = estimate_manifest_entry_num_rows(
-                    entry=entry,
-                    operation_type=OperationType.PYARROW_DOWNLOAD,
-                    estimate_resources_params=estimate_resources_params,
+                current_entry_rows: int = (
+                    estimate_manifest_entry_num_rows(
+                        entry=entry,
+                        operation_type=OperationType.PYARROW_DOWNLOAD,
+                        estimate_resources_params=estimate_resources_params,
+                    )
+                    or 0
                 )
-
+                # NOTE: We can treat the current_entry_size and current_entry_rows as 0 as a None estimated entry size implies a 0 value
                 data_size += current_entry_size
                 num_rows += current_entry_rows
-
                 if primary_keys:
-                    pk_size = estimate_manifest_entry_column_size_bytes(
+                    pk_size: Optional[
+                        float
+                    ] = estimate_manifest_entry_column_size_bytes(
                         entry=entry,
                         columns=primary_keys,
                         operation_type=OperationType.PYARROW_DOWNLOAD,
                         estimate_resources_params=estimate_resources_params,
                     )
-
-                    if pk_size is None:
+                    if not pk_size:
                         pk_size_bytes += current_entry_size
                     else:
                         pk_size_bytes += pk_size
@@ -159,7 +178,6 @@ def _get_merge_task_options(
         f"[Merge task {index}]: Params used for calculating merge memory: {debug_memory_params}",
         memory_logs_enabled,
     )
-
     return _get_task_options(0.01, total_memory, ray_custom_resources)
 
 
@@ -255,8 +273,6 @@ def merge_resource_options_provider(
     compacted_delta_manifest: Optional[Manifest] = None,
     ray_custom_resources: Optional[Dict] = None,
     primary_keys: Optional[List[str]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
-    deltacat_storage_kwargs: Optional[Dict] = {},
     memory_logs_enabled: Optional[bool] = None,
     **kwargs,
 ) -> Dict:
@@ -286,8 +302,6 @@ def merge_resource_options_provider(
         round_completion_info=round_completion_info,
         compacted_delta_manifest=compacted_delta_manifest,
         primary_keys=primary_keys,
-        deltacat_storage=deltacat_storage,
-        deltacat_storage_kwargs=deltacat_storage_kwargs,
         memory_logs_enabled=memory_logs_enabled,
         estimate_resources_params=estimate_resources_params,
     )
@@ -302,7 +316,7 @@ def local_merge_resource_options_provider(
     compacted_delta_manifest: Optional[Manifest] = None,
     ray_custom_resources: Optional[Dict] = None,
     primary_keys: Optional[List[str]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[Dict] = {},
     memory_logs_enabled: Optional[bool] = None,
     **kwargs,
@@ -328,8 +342,6 @@ def local_merge_resource_options_provider(
         round_completion_info=round_completion_info,
         compacted_delta_manifest=compacted_delta_manifest,
         primary_keys=primary_keys,
-        deltacat_storage=deltacat_storage,
-        deltacat_storage_kwargs=deltacat_storage_kwargs,
         memory_logs_enabled=memory_logs_enabled,
         estimate_resources_params=estimate_resources_params,
     )

deltacat/compute/converter/constants.py

@@ -2,3 +2,8 @@ DEFAULT_CONVERTER_TASK_MAX_PARALLELISM = 4096
 
 # Safe limit ONLY considering CPU limit, typically 32 for a 8x-large worker
 DEFAULT_MAX_PARALLEL_DATA_FILE_DOWNLOAD = 30
+
+
+# Unique identifier delimiter to ensure different primary key don't end up with same hash when concatenated.
+# e.g.: pk column a with value: 1, 12; pk column b with value: 12, 1; Without delimiter will both become "121".
+IDENTIFIER_FIELD_DELIMITER = "c303282d"