deltacat 1.1.38__py3-none-any.whl → 2.0.0__py3-none-any.whl

deltacat/compute/compactor_v2/private/compaction_utils.py

@@ -5,13 +5,15 @@ import ray
 import time
 import json
 from math import ceil
+from urllib.parse import urlparse
+import pyarrow
 
 from deltacat.compute.compactor import (
     PyArrowWriteResult,
     HighWatermark,
     RoundCompletionInfo,
 )
-from deltacat.aws import s3u as s3_utils
+from deltacat.utils.filesystem import resolve_path_and_filesystem
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
 from deltacat.compute.compactor_v2.model.evaluate_compaction_result import (
     ExecutionCompactionResult,
@@ -32,7 +34,7 @@ from deltacat.compute.compactor_v2.utils.merge import (
 from deltacat.compute.compactor_v2.utils.task_options import (
     hash_bucket_resource_options_provider,
 )
-from deltacat.compute.compactor.utils import round_completion_file as rcf
+from deltacat.compute.compactor.utils import round_completion_reader as rci
 from deltacat.compute.compactor import DeltaAnnotated
 from deltacat.compute.compactor_v2.utils.delta import contains_delete_deltas
 from deltacat.compute.compactor_v2.deletes.delete_strategy import (
@@ -48,6 +50,7 @@ from deltacat.storage import (
     DeltaType,
     DeltaLocator,
     Partition,
+    PartitionLocator,
     Manifest,
     Stream,
     StreamLocator,
@@ -63,7 +66,7 @@ from deltacat.compute.compactor_v2.steps import merge as mg
 from deltacat.compute.compactor_v2.steps import hash_bucket as hb
 from deltacat.compute.compactor_v2.utils import io
 
-from typing import List, Optional
+from typing import List, Optional, Union
 from collections import defaultdict
 from deltacat.compute.compactor.model.compaction_session_audit_info import (
     CompactionSessionAuditInfo,
@@ -77,21 +80,39 @@ from deltacat.compute.compactor_v2.utils.task_options import (
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
+def _get_rci_source_partition_locator(
+    params: CompactPartitionParams,
+) -> PartitionLocator:
+    return params.rebase_source_partition_locator or params.source_partition_locator
+
+
+def _is_inplace_compacted(
+    rci_source_partition_locator: PartitionLocator,
+    destination_partition_locator: PartitionLocator,
+) -> bool:
+    return (
+        rci_source_partition_locator.partition_values
+        == destination_partition_locator.partition_values
+        and rci_source_partition_locator.stream_id
+        == destination_partition_locator.stream_id
+    )
+
+
 def _fetch_compaction_metadata(
     params: CompactPartitionParams,
 ) -> tuple[Optional[Manifest], Optional[RoundCompletionInfo]]:
 
     # read the results from any previously completed compaction round
     round_completion_info: Optional[RoundCompletionInfo] = None
-    high_watermark: Optional[HighWatermark] = None
+    high_watermark: Optional[Union[HighWatermark, int]] = None
     previous_compacted_delta_manifest: Optional[Manifest] = None
 
     if not params.rebase_source_partition_locator:
-        round_completion_info = rcf.read_round_completion_file(
-            params.compaction_artifact_s3_bucket,
-            params.source_partition_locator,
-            params.destination_partition_locator,
-            **params.s3_client_kwargs,
+        round_completion_info = rci.read_round_completion_info(
+            source_partition_locator=params.source_partition_locator,
+            destination_partition_locator=params.destination_partition_locator,
+            deltacat_storage=params.deltacat_storage,
+            deltacat_storage_kwargs=params.deltacat_storage_kwargs,
         )
         if not round_completion_info:
             logger.info(
@@ -111,10 +132,10 @@ def _fetch_compaction_metadata(
             assert (
                 params.hash_bucket_count == round_completion_info.hash_bucket_count
             ), (
-                "The hash bucket count has changed. "
-                "Kindly run rebase compaction and trigger incremental again. "
-                f"Hash Bucket count in RCF={round_completion_info.hash_bucket_count} "
-                f"not equal to Hash bucket count in args={params.hash_bucket_count}."
+                "Partition hash bucket count for compaction has changed. "
+                "Rebase compaction with the desired hash bucket count before running another incremental compaction. "
+                f"Hash bucket count in RCI={round_completion_info.hash_bucket_count} "
+                f"!= hash bucket count in params={params.hash_bucket_count}."
             )
 
         logger.info(f"Round completion file: {round_completion_info}")
@@ -129,7 +150,7 @@ def _build_uniform_deltas(
     mutable_compaction_audit: CompactionSessionAuditInfo,
     input_deltas: List[Delta],
     delta_discovery_start: float,
-) -> tuple[List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope], Partition]:
+) -> tuple[List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope]]:
 
     delete_strategy: Optional[DeleteStrategy] = None
     delete_file_envelopes: Optional[List[DeleteFileEnvelope]] = None
@@ -149,6 +170,7 @@ def _build_uniform_deltas(
         hash_bucket_count=params.hash_bucket_count,
         compaction_audit=mutable_compaction_audit,
         compact_partition_params=params,
+        all_column_names=params.all_column_names,
         deltacat_storage=params.deltacat_storage,
         deltacat_storage_kwargs=params.deltacat_storage_kwargs,
     )
@@ -159,10 +181,9 @@ def _build_uniform_deltas(
         delta_discovery_end - delta_discovery_start
     )
 
-    s3_utils.upload(
-        mutable_compaction_audit.audit_url,
-        str(json.dumps(mutable_compaction_audit)),
-        **params.s3_client_kwargs,
+    _upload_audit_data(
+        params,
+        mutable_compaction_audit,
     )
 
     return (
@@ -222,7 +243,7 @@ def _run_hash_and_merge(
     uniform_deltas: List[DeltaAnnotated],
     round_completion_info: RoundCompletionInfo,
     delete_strategy: Optional[DeleteStrategy],
-    delete_file_envelopes: Optional[DeleteFileEnvelope],
+    delete_file_envelopes: Optional[List[DeleteFileEnvelope]],
     mutable_compaction_audit: CompactionSessionAuditInfo,
     previous_compacted_delta_manifest: Optional[Manifest],
     compacted_partition: Partition,
@@ -267,10 +288,9 @@ def _run_hash_and_merge(
             hb_end - hb_start,
         )
 
-        s3_utils.upload(
-            mutable_compaction_audit.audit_url,
-            str(json.dumps(mutable_compaction_audit)),
-            **params.s3_client_kwargs,
+        _upload_audit_data(
+            params,
+            mutable_compaction_audit,
         )
 
         hb_data_processed_size_bytes = np.int64(0)
@@ -389,7 +409,7 @@ def _merge(
     all_hash_group_idx_to_obj_id: dict,
     compacted_partition: Partition,
     delete_strategy: DeleteStrategy,
-    delete_file_envelopes: DeleteFileEnvelope,
+    delete_file_envelopes: List[DeleteFileEnvelope],
 ) -> tuple[List[MergeResult], float]:
     merge_options_provider = functools.partial(
         task_resource_options_provider,
@@ -402,13 +422,24 @@ def _merge(
         round_completion_info=round_completion_info,
         compacted_delta_manifest=previous_compacted_delta_manifest,
         primary_keys=params.primary_keys,
-        deltacat_storage=params.deltacat_storage,
-        deltacat_storage_kwargs=params.deltacat_storage_kwargs,
         ray_custom_resources=params.ray_custom_resources,
         memory_logs_enabled=params.memory_logs_enabled,
         estimate_resources_params=params.estimate_resources_params,
     )
 
+    # set previous compacted delta manifest on input so that we don't need a transaction to retrieve it
+    if round_completion_info:
+        previous_compacted_delta_manifest = params.deltacat_storage.get_delta_manifest(
+            round_completion_info.compacted_delta_locator,
+            **params.deltacat_storage_kwargs,
+        )
+
+    # create a copy of deltacat storage kwargs without any parent transaction context
+    # (can't be serialized by Ray, and we're only downloading already-resolved manifest entries)
+    deltacat_storage_kwargs_copy = {
+        k: v for k, v in params.deltacat_storage_kwargs.items() if k != "transaction"
+    }
+
     def merge_input_provider(index, item) -> dict[str, MergeInput]:
         return {
             "input": MergeInput.of(
@@ -422,23 +453,26 @@ def _merge(
                 write_to_partition=compacted_partition,
                 compacted_file_content_type=params.compacted_file_content_type,
                 primary_keys=params.primary_keys,
+                all_column_names=params.all_column_names,
                 sort_keys=params.sort_keys,
                 merge_task_index=index,
                 drop_duplicates=params.drop_duplicates,
                 max_records_per_output_file=params.records_per_compacted_file,
                 enable_profiler=params.enable_profiler,
                 metrics_config=params.metrics_config,
-                s3_table_writer_kwargs=params.s3_table_writer_kwargs,
+                table_writer_kwargs=params.table_writer_kwargs,
                 read_kwargs_provider=params.read_kwargs_provider,
                 round_completion_info=round_completion_info,
                 object_store=params.object_store,
                 deltacat_storage=params.deltacat_storage,
-                deltacat_storage_kwargs=params.deltacat_storage_kwargs,
+                deltacat_storage_kwargs=deltacat_storage_kwargs_copy,
                 delete_strategy=delete_strategy,
                 delete_file_envelopes=delete_file_envelopes,
                 memory_logs_enabled=params.memory_logs_enabled,
                 disable_copy_by_reference=params.disable_copy_by_reference,
                 hash_bucket_count=params.hash_bucket_count,
+                original_fields=params.original_fields,
+                compacted_manifest=previous_compacted_delta_manifest,
             )
         }
 
@@ -474,6 +508,12 @@ def _hash_bucket(
         estimate_resources_params=params.estimate_resources_params,
     )
 
+    # create a copy of deltacat storage kwargs without any parent transaction context
+    # (can't be serialized by Ray, and we're only downloading already-resolved manifest entries)
+    deltacat_storage_kwargs_copy = {
+        k: v for k, v in params.deltacat_storage_kwargs.items() if k != "transaction"
+    }
+
     def hash_bucket_input_provider(index, item) -> dict[str, HashBucketInput]:
         return {
             "input": HashBucketInput.of(
@@ -482,12 +522,13 @@ def _hash_bucket(
                 hb_task_index=index,
                 num_hash_buckets=params.hash_bucket_count,
                 num_hash_groups=params.hash_group_count,
+                all_column_names=params.all_column_names,
                 enable_profiler=params.enable_profiler,
                 metrics_config=params.metrics_config,
                 read_kwargs_provider=params.read_kwargs_provider,
                 object_store=params.object_store,
                 deltacat_storage=params.deltacat_storage,
-                deltacat_storage_kwargs=params.deltacat_storage_kwargs,
+                deltacat_storage_kwargs=deltacat_storage_kwargs_copy,
                 memory_logs_enabled=params.memory_logs_enabled,
             )
         }
@@ -596,10 +637,9 @@ def _process_merge_results(
         file_index += mat_result.pyarrow_write_result.files
         previous_task_index = mat_result.task_index
 
-    s3_utils.upload(
-        mutable_compaction_audit.audit_url,
-        str(json.dumps(mutable_compaction_audit)),
-        **params.s3_client_kwargs,
+    _upload_audit_data(
+        params,
+        mutable_compaction_audit,
     )
     deltas: List[Delta] = [m.delta for m in mat_results]
     # Note: An appropriate last stream position must be set
@@ -634,21 +674,20 @@ def _update_and_upload_compaction_audit(
             + round_completion_info.compacted_pyarrow_write_result.records
         )
 
-    s3_utils.upload(
-        mutable_compaction_audit.audit_url,
-        str(json.dumps(mutable_compaction_audit)),
-        **params.s3_client_kwargs,
+    _upload_audit_data(
+        params,
+        mutable_compaction_audit,
     )
     return
 
 
-def _write_new_round_completion_file(
+def _create_round_completion_info(
     params: CompactPartitionParams,
     mutable_compaction_audit: CompactionSessionAuditInfo,
     compacted_partition: Partition,
     audit_url: str,
     hb_id_to_entry_indices_range: dict,
-    rcf_source_partition_locator: rcf.PartitionLocator,
+    rci_source_partition_locator: PartitionLocator,
     new_compacted_delta_locator: DeltaLocator,
     pyarrow_write_result: PyArrowWriteResult,
     prev_round_completion_info: Optional[RoundCompletionInfo] = None,
@@ -690,6 +729,27 @@ def _write_new_round_completion_file(
         prev_round_completion_info,
     )
 
+    # Check if this is an in-place compaction before creating RoundCompletionInfo
+    logger.info(
+        f"Checking if partition {rci_source_partition_locator} is inplace compacted against {params.destination_partition_locator}..."
+    )
+    is_inplace_compacted: bool = _is_inplace_compacted(
+        rci_source_partition_locator, params.destination_partition_locator
+    )
+
+    # Determine the prev_source_partition_locator based on compaction type
+    if is_inplace_compacted:
+        logger.info(
+            "In-place compaction detected. Using compacted partition locator as prev_source_partition_locator. "
+            + f"Got compacted partition partition_id of {compacted_partition.locator.partition_id} "
+            f"and rci source partition_id of {rci_source_partition_locator.partition_id}."
+        )
+        prev_source_partition_locator = compacted_partition.locator
+        # Update rci_source_partition_locator for backward compatibility
+        rci_source_partition_locator = compacted_partition.locator
+    else:
+        prev_source_partition_locator = rci_source_partition_locator
+
     new_round_completion_info = RoundCompletionInfo.of(
         high_watermark=params.last_stream_position_to_compact,
         compacted_delta_locator=new_compacted_delta_locator,
@@ -702,41 +762,17 @@ def _write_new_round_completion_file(
         compactor_version=CompactorVersion.V2.value,
         input_inflation=input_inflation,
         input_average_record_size_bytes=input_average_record_size_bytes,
+        prev_source_partition_locator=prev_source_partition_locator,
     )
 
     logger.info(
         f"Partition-{params.source_partition_locator.partition_values},"
         f"compacted at: {params.last_stream_position_to_compact},"
     )
-    logger.info(
-        f"Checking if partition {rcf_source_partition_locator} is inplace compacted against {params.destination_partition_locator}..."
-    )
-    is_inplace_compacted: bool = (
-        rcf_source_partition_locator.partition_values
-        == params.destination_partition_locator.partition_values
-        and rcf_source_partition_locator.stream_id
-        == params.destination_partition_locator.stream_id
-    )
-    if is_inplace_compacted:
-        logger.info(
-            "Overriding round completion file source partition locator as in-place compacted. "
-            + f"Got compacted partition partition_id of {compacted_partition.locator.partition_id} "
-            f"and rcf source partition_id of {rcf_source_partition_locator.partition_id}."
-        )
-        rcf_source_partition_locator = compacted_partition.locator
-
-    round_completion_file_s3_url = rcf.write_round_completion_file(
-        params.compaction_artifact_s3_bucket,
-        rcf_source_partition_locator,
-        compacted_partition.locator,
-        new_round_completion_info,
-        **params.s3_client_kwargs,
-    )
 
     return ExecutionCompactionResult(
         compacted_partition,
         new_round_completion_info,
-        round_completion_file_s3_url,
         is_inplace_compacted,
     )
 
@@ -752,21 +788,29 @@ def _commit_compaction_result(
         f"Partition-{params.source_partition_locator} -> "
         f"{compaction_session_type} Compaction session data processing completed"
     )
+    # TODO(pdames): Uncomment this once we support concurrent writes to the same
+    #   partition (via write_to_table). This requires updating the commit_partition
+    #   method to support previous partition as input. Right now, a concurrent write
+    #   to the same partition will cause the commit_partition method to fail.
     if execute_compaction_result.new_compacted_partition:
         previous_partition: Optional[Partition] = None
-        if execute_compaction_result.is_inplace_compacted:
-            previous_partition: Optional[
-                Partition
-            ] = params.deltacat_storage.get_partition(
-                params.source_partition_locator.stream_locator,
-                params.source_partition_locator.partition_values,
-                **params.deltacat_storage_kwargs,
-            )
-            # NOTE: Retrieving the previous partition again as the partition_id may have changed by the time commit_partition is called.
+        #   if execute_compaction_result.is_inplace_compacted:
+        #       previous_partition: Optional[
+        #           Partition
+        #       ] = params.deltacat_storage.get_partition(
+        #           params.source_partition_locator.stream_locator,
+        #           params.source_partition_locator.partition_values,
+        #           **params.deltacat_storage_kwargs,
+        #       )
+        #       # NOTE: Retrieving the previous partition again as the partition_id may have changed by the time commit_partition is called.
         logger.info(
             f"Committing compacted partition to: {execute_compaction_result.new_compacted_partition.locator} "
             f"using previous partition: {previous_partition.locator if previous_partition else None}"
         )
+        # Set the round completion info on the partition before committing
+        execute_compaction_result.new_compacted_partition.compaction_round_completion_info = (
+            execute_compaction_result.new_round_completion_info
+        )
         committed_partition: Partition = params.deltacat_storage.commit_partition(
             execute_compaction_result.new_compacted_partition,
             previous_partition,
@@ -777,3 +821,57 @@ def _commit_compaction_result(
         logger.warning("No new partition was committed during compaction.")
 
     logger.info(f"Completed compaction session for: {params.source_partition_locator}")
+
+
+def _upload_audit_data(
+    params: CompactPartitionParams,
+    audit_info: CompactionSessionAuditInfo,
+) -> None:
+    """
+    Upload audit data to the specified URL using the filesystem from catalog properties.
+    """
+    audit_url = audit_info.audit_url
+    audit_data = json.dumps(audit_info.to_serializable(params.catalog.root))
+    if params.catalog and params.catalog.filesystem:
+        # Use the filesystem from catalog properties
+        filesystem = params.catalog.filesystem
+        parsed_url = urlparse(audit_url)
+        # For filesystem paths, use the path component
+        path = parsed_url.path if parsed_url.scheme else audit_url
+
+        # Ensure parent directories exist
+        import os
+
+        parent_dir = os.path.dirname(path)
+        if (
+            parent_dir
+            and not filesystem.get_file_info(parent_dir).type
+            == pyarrow.fs.FileType.Directory
+        ):
+            try:
+                filesystem.create_dir(parent_dir, recursive=True)
+            except Exception as e:
+                logger.warning(f"Failed to create directory {parent_dir}: {e}")
+
+        with filesystem.open_output_stream(path) as output_stream:
+            output_stream.write(audit_data.encode("utf-8"))
+    else:
+        # Fallback: resolve filesystem from the URL
+        path, filesystem = resolve_path_and_filesystem(audit_url)
+
+        # Ensure parent directories exist
+        import os
+
+        parent_dir = os.path.dirname(path)
+        if (
+            parent_dir
+            and not filesystem.get_file_info(parent_dir).type
+            == pyarrow.fs.FileType.Directory
+        ):
+            try:
+                filesystem.create_dir(parent_dir, recursive=True)
+            except Exception as e:
+                logger.warning(f"Failed to create directory {parent_dir}: {e}")
+
+        with filesystem.open_output_stream(path) as output_stream:
+            output_stream.write(audit_data.encode("utf-8"))

deltacat/compute/compactor_v2/steps/hash_bucket.py

@@ -18,7 +18,7 @@ from deltacat.compute.compactor_v2.utils.primary_key_index import (
     group_hash_bucket_indices,
     group_by_pk_hash_bucket,
 )
-from deltacat.storage import interface as unimplemented_deltacat_storage
+from deltacat.storage import metastore
 from deltacat.utils.ray_utils.runtime import (
     get_current_ray_task_id,
     get_current_ray_worker_id,
@@ -50,8 +50,9 @@ def _group_file_records_by_pk_hash_bucket(
     annotated_delta: DeltaAnnotated,
     num_hash_buckets: int,
     primary_keys: List[str],
+    all_column_names: List[str],
     read_kwargs_provider: Optional[ReadKwargsProvider] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
+    deltacat_storage=metastore,
     deltacat_storage_kwargs: Optional[dict] = None,
 ) -> Tuple[Optional[DeltaFileEnvelopeGroups], int, int]:
     # read input parquet s3 objects into a list of delta file envelopes
@@ -61,6 +62,7 @@ def _group_file_records_by_pk_hash_bucket(
         total_size_bytes,
     ) = read_delta_file_envelopes(
         annotated_delta,
+        all_column_names,
         read_kwargs_provider,
         deltacat_storage,
         deltacat_storage_kwargs,
@@ -116,6 +118,7 @@ def _timed_hash_bucket(input: HashBucketInput):
             annotated_delta=input.annotated_delta,
             num_hash_buckets=input.num_hash_buckets,
             primary_keys=input.primary_keys,
+            all_column_names=input.all_column_names,
             read_kwargs_provider=input.read_kwargs_provider,
             deltacat_storage=input.deltacat_storage,
             deltacat_storage_kwargs=input.deltacat_storage_kwargs,