deltacat 2.0.0b11__py3-none-any.whl → 2.0.0.post1__py3-none-any.whl

deltacat/compute/compactor/utils/round_completion_reader.py

@@ -0,0 +1,117 @@
+import logging
+from typing import Optional
+from deltacat import logs
+from deltacat.compute.compactor import RoundCompletionInfo
+from deltacat.storage import PartitionLocator
+from deltacat.storage.model.partition import Partition
+from deltacat.utils.metrics import metrics
+from deltacat.exceptions import PartitionNotFoundError
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+@metrics
+def read_round_completion_info(
+    source_partition_locator: PartitionLocator,
+    destination_partition_locator: PartitionLocator,
+    deltacat_storage,
+    deltacat_storage_kwargs: Optional[dict] = None,
+    destination_partition: Optional[Partition] = None,
+) -> Optional[RoundCompletionInfo]:
+    """
+    Read round completion info from the partition metafile.
+
+    Args:
+        source_partition_locator: Source partition locator for validation
+        destination_partition_locator: Destination partition locator
+        deltacat_storage: Storage implementation
+        deltacat_storage_kwargs: Optional storage kwargs
+        destination_partition: Optional destination partition to avoid redundant get_partition calls
+
+    Returns:
+        RoundCompletionInfo if found in partition, None otherwise
+    """
+    if not destination_partition_locator:
+        return None
+
+    if deltacat_storage_kwargs is None:
+        deltacat_storage_kwargs = {}
+
+    try:
+        # Use provided partition or get it from storage
+        if destination_partition:
+            partition = destination_partition
+        else:
+            # First get the current partition to access its previous_partition_id
+            current_partition: Partition = deltacat_storage.get_partition(
+                destination_partition_locator.stream_locator,
+                destination_partition_locator.partition_values,
+                **deltacat_storage_kwargs,
+            )
+
+            # If current partition has round completion info, use it
+            if current_partition.compaction_round_completion_info:
+                partition = current_partition
+            elif current_partition.previous_partition_id is not None:
+                # For incremental compaction, we need to get the previous committed partition
+                # that contains the round completion info.
+                # Get the previous partition by ID - this is where the round completion info should be
+                logger.info(
+                    f"Current partition {destination_partition_locator} does not have round completion info, "
+                    f"getting previous partition with ID: {current_partition.previous_partition_id}"
+                )
+                previous_partition = deltacat_storage.get_partition_by_id(
+                    destination_partition_locator.stream_locator,
+                    current_partition.previous_partition_id,
+                    **deltacat_storage_kwargs,
+                )
+                if previous_partition is not None:
+                    logger.info(
+                        f"Found previous partition: {previous_partition.locator}"
+                    )
+                    partition = previous_partition
+                else:
+                    raise PartitionNotFoundError(
+                        f"Previous partition with ID {current_partition.previous_partition_id} not found"
+                    )
+            else:
+                logger.info(f"No previous partition ID found, using current partition")
+                partition = current_partition
+
+        if partition:
+            round_completion_info = partition.compaction_round_completion_info
+            if round_completion_info:
+                # Validate that prev_source_partition_locator matches current source
+                if (
+                    not source_partition_locator
+                    or not round_completion_info.prev_source_partition_locator
+                ):
+                    raise ValueError(
+                        f"Source partition locator ({source_partition_locator}) and "
+                        f"prev_source_partition_locator ({round_completion_info.prev_source_partition_locator}) "
+                        f"must both be provided."
+                    )
+
+                if (
+                    round_completion_info.prev_source_partition_locator.canonical_string()
+                    != source_partition_locator.canonical_string()
+                ):
+                    logger.warning(
+                        f"Previous source partition locator mismatch: "
+                        f"expected {source_partition_locator.canonical_string()}, "
+                        f"but found {round_completion_info.prev_source_partition_locator.canonical_string()} "
+                        f"in round completion info. Ignoring cached round completion info."
+                    )
+                    return None
+
+                logger.info(
+                    f"Read round completion info from partition metafile: {round_completion_info}"
+                )
+                return round_completion_info
+
+    except Exception as e:
+        logger.debug(
+            f"Failed to read round completion info from partition metafile: {e}"
+        )
+
+    return None

deltacat/compute/compactor/utils/system_columns.py

@@ -294,7 +294,9 @@ def append_dedupe_task_idx_col(table: pa.Table, dedupe_task_indices) -> pa.Table
 
 
 def delta_type_to_field(delta_type: DeltaType) -> bool:
-    return True if delta_type is DeltaType.UPSERT else False
+    # For deduplication purposes, treat both UPSERT and APPEND as UPSERT (True)
+    # Only DELETE should be treated as DELETE (False)
+    return delta_type is not DeltaType.DELETE
 
 
 def delta_type_from_field(delta_type_field: bool) -> DeltaType:

deltacat/compute/compactor_v2/compaction_session.py

@@ -14,7 +14,6 @@ from deltacat.compute.compactor_v2.model.evaluate_compaction_result import (
     ExecutionCompactionResult,
 )
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
-from deltacat.compute.compactor.utils import round_completion_file as rcf
 from deltacat.compute.compactor import DeltaAnnotated
 from deltacat.compute.compactor_v2.deletes.delete_strategy import (
     DeleteStrategy,
@@ -27,6 +26,7 @@ from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
 from deltacat.storage import (
     Delta,
     DeltaLocator,
+    PartitionLocator,
 )
 from deltacat.storage.model.manifest import Manifest
 from deltacat.compute.compactor.model.compact_partition_params import (
@@ -36,13 +36,14 @@ from deltacat.utils.resources import (
     get_current_process_peak_memory_usage_in_bytes,
 )
 from deltacat.compute.compactor_v2.private.compaction_utils import (
+    _get_rci_source_partition_locator,
     _fetch_compaction_metadata,
     _build_uniform_deltas,
     _group_uniform_deltas,
     _stage_new_partition,
     _run_hash_and_merge,
     _process_merge_results,
-    _write_new_round_completion_file,
+    _create_round_completion_info,
     _commit_compaction_result,
 )
 from deltacat.utils.metrics import metrics
@@ -64,24 +65,26 @@ logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 @metrics(prefix=COMPACT_PARTITION_METRIC_PREFIX)
 @categorize_errors
-def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]:
+def compact_partition(params: CompactPartitionParams, **kwargs) -> None:
     assert (
         params.hash_bucket_count is not None and params.hash_bucket_count >= 1
     ), "hash_bucket_count is a required arg for compactor v2"
+    assert type(params.hash_bucket_count) is int, "Hash bucket count must be an integer"
     if params.num_rounds > 1:
         assert (
             not params.drop_duplicates
         ), "num_rounds > 1, drop_duplicates must be False but is True"
 
-    with memray.Tracker(
-        "compaction_partition.bin"
-    ) if params.enable_profiler else nullcontext():
+    with (
+        memray.Tracker("compaction_partition.bin")
+        if params.enable_profiler
+        else nullcontext()
+    ):
         execute_compaction_result: ExecutionCompactionResult = _execute_compaction(
             params,
             **kwargs,
         )
         _commit_compaction_result(params, execute_compaction_result)
-        return execute_compaction_result.round_completion_file_s3_url
 
 
 def _execute_compaction(
@@ -96,12 +99,12 @@ def _execute_compaction(
         previous_compacted_delta_manifest,
         round_completion_info,
     ) = fetch_compaction_metadata_result
-    rcf_source_partition_locator: rcf.PartitionLocator = (
-        params.rebase_source_partition_locator or params.source_partition_locator
+    rci_source_partition_locator: PartitionLocator = _get_rci_source_partition_locator(
+        params
     )
 
-    base_audit_url: str = rcf_source_partition_locator.path(
-        f"s3://{params.compaction_artifact_s3_bucket}/compaction-audit"
+    base_audit_url: str = rci_source_partition_locator.path(
+        f"{params.compaction_artifact_path}/compaction-audit"
     )
     audit_url: str = f"{base_audit_url}.json"
     logger.info(f"Compaction audit will be written to {audit_url}")
@@ -136,7 +139,7 @@ def _execute_compaction(
     )
     if not input_deltas:
         logger.info("No input deltas found to compact.")
-        return ExecutionCompactionResult(None, None, None, False)
+        return ExecutionCompactionResult(None, None, False)
     build_uniform_deltas_result: tuple[
         List[DeltaAnnotated], DeleteStrategy, List[DeleteFileEnvelope]
     ] = _build_uniform_deltas(
@@ -199,13 +202,13 @@ def _execute_compaction(
 
     compaction_audit.save_round_completion_stats(mat_results)
 
-    compaction_result: ExecutionCompactionResult = _write_new_round_completion_file(
+    compaction_result: ExecutionCompactionResult = _create_round_completion_info(
         params,
         compaction_audit,
         compacted_partition,
         audit_url,
         hb_id_to_entry_indices_range,
-        rcf_source_partition_locator,
+        rci_source_partition_locator,
         new_compacted_delta_locator,
         pyarrow_write_result,
         round_completion_info,

deltacat/compute/compactor_v2/constants.py

@@ -1,3 +1,5 @@
+from deltacat.utils.common import env_bool, env_integer, env_string
+
 TOTAL_BYTES_IN_SHA1_HASH = 20
 
 PK_DELIMITER = "L6kl7u5f"
@@ -31,7 +33,9 @@ TOTAL_MEMORY_BUFFER_PERCENTAGE = 30
 # The total size of records that will be hash bucketed at once
 # Since, sorting is nlogn, we ensure that is not performed
 # on a very large dataset for best performance.
-MAX_SIZE_OF_RECORD_BATCH_IN_GIB = 2 * 1024 * 1024 * 1024
+MAX_SIZE_OF_RECORD_BATCH_IN_GIB = env_integer(
+    "MAX_SIZE_OF_RECORD_BATCH_IN_GIB", 2 * 1024 * 1024 * 1024
+)
 
 # Whether to drop duplicates during merge.
 DROP_DUPLICATES = True
@@ -78,3 +82,28 @@ COMPACT_PARTITION_METRIC_PREFIX = "compact_partition"
 # Number of rounds to run hash/merge for a single
 # partition. (For large table support)
 DEFAULT_NUM_ROUNDS = 1
+
+# Whether to perform sha1 hashing when required to
+# optimize memory. For example, hashing is always
+# required for bucketing where it's not mandatory
+# when dropping duplicates. Setting this to True
+# will disable sha1 hashing in cases where it isn't
+# mandatory. This flag is False by default.
+SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED = env_bool(
+    "SHA1_HASHING_FOR_MEMORY_OPTIMIZATION_DISABLED", False
+)
+
+# This env variable specifies whether to check bucketing spec
+# compliance of the existing compacted table.
+# PRINT_LOG: Enable logging if any partition is found
+# to be non-compliant with the bucketing spec.
+# ASSERT: Fail the job with ValidationError if the
+# current compacted partition is found to be non-compliant
+# with bucketing spec. Note, logging is implicitly enabled
+# in this case.
+BUCKETING_SPEC_COMPLIANCE_PROFILE = env_string(
+    "BUCKETING_SPEC_COMPLIANCE_PROFILE", None
+)
+
+BUCKETING_SPEC_COMPLIANCE_PRINT_LOG = "PRINT_LOG"
+BUCKETING_SPEC_COMPLIANCE_ASSERT = "ASSERT"

deltacat/compute/compactor_v2/model/evaluate_compaction_result.py

@@ -13,7 +13,6 @@ from typing import Optional
 class ExecutionCompactionResult:
     new_compacted_partition: Optional[Partition]
     new_round_completion_info: Optional[RoundCompletionInfo]
-    round_completion_file_s3_url: Optional[str]
     is_inplace_compacted: bool
 
     def __iter__(self):

deltacat/compute/compactor_v2/model/hash_bucket_input.py

@@ -4,7 +4,7 @@ from typing import Dict, List, Optional, Any
 from deltacat.utils.metrics import MetricsConfig
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.io.object_store import IObjectStore
-from deltacat.storage import interface as unimplemented_deltacat_storage
+from deltacat.storage import metastore
 from deltacat.compute.compactor import DeltaAnnotated
 
 
@@ -15,12 +15,13 @@ class HashBucketInput(Dict):
         primary_keys: List[str],
         num_hash_buckets: int,
         num_hash_groups: int,
+        all_column_names: List[str],
         hb_task_index: Optional[int] = 0,
         enable_profiler: Optional[bool] = False,
         metrics_config: Optional[MetricsConfig] = None,
         read_kwargs_provider: Optional[ReadKwargsProvider] = None,
         object_store: Optional[IObjectStore] = None,
-        deltacat_storage=unimplemented_deltacat_storage,
+        deltacat_storage=metastore,
         deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
         memory_logs_enabled: Optional[bool] = None,
     ) -> HashBucketInput:
@@ -31,6 +32,7 @@ class HashBucketInput(Dict):
         result["hb_task_index"] = hb_task_index
         result["num_hash_buckets"] = num_hash_buckets
         result["num_hash_groups"] = num_hash_groups
+        result["all_column_names"] = all_column_names
         result["enable_profiler"] = enable_profiler
         result["metrics_config"] = metrics_config
         result["read_kwargs_provider"] = read_kwargs_provider
@@ -61,6 +63,10 @@ class HashBucketInput(Dict):
     def num_hash_groups(self) -> int:
         return self["num_hash_groups"]
 
+    @property
+    def all_column_names(self) -> List[str]:
+        return self["all_column_names"]
+
     @property
     def enable_profiler(self) -> Optional[bool]:
         return self.get("enable_profiler")
@@ -78,7 +84,7 @@ class HashBucketInput(Dict):
         return self.get("object_store")
 
     @property
-    def deltacat_storage(self) -> unimplemented_deltacat_storage:
+    def deltacat_storage(self) -> metastore:
         return self.get("deltacat_storage")
 
     @property

deltacat/compute/compactor_v2/model/merge_file_group.py

@@ -16,7 +16,7 @@ from deltacat.compute.compactor_v2.utils.primary_key_index import (
     hash_group_index_to_hash_bucket_indices,
 )
 
-from deltacat.storage import interface as unimplemented_deltacat_storage
+from deltacat.storage import metastore
 
 from deltacat.io.object_store import IObjectStore
 
@@ -87,11 +87,13 @@ class LocalMergeFileGroupsProvider(MergeFileGroupsProvider):
     def __init__(
         self,
         uniform_deltas: List[DeltaAnnotated],
+        all_column_names: List[str],
         read_kwargs_provider: Optional[ReadKwargsProvider],
-        deltacat_storage=unimplemented_deltacat_storage,
+        deltacat_storage=metastore,
         deltacat_storage_kwargs: Optional[dict] = None,
     ):
         self._deltas = uniform_deltas
+        self._all_column_names = all_column_names
         self._read_kwargs_provider = read_kwargs_provider
         self._deltacat_storage = deltacat_storage
         self._deltacat_storage_kwargs = deltacat_storage_kwargs
@@ -110,6 +112,7 @@ class LocalMergeFileGroupsProvider(MergeFileGroupsProvider):
                 total_size_bytes,
             ) = read_delta_file_envelopes(
                 annotated_delta,
+                self._all_column_names,
                 self._read_kwargs_provider,
                 self._deltacat_storage,
                 self._deltacat_storage_kwargs,

deltacat/compute/compactor_v2/model/merge_input.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Dict, List, Optional, Any
+from typing import Dict, List, Optional, Any, Set
 
 from deltacat.compute.compactor_v2.model.merge_file_group import (
     MergeFileGroupsProvider,
@@ -12,9 +12,10 @@ from deltacat.utils.metrics import MetricsConfig
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.io.object_store import IObjectStore
 from deltacat.storage import (
+    Manifest,
     Partition,
     SortKey,
-    interface as unimplemented_deltacat_storage,
+    metastore,
 )
 from deltacat.compute.compactor_v2.constants import (
     DROP_DUPLICATES,
@@ -32,22 +33,26 @@ class MergeInput(Dict):
         write_to_partition: Partition,
         compacted_file_content_type: ContentType,
         primary_keys: List[str],
+        all_column_names: List[str],
         drop_duplicates: Optional[bool] = DROP_DUPLICATES,
         sort_keys: Optional[List[SortKey]] = None,
         merge_task_index: Optional[int] = 0,
         max_records_per_output_file: Optional[int] = MAX_RECORDS_PER_COMPACTED_FILE,
         enable_profiler: Optional[bool] = False,
         metrics_config: Optional[MetricsConfig] = None,
-        s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
+        table_writer_kwargs: Optional[Dict[str, Any]] = None,
         read_kwargs_provider: Optional[ReadKwargsProvider] = None,
         round_completion_info: Optional[RoundCompletionInfo] = None,
         object_store: Optional[IObjectStore] = None,
         delete_strategy: Optional[DeleteStrategy] = None,
         delete_file_envelopes: Optional[List[DeleteFileEnvelope]] = None,
-        deltacat_storage=unimplemented_deltacat_storage,
+        deltacat_storage=metastore,
         deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
         memory_logs_enabled: Optional[bool] = None,
         disable_copy_by_reference: Optional[bool] = None,
+        hash_bucket_count: Optional[int] = None,
+        original_fields: Optional[Set[str]] = None,
+        compacted_manifest: Optional[Manifest] = None,
     ) -> MergeInput:
 
         result = MergeInput()
@@ -55,13 +60,14 @@ class MergeInput(Dict):
         result["write_to_partition"] = write_to_partition
         result["compacted_file_content_type"] = compacted_file_content_type
         result["primary_keys"] = primary_keys
+        result["all_column_names"] = all_column_names
         result["drop_duplicates"] = drop_duplicates
         result["sort_keys"] = sort_keys
         result["merge_task_index"] = merge_task_index
         result["max_records_per_output_file"] = max_records_per_output_file
         result["enable_profiler"] = enable_profiler
         result["metrics_config"] = metrics_config
-        result["s3_table_writer_kwargs"] = s3_table_writer_kwargs or {}
+        result["table_writer_kwargs"] = table_writer_kwargs or {}
         result["read_kwargs_provider"] = read_kwargs_provider
         result["round_completion_info"] = round_completion_info
         result["object_store"] = object_store
@@ -71,6 +77,9 @@ class MergeInput(Dict):
         result["deltacat_storage_kwargs"] = deltacat_storage_kwargs or {}
         result["memory_logs_enabled"] = memory_logs_enabled
         result["disable_copy_by_reference"] = disable_copy_by_reference
+        result["hash_bucket_count"] = hash_bucket_count
+        result["original_fields"] = original_fields
+        result["compacted_manifest"] = compacted_manifest
         return result
 
     @property
@@ -89,6 +98,10 @@ class MergeInput(Dict):
     def primary_keys(self) -> List[str]:
         return self["primary_keys"]
 
+    @property
+    def all_column_names(self) -> List[str]:
+        return self["all_column_names"]
+
     @property
     def drop_duplicates(self) -> int:
         return self["drop_duplicates"]
@@ -114,8 +127,8 @@ class MergeInput(Dict):
         return self.get("metrics_config")
 
     @property
-    def s3_table_writer_kwargs(self) -> Optional[Dict[str, Any]]:
-        return self.get("s3_table_writer_kwargs")
+    def table_writer_kwargs(self) -> Optional[Dict[str, Any]]:
+        return self.get("table_writer_kwargs")
 
     @property
     def read_kwargs_provider(self) -> Optional[ReadKwargsProvider]:
@@ -130,7 +143,7 @@ class MergeInput(Dict):
         return self.get("object_store")
 
     @property
-    def deltacat_storage(self) -> unimplemented_deltacat_storage:
+    def deltacat_storage(self) -> metastore:
         return self["deltacat_storage"]
 
     @property
@@ -154,3 +167,15 @@ class MergeInput(Dict):
     @property
     def disable_copy_by_reference(self) -> bool:
         return self["disable_copy_by_reference"]
+
+    @property
+    def hash_bucket_count(self) -> int:
+        return self["hash_bucket_count"]
+
+    @property
+    def original_fields(self) -> Optional[Set[str]]:
+        return self.get("original_fields")
+
+    @property
+    def compacted_manifest(self) -> Optional[Manifest]:
+        return self.get("compacted_manifest")