deltacat 1.0.2__py3-none-any.whl → 1.1.1__py3-none-any.whl

deltacat/compute/compactor_v2/deletes/delete_strategy_equality_delete.py

@@ -0,0 +1,161 @@
+from typing import List, Optional
+import logging
+import pyarrow as pa
+from deltacat import logs
+
+from typing import Callable, Tuple
+from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
+    DeleteFileEnvelope,
+)
+from deltacat.compute.compactor_v2.deletes.delete_strategy import (
+    DeleteStrategy,
+)
+import pyarrow.compute as pc
+import numpy as np
+
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+class EqualityDeleteStrategy(DeleteStrategy):
+    """
+    A strategy for applying row-level deletes on tables during compaction based on equality conditions on one or more columns. It
+    implements the "equality delete" approach, which marks a row as deleted by one or more column values like pk1=3 or col1="foo", col2="bar".
+
+    Attributes:
+        _name (str): The name of the delete strategy.
+
+    Methods:
+        name(self) -> str:
+            Returns the name of the delete strategy.
+
+        apply_deletes(self, table, delete_file_envelope, *args, **kwargs) -> Tuple[pa.Table, int]:
+            Apply delete operations on the given table using the provided delete file envelope.
+
+        apply_many_deletes(self, table, delete_file_envelopes, *args, **kwargs) -> Tuple[pa.Table, int]:
+            Apply delete operations on the given table using all provided delete file envelopes.
+    """
+
+    _name = "EqualityDeleteStrategy"
+
+    @property
+    def name(self) -> str:
+        """
+        The name of the delete strategy.
+        """
+        return self._name
+
+    def _drop_rows(
+        self,
+        table: pa.Table,
+        delete_table: pa.Table,
+        delete_column_names: List[str],
+        equality_predicate_operation: Optional[Callable] = pa.compute.and_,
+    ) -> Tuple[pa.Table, int]:
+        """
+        Drop rows from the given table based on the provided delete table and column names.
+
+        Args:
+            table (pa.Table): The input table to drop rows from.
+            delete_table (pa.Table): The table containing the values to match for deletion.
+            delete_column_names (List[str]): A list of column names to check for equality with the delete table.
+            equality_predicate_operation (Optional[Callable], optional): The operation to combine equality predicates for multiple columns.
+                Defaults to pa.compute.and_.
+
+        Returns:
+            Tuple[pa.Table, int]: A tuple containing the updated table after dropping rows,
+                and the number of rows dropped.
+        """
+        if len(delete_column_names) < 1:
+            return table, 0
+        # all 1s -> all True so wont discard any from the curr_boolean_mask
+        prev_boolean_mask = pa.array(np.ones(len(table), dtype=bool))
+        # all 0s -> all False so if mask is never modified all rows will be kept
+        curr_boolean_mask = pa.array(np.zeros(len(table), dtype=bool))
+        for delete_column_name in delete_column_names:
+            if delete_column_name not in table.column_names:
+                logger.warning(
+                    f"Column name {delete_column_name} not in table column names. Skipping dropping rows for this column."
+                )
+                continue
+            curr_boolean_mask = pc.is_in(
+                table[delete_column_name],
+                value_set=delete_table[delete_column_name],
+            )
+            curr_boolean_mask = equality_predicate_operation(
+                prev_boolean_mask, curr_boolean_mask
+            )
+            prev_boolean_mask = curr_boolean_mask
+        number_of_rows_before_dropping = len(table)
+        logger.debug(
+            f"Number of table rows before dropping: {number_of_rows_before_dropping}."
+        )
+        table = table.filter(pc.invert(curr_boolean_mask))
+        number_of_rows_after_dropping = len(table)
+        logger.debug(
+            f"Number of table rows after dropping: {number_of_rows_after_dropping}."
+        )
+        dropped_rows = number_of_rows_before_dropping - number_of_rows_after_dropping
+        return table, dropped_rows
+
+    def apply_deletes(
+        self,
+        table: Optional[pa.Table],
+        delete_file_envelope: DeleteFileEnvelope,
+        *args,
+        **kwargs,
+    ) -> Tuple[pa.Table, int]:
+        """
+        Apply delete operations on the given table using the provided delete file envelope.
+
+        Args:
+            table (Optional[pa.Table]): The pyarrow table to apply deletes on.
+            delete_file_envelope (DeleteFileEnvelope): The delete file envelope containing delete parameters.
+
+        Returns:
+            Tuple[pa.Table, int]: A tuple containing the updated Arrow table after applying deletes,
+                and the number of rows deleted.
+        """
+        if not table or not delete_file_envelope.table:
+            logger.debug(
+                f"No table passed or no delete file envelope delete table found. DeleteFileEnvelope: {delete_file_envelope}"
+            )
+            return table, 0
+        delete_columns = delete_file_envelope.delete_columns
+        delete_table = delete_file_envelope.table
+        table, number_of_rows_dropped = self._drop_rows(
+            table, delete_table, delete_columns
+        )
+        return table, number_of_rows_dropped
+
+    def apply_many_deletes(
+        self,
+        table: Optional[pa.Table],
+        delete_file_envelopes: List[DeleteFileEnvelope],
+        *args,
+        **kwargs,
+    ) -> Tuple[pa.Table, int]:
+        """
+        Apply delete operations on the given table using all provided delete file envelopes.
+
+        Args:
+            table (Optional[pa.Table]): The pyarrow table to apply deletes on.
+            delete_file_envelopes (List[DeleteFileEnvelope]): A list of delete file envelopes containing delete parameters.
+
+        Returns:
+            Tuple[pa.Table, int]: A tuple containing the updated pyarrow table after applying all deletes,
+                and the total number of rows deleted.
+        """
+        # looking up table references are lighter than the actual table
+        if not table:
+            logger.debug("No table passed.")
+            return table, 0
+        total_dropped_rows = 0
+        for delete_file_envelope in delete_file_envelopes:
+            if delete_file_envelope.table_reference is None:
+                continue
+            table, number_of_rows_dropped = self.apply_deletes(
+                table, delete_file_envelope
+            )
+            total_dropped_rows += number_of_rows_dropped
+        return table, total_dropped_rows

deltacat/compute/compactor_v2/deletes/model.py

@@ -0,0 +1,23 @@
+from deltacat.storage import (
+    Delta,
+)
+
+from typing import List
+
+from dataclasses import dataclass, fields
+from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
+    DeleteFileEnvelope,
+)
+from deltacat.compute.compactor_v2.deletes.delete_strategy import (
+    DeleteStrategy,
+)
+
+
+@dataclass
+class PrepareDeleteResult:
+    non_delete_deltas: List[Delta]
+    delete_file_envelopes: List[DeleteFileEnvelope]
+    delete_strategy: DeleteStrategy
+
+    def __iter__(self):
+        return (getattr(self, field.name) for field in fields(self))

deltacat/compute/compactor_v2/deletes/utils.py

@@ -0,0 +1,164 @@
+import itertools
+
+from deltacat.compute.compactor_v2.deletes.model import (
+    PrepareDeleteResult,
+    DeleteFileEnvelope,
+)
+from deltacat.storage import (
+    DeltaType,
+)
+from collections import defaultdict
+import logging
+
+from typing import Optional, List, Dict, Tuple
+from deltacat.types.media import StorageType
+from deltacat.compute.compactor.model.compact_partition_params import (
+    CompactPartitionParams,
+)
+from deltacat.compute.compactor_v2.deletes.delete_strategy_equality_delete import (
+    EqualityDeleteStrategy,
+)
+import pyarrow as pa
+from deltacat.storage import (
+    Delta,
+)
+from deltacat import logs
+
+
+logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
+
+
+def _aggregate_delete_deltas(input_deltas: List[Delta]) -> Dict[int, List[Delta]]:
+    """
+    Aggregates consecutive DELETE deltas with the same delete parameters into groups.
+
+    Args:
+        input_deltas (List[Delta]): A list of Delta objects representing delete operations.
+    Returns:
+        Dict[int, List[Delta]]: A dictionary where the keys are the stream positions of the
+        earliest delta in each group of consecutive DELETE deltas with the same delete parameters,
+        and the values are lists containing those deltas.
+    """
+    start_stream_spos_to_delete_delta_sequence: Dict[int, List[Delta]] = defaultdict(
+        list
+    )
+    delete_deltas_sequence_grouped_by_delete_parameters: List[
+        Tuple[bool, List[Delta]]
+    ] = [
+        (is_delete, list(delete_delta_group))
+        for (is_delete, _), delete_delta_group in itertools.groupby(
+            input_deltas, lambda d: (d.type is DeltaType.DELETE, d.delete_parameters)
+        )
+    ]
+    for (
+        is_delete_delta_sequence,
+        delete_delta_sequence,
+    ) in delete_deltas_sequence_grouped_by_delete_parameters:
+        if not is_delete_delta_sequence:
+            continue
+        starting_stream_position_of_delete_sequence: int = delete_delta_sequence[
+            0
+        ].stream_position
+        start_stream_spos_to_delete_delta_sequence[
+            starting_stream_position_of_delete_sequence
+        ] = delete_delta_sequence
+    return start_stream_spos_to_delete_delta_sequence
+
+
+def _get_delete_file_envelopes(
+    params: CompactPartitionParams,
+    delete_spos_to_delete_deltas: Dict[int, List],
+) -> List[DeleteFileEnvelope]:
+    """
+    Create a list of DeleteFileEnvelope objects from the given dictionary of delete deltas.
+    Args:
+        params (CompactPartitionParams): compaction session parameters
+        delete_spos_to_delete_deltas (Dict[int, List]): A dictionary where the keys are the stream positions of the
+            earliest delta in each group of consecutive DELETE deltas with the same delete parameters,
+            and the values are lists containing those deltas.
+    Returns:
+        List[DeleteFileEnvelope]: A list of DeleteFileEnvelope objects.
+    """
+    delete_file_envelopes = []
+    for (
+        start_stream_position,
+        delete_delta_sequence,
+    ) in delete_spos_to_delete_deltas.items():
+        consecutive_delete_tables: List[pa.Table] = []
+        for delete_delta in delete_delta_sequence:
+            assert (
+                delete_delta.delete_parameters is not None
+            ), "Delete type deltas are required to have delete parameters defined"
+            delete_columns: Optional[
+                List[str]
+            ] = delete_delta.delete_parameters.equality_column_names
+            assert len(delete_columns) > 0, "At least 1 delete column is required"
+            # delete columns should exist in underlying table
+            delete_dataset = params.deltacat_storage.download_delta(
+                delete_delta,
+                file_reader_kwargs_provider=params.read_kwargs_provider,
+                columns=delete_columns,
+                storage_type=StorageType.LOCAL,
+                max_parallelism=1,
+                **params.deltacat_storage_kwargs,
+            )
+            consecutive_delete_tables.extend(delete_dataset)
+        delete_table: pa.Table = pa.concat_tables(consecutive_delete_tables)
+        delete_file_envelopes.append(
+            DeleteFileEnvelope.of(
+                start_stream_position,
+                delta_type=DeltaType.DELETE,
+                table=delete_table,
+                delete_columns=delete_columns,
+            )
+        )
+    return delete_file_envelopes
+
+
+def prepare_deletes(
+    params: CompactPartitionParams,
+    input_deltas: List[Delta],
+    *args,
+    **kwargs,
+) -> PrepareDeleteResult:
+    """
+    Prepares delete operations for a compaction process.
+
+    This function processes all the deltas and consolidates consecutive DELETE deltas.
+    It creates a list of these delete file envelopes.
+    Additionally, non-DELETE deltas are accumulated in a separate list.
+
+    Args:
+        params (CompactPartitionParams): Parameters for the compaction process.
+        input_deltas (List[Delta]): A list of Delta objects representing delete operations.
+
+    Returns:
+        PrepareDeleteResult:
+            - A list of Deltas excluding all DELETE operations.
+            - A list of DeleteFileEnvelope objects representing the consolidated delete operations.
+            - An instance of the EqualityDeleteStrategy class.
+
+    Raises:
+        AssertionError: If the input_deltas list is not sorted in non-decreasing order by stream_position.
+        AssertionError: If the number of delete file envelopes does not match the number of DELETE-type Delta sequences.
+    """
+    if not input_deltas:
+        return PrepareDeleteResult(input_deltas, [], None)
+    assert all(
+        input_deltas[i].stream_position <= input_deltas[i + 1].stream_position
+        for i in range(len(input_deltas) - 1)
+    ), "Uniform deltas must be in non-decreasing order by stream position"
+    start_stream_spos_to_delete_delta_sequence: Dict[
+        int, List[Delta]
+    ] = _aggregate_delete_deltas(input_deltas)
+    delete_file_envelopes: List[DeleteFileEnvelope] = _get_delete_file_envelopes(
+        params, start_stream_spos_to_delete_delta_sequence
+    )
+    assert len(start_stream_spos_to_delete_delta_sequence) == len(
+        delete_file_envelopes
+    ), "The number of delete file envelopes should match the number of DELETE-type Delta sequences"
+    return PrepareDeleteResult(
+        [delta for delta in input_deltas if delta.type is not DeltaType.DELETE],
+        delete_file_envelopes,
+        EqualityDeleteStrategy(),
+    )

deltacat/compute/compactor_v2/model/hash_bucket_input.py

@@ -22,6 +22,7 @@ class HashBucketInput(Dict):
         object_store: Optional[IObjectStore] = None,
         deltacat_storage=unimplemented_deltacat_storage,
         deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
+        memory_logs_enabled: Optional[bool] = None,
     ) -> HashBucketInput:
 
         result = HashBucketInput()
@@ -36,6 +37,7 @@ class HashBucketInput(Dict):
         result["object_store"] = object_store
         result["deltacat_storage"] = deltacat_storage
         result["deltacat_storage_kwargs"] = deltacat_storage_kwargs or {}
+        result["memory_logs_enabled"] = memory_logs_enabled
 
         return result
 
@@ -82,3 +84,7 @@ class HashBucketInput(Dict):
     @property
     def deltacat_storage_kwargs(self) -> Optional[Dict[str, Any]]:
         return self.get("deltacat_storage_kwargs")
+
+    @property
+    def memory_logs_enabled(self) -> Optional[bool]:
+        return self.get("memory_logs_enabled")

deltacat/compute/compactor_v2/model/merge_input.py

@@ -5,6 +5,9 @@ from typing import Dict, List, Optional, Any
 from deltacat.compute.compactor_v2.model.merge_file_group import (
     MergeFileGroupsProvider,
 )
+from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
+    DeleteFileEnvelope,
+)
 from deltacat.utils.metrics import MetricsConfig
 from deltacat.utils.common import ReadKwargsProvider
 from deltacat.io.object_store import IObjectStore
@@ -17,6 +20,7 @@ from deltacat.compute.compactor_v2.constants import (
     DROP_DUPLICATES,
     MAX_RECORDS_PER_COMPACTED_FILE,
 )
+from deltacat.compute.compactor_v2.deletes.delete_strategy import DeleteStrategy
 from deltacat.types.media import ContentType
 from deltacat.compute.compactor.model.round_completion_info import RoundCompletionInfo
 
@@ -38,8 +42,11 @@ class MergeInput(Dict):
         read_kwargs_provider: Optional[ReadKwargsProvider] = None,
         round_completion_info: Optional[RoundCompletionInfo] = None,
         object_store: Optional[IObjectStore] = None,
+        delete_strategy: Optional[DeleteStrategy] = None,
+        delete_file_envelopes: Optional[List] = None,
         deltacat_storage=unimplemented_deltacat_storage,
         deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
+        memory_logs_enabled: Optional[bool] = None,
     ) -> MergeInput:
 
         result = MergeInput()
@@ -57,9 +64,11 @@ class MergeInput(Dict):
         result["read_kwargs_provider"] = read_kwargs_provider
         result["round_completion_info"] = round_completion_info
         result["object_store"] = object_store
+        result["delete_file_envelopes"] = delete_file_envelopes
+        result["delete_strategy"] = delete_strategy
         result["deltacat_storage"] = deltacat_storage
         result["deltacat_storage_kwargs"] = deltacat_storage_kwargs or {}
-
+        result["memory_logs_enabled"] = memory_logs_enabled
         return result
 
     @property
@@ -125,3 +134,17 @@ class MergeInput(Dict):
     @property
     def deltacat_storage_kwargs(self) -> Optional[Dict[str, Any]]:
         return self.get("deltacat_storage_kwargs")
+
+    @property
+    def memory_logs_enabled(self) -> Optional[bool]:
+        return self.get("memory_logs_enabled")
+
+    @property
+    def delete_file_envelopes(
+        self,
+    ) -> Optional[List[DeleteFileEnvelope]]:
+        return self.get("delete_file_envelopes")
+
+    @property
+    def delete_strategy(self) -> Optional[DeleteStrategy]:
+        return self.get("delete_strategy")

deltacat/compute/compactor_v2/model/merge_result.py

@@ -8,6 +8,7 @@ class MergeResult(NamedTuple):
     materialize_results: List[MaterializeResult]
     input_record_count: np.int64
     deduped_record_count: np.int64
+    deleted_record_count: np.int64
     peak_memory_usage_bytes: np.double
     telemetry_time_in_seconds: np.double
     task_completed_at: np.double

deltacat/compute/compactor_v2/steps/hash_bucket.py

@@ -70,14 +70,12 @@ def _group_file_records_by_pk_hash_bucket(
     hb_to_delta_file_envelopes = np.empty([num_hash_buckets], dtype="object")
     for dfe in delta_file_envelopes:
         logger.info("Grouping by pk hash bucket")
-        start = time.monotonic()
+        group_start = time.monotonic()
         hash_bucket_to_table = group_by_pk_hash_bucket(
-            dfe.table,
-            num_hash_buckets,
-            primary_keys,
+            dfe.table, num_hash_buckets, primary_keys
         )
         group_end = time.monotonic()
-        logger.info(f"Grouping took: {group_end - start}")
+        logger.info(f"Grouping took: {group_end - group_start}")
         for hb, table in enumerate(hash_bucket_to_table):
             if table:
                 if hb_to_delta_file_envelopes[hb] is None:
@@ -144,7 +142,8 @@ def hash_bucket(input: HashBucketInput) -> HashBucketResult:
                 f"({process_util.max_memory/BYTES_PER_GIBIBYTE} GB)"
             )
 
-        process_util.schedule_callback(log_peak_memory, 10)
+        if input.memory_logs_enabled:
+            process_util.schedule_callback(log_peak_memory, 10)
 
         hash_bucket_result, duration = timed_invocation(
             func=_timed_hash_bucket, input=input