deltacat 1.0.2__py3-none-any.whl → 1.1.1__py3-none-any.whl

deltacat/__init__.py

@@ -44,7 +44,7 @@ from deltacat.types.tables import TableWriteMode
 
 deltacat.logs.configure_deltacat_logger(logging.getLogger(__name__))
 
-__version__ = "1.0.2"
+__version__ = "1.1.1"
 
 
 __all__ = [

deltacat/compute/compactor/model/compact_partition_params.py

@@ -20,6 +20,7 @@ from deltacat.compute.compactor_v2.constants import (
     AVERAGE_RECORD_SIZE_BYTES,
     TASK_MAX_PARALLELISM,
     DROP_DUPLICATES,
+    TOTAL_MEMORY_BUFFER_PERCENTAGE,
 )
 from deltacat.constants import PYARROW_INFLATION_MULTIPLIER
 from deltacat.compute.compactor.utils.sort_key import validate_sort_keys
@@ -57,6 +58,7 @@ class CompactPartitionParams(dict):
             "compacted_file_content_type", ContentType.PARQUET
         )
         result.object_store = params.get("object_store", RayPlasmaObjectStore())
+
         result.enable_profiler = params.get("enable_profiler", False)
         result.deltacat_storage = params.get(
             "deltacat_storage", unimplemented_deltacat_storage
@@ -84,12 +86,17 @@ class CompactPartitionParams(dict):
         result.average_record_size_bytes = params.get(
             "average_record_size_bytes", AVERAGE_RECORD_SIZE_BYTES
         )
+        result.total_memory_buffer_percentage = params.get(
+            "total_memory_buffer_percentage", TOTAL_MEMORY_BUFFER_PERCENTAGE
+        )
         result.hash_group_count = params.get(
             "hash_group_count", result.hash_bucket_count
         )
         result.drop_duplicates = params.get("drop_duplicates", DROP_DUPLICATES)
         result.ray_custom_resources = params.get("ray_custom_resources")
 
+        result.memory_logs_enabled = params.get("memory_logs_enabled", False)
+
         result.metrics_config = params.get("metrics_config")
 
         if not importlib.util.find_spec("memray"):
@@ -189,6 +196,16 @@ class CompactPartitionParams(dict):
     def average_record_size_bytes(self, average_record_size_bytes: float) -> None:
         self["average_record_size_bytes"] = average_record_size_bytes
 
+    @property
+    def total_memory_buffer_percentage(self) -> int:
+        return self["total_memory_buffer_percentage"]
+
+    @total_memory_buffer_percentage.setter
+    def total_memory_buffer_percentage(
+        self, total_memory_buffer_percentage: int
+    ) -> None:
+        self["total_memory_buffer_percentage"] = total_memory_buffer_percentage
+
     @property
     def min_files_in_batch(self) -> float:
         return self["min_files_in_batch"]
@@ -354,6 +371,14 @@ class CompactPartitionParams(dict):
     def sort_keys(self, keys: List[SortKey]) -> None:
         self["sort_keys"] = keys
 
+    @property
+    def memory_logs_enabled(self) -> bool:
+        return self.get("memory_logs_enabled")
+
+    @memory_logs_enabled.setter
+    def memory_logs_enabled(self, value: bool) -> None:
+        self["memory_logs_enabled"] = value
+
     @property
     def metrics_config(self) -> Optional[MetricsConfig]:
         return self.get("metrics_config")

deltacat/compute/compactor/model/compaction_session_audit_info.py

@@ -84,6 +84,13 @@ class CompactionSessionAuditInfo(dict):
         """
         return self.get("recordsDeduped")
 
+    @property
+    def records_deleted(self) -> int:
+        """
+        The total count of deleted records in a compaction session if delete deltas are present.
+        """
+        return self.get("recordsDeleted")
+
     @property
     def input_size_bytes(self) -> float:
         """
@@ -461,6 +468,10 @@ class CompactionSessionAuditInfo(dict):
         self["recordsDeduped"] = records_deduped
         return self
 
+    def set_records_deleted(self, records_deleted: int) -> CompactionSessionAuditInfo:
+        self["recordsDeleted"] = records_deleted
+        return self
+
     def set_input_size_bytes(
         self, input_size_bytes: float
     ) -> CompactionSessionAuditInfo:

deltacat/compute/compactor/model/delta_file_envelope.py

@@ -5,6 +5,9 @@ import numpy as np
 import pyarrow as pa
 
 from deltacat.storage import DeltaType, LocalTable
+from deltacat.compute.compactor.model.table_object_store import (
+    LocalTableStorageStrategy,
+)
 
 from typing import Optional
 
@@ -20,18 +23,21 @@ class DeltaFileEnvelope(dict):
         file_index: int = None,
         is_src_delta: np.bool_ = True,
         file_record_count: Optional[int] = None,
+        table_storage_strategy: [LocalTableStorageStrategy] = None,
     ) -> DeltaFileEnvelope:
-        """Static factory builder for a Delta File Envelope
+        """
+        Static factory builder for a Delta File Envelope
         `
         Args:
             stream_position: Stream position of a delta.
-            file_index: Manifest file index number of a delta.
             delta_type: A delta type.
             table: The table object that represents the delta file.
+            file_index: Manifest file index number of a delta.
             is_src_delta: True if this Delta File Locator is
                 pointing to a file from the uncompacted source table, False if
                 this Locator is pointing to a file in the compacted destination
                 table.
+            table_storage_strategy: The way the table object is stored in the delta file envelope. If None just stores the table normally
         Returns:
             A delta file envelope.
 
@@ -46,7 +52,11 @@ class DeltaFileEnvelope(dict):
         delta_file_envelope["streamPosition"] = stream_position
         delta_file_envelope["fileIndex"] = file_index
         delta_file_envelope["deltaType"] = delta_type.value
-        delta_file_envelope["table"] = table
+        if table_storage_strategy is None:
+            delta_file_envelope["table"] = table
+        else:
+            delta_file_envelope["table"] = table_storage_strategy.store_table(table)
+        delta_file_envelope["table_storage_strategy"] = table_storage_strategy
         delta_file_envelope["is_src_delta"] = is_src_delta
         delta_file_envelope["file_record_count"] = file_record_count
         return delta_file_envelope
@@ -63,8 +73,16 @@ class DeltaFileEnvelope(dict):
     def delta_type(self) -> DeltaType:
         return DeltaType(self["deltaType"])
 
+    @property
+    def table_storage_strategy(self) -> Optional[LocalTableStorageStrategy]:
+        return self["table_storage_strategy"]
+
     @property
     def table(self) -> LocalTable:
+        val = self.table_storage_strategy
+        if val is not None:
+            table_storage_strategy = val
+            return table_storage_strategy.get_table(self["table"])
         return self["table"]
 
     @property

deltacat/compute/compactor/model/table_object_store.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+from ray.types import ObjectRef
+
+from typing import Any, Union
+
+from abc import ABC, abstractmethod, abstractproperty
+from deltacat.io.ray_plasma_object_store import RayPlasmaObjectStore
+from deltacat.storage import (
+    LocalTable,
+)
+from deltacat.io.object_store import IObjectStore
+
+LocalTableReference = Union[ObjectRef, LocalTable]
+
+
+class LocalTableStorageStrategy(ABC):
+    @abstractproperty
+    def object_store(cls) -> IObjectStore:
+        pass
+
+    @abstractmethod
+    def store_table(self, table: LocalTable) -> LocalTableReference:
+        pass
+
+    @abstractmethod
+    def get_table(self, table_like: LocalTableReference) -> LocalTable:
+        pass
+
+
+class LocalTableRayObjectStoreReferenceStorageStrategy(LocalTableStorageStrategy):
+    """
+    Stores the table in the RayPlasmaObjectStore - see deltacat/io/ray_plasma_object_store.py
+    """
+
+    _object_store: IObjectStore = RayPlasmaObjectStore()
+
+    @property
+    def object_store(cls) -> IObjectStore:
+        return cls._object_store
+
+    def store_table(self, table: LocalTable) -> LocalTableReference:
+        obj_ref: ObjectRef = self.object_store.put(table)
+        return obj_ref
+
+    def get_table(self, table_like: LocalTableReference) -> LocalTable:
+        table = self.object_store.get(table_like)
+        return table
+
+    def get_table_reference(self, table_ref: Any) -> LocalTableReference:
+        return self.object_store.deserialize_references([table_ref])[0]

deltacat/compute/compactor/utils/io.py

@@ -378,7 +378,7 @@ def _discover_deltas(
     )
     deltas = deltas_list_result.all_items()
     if not deltas:
-        logger.warn(
+        logger.warning(
             f"Couldn't find any deltas to "
             f"compact in delta stream position range "
             f"('{start_position_exclusive}', "

deltacat/compute/compactor_v2/compaction_session.py

@@ -24,6 +24,16 @@ from deltacat.compute.compactor.model.materialize_result import MaterializeResul
 from deltacat.compute.compactor_v2.utils.merge import (
     generate_local_merge_input,
 )
+from deltacat.compute.compactor import DeltaAnnotated
+from deltacat.compute.compactor_v2.utils.delta import contains_delete_deltas
+from deltacat.compute.compactor_v2.deletes.delete_strategy import (
+    DeleteStrategy,
+)
+from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
+    DeleteFileEnvelope,
+)
+from deltacat.compute.compactor_v2.deletes.utils import prepare_deletes
+
 from deltacat.storage import (
     Delta,
     DeltaLocator,
@@ -52,6 +62,7 @@ from deltacat.utils.resources import (
 from deltacat.compute.compactor_v2.utils.task_options import (
     hash_bucket_resource_options_provider,
     merge_resource_options_provider,
+    local_merge_resource_options_provider,
 )
 from deltacat.compute.compactor.model.compactor_version import CompactorVersion
 
@@ -95,7 +106,7 @@ def compact_partition(params: CompactPartitionParams, **kwargs) -> Optional[str]
                 **params.s3_client_kwargs,
             )
         else:
-            logger.warn("No new partition was committed during compaction.")
+            logger.warning("No new partition was committed during compaction.")
 
         logger.info(
             f"Completed compaction session for: {params.source_partition_locator}"
@@ -149,7 +160,7 @@ def _execute_compaction(
         )
         if not round_completion_info:
             logger.info(
-                f"Both rebase partition and round completion file not found. Performing an entire backfill on source."
+                "Both rebase partition and round completion file not found. Performing an entire backfill on source."
             )
         else:
             compacted_delta_locator = round_completion_info.compacted_delta_locator
@@ -175,7 +186,7 @@ def _execute_compaction(
 
     delta_discovery_start = time.monotonic()
 
-    input_deltas = io.discover_deltas(
+    input_deltas: List[Delta] = io.discover_deltas(
         params.source_partition_locator,
         params.last_stream_position_to_compact,
         params.rebase_source_partition_locator,
@@ -185,8 +196,24 @@ def _execute_compaction(
         params.deltacat_storage_kwargs,
         params.list_deltas_kwargs,
     )
+    if not input_deltas:
+        logger.info("No input deltas found to compact.")
+        return None, None, None
 
-    uniform_deltas = io.create_uniform_input_deltas(
+    delete_strategy: Optional[DeleteStrategy] = None
+    delete_file_envelopes: Optional[List[DeleteFileEnvelope]] = None
+    delete_file_size_bytes: int = 0
+    if contains_delete_deltas(input_deltas):
+        input_deltas, delete_file_envelopes, delete_strategy = prepare_deletes(
+            params, input_deltas
+        )
+        for delete_file_envelope in delete_file_envelopes:
+            delete_file_size_bytes += delete_file_envelope.table_size_bytes
+        logger.info(
+            f" Input deltas contain DELETE-type deltas. Total delete file size={delete_file_size_bytes}."
+            f" Total length of delete file envelopes={len(delete_file_envelopes)}"
+        )
+    uniform_deltas: List[DeltaAnnotated] = io.create_uniform_input_deltas(
         input_deltas=input_deltas,
         hash_bucket_count=params.hash_bucket_count,
         compaction_audit=compaction_audit,
@@ -212,10 +239,6 @@ def _execute_compaction(
         **params.s3_client_kwargs,
     )
 
-    if not input_deltas:
-        logger.info("No input deltas found to compact.")
-        return None, None, None
-
     # create a new stream for this round
     compacted_stream_locator = params.destination_partition_locator.stream_locator
     compacted_stream = params.deltacat_storage.get_stream(
@@ -236,8 +259,10 @@ def _execute_compaction(
         resource_amount_provider=hash_bucket_resource_options_provider,
         previous_inflation=params.previous_inflation,
         average_record_size_bytes=params.average_record_size_bytes,
+        total_memory_buffer_percentage=params.total_memory_buffer_percentage,
         primary_keys=params.primary_keys,
         ray_custom_resources=params.ray_custom_resources,
+        memory_logs_enabled=params.memory_logs_enabled,
     )
 
     total_input_records_count = np.int64(0)
@@ -246,9 +271,36 @@ def _execute_compaction(
     if params.hash_bucket_count == 1:
         merge_start = time.monotonic()
         local_merge_input = generate_local_merge_input(
-            params, uniform_deltas, compacted_partition, round_completion_info
+            params,
+            uniform_deltas,
+            compacted_partition,
+            round_completion_info,
+            delete_strategy,
+            delete_file_envelopes,
+        )
+        estimated_da_bytes = (
+            compaction_audit.estimated_in_memory_size_bytes_during_discovery
+        )
+        estimated_num_records = sum(
+            [
+                entry.meta.record_count
+                for delta in uniform_deltas
+                for entry in delta.manifest.entries
+            ]
+        )
+        local_merge_options = local_merge_resource_options_provider(
+            estimated_da_size=estimated_da_bytes,
+            estimated_num_rows=estimated_num_records,
+            total_memory_buffer_percentage=params.total_memory_buffer_percentage,
+            round_completion_info=round_completion_info,
+            compacted_delta_manifest=previous_compacted_delta_manifest,
+            ray_custom_resources=params.ray_custom_resources,
+            primary_keys=params.primary_keys,
+            memory_logs_enabled=params.memory_logs_enabled,
+        )
+        local_merge_result = ray.get(
+            mg.merge.options(**local_merge_options).remote(local_merge_input)
         )
-        local_merge_result = ray.get(mg.merge.remote(local_merge_input))
         total_input_records_count += local_merge_result.input_record_count
         merge_results = [local_merge_result]
         merge_invoke_end = time.monotonic()
@@ -269,6 +321,7 @@ def _execute_compaction(
                     object_store=params.object_store,
                     deltacat_storage=params.deltacat_storage,
                     deltacat_storage_kwargs=params.deltacat_storage_kwargs,
+                    memory_logs_enabled=params.memory_logs_enabled,
                 )
             }
 
@@ -345,6 +398,9 @@ def _execute_compaction(
         )
 
         # BSP Step 2: Merge
+        # NOTE: DELETE-type deltas are stored in Plasma object store
+        # in prepare_deletes and therefore don't need to included
+        # in merge task resource estimation
         merge_options_provider = functools.partial(
             task_resource_options_provider,
             pg_config=params.pg_config,
@@ -352,12 +408,14 @@ def _execute_compaction(
             num_hash_groups=params.hash_group_count,
             hash_group_size_bytes=all_hash_group_idx_to_size_bytes,
             hash_group_num_rows=all_hash_group_idx_to_num_rows,
+            total_memory_buffer_percentage=params.total_memory_buffer_percentage,
             round_completion_info=round_completion_info,
             compacted_delta_manifest=previous_compacted_delta_manifest,
             primary_keys=params.primary_keys,
             deltacat_storage=params.deltacat_storage,
             deltacat_storage_kwargs=params.deltacat_storage_kwargs,
             ray_custom_resources=params.ray_custom_resources,
+            memory_logs_enabled=params.memory_logs_enabled,
         )
 
         def merge_input_provider(index, item):
@@ -385,6 +443,9 @@ def _execute_compaction(
                     object_store=params.object_store,
                     deltacat_storage=params.deltacat_storage,
                     deltacat_storage_kwargs=params.deltacat_storage_kwargs,
+                    delete_strategy=delete_strategy,
+                    delete_file_envelopes=delete_file_envelopes,
+                    memory_logs_enabled=params.memory_logs_enabled,
                 )
             }
 
@@ -406,7 +467,12 @@ def _execute_compaction(
     merge_end = time.monotonic()
 
     total_dd_record_count = sum([ddr.deduped_record_count for ddr in merge_results])
-    logger.info(f"Deduped {total_dd_record_count} records...")
+    total_deleted_record_count = sum(
+        [ddr.deleted_record_count for ddr in merge_results]
+    )
+    logger.info(
+        f"Deduped {total_dd_record_count} records and deleted {total_deleted_record_count} records..."
+    )
 
     compaction_audit.set_input_records(total_input_records_count.item())
 
@@ -419,7 +485,7 @@ def _execute_compaction(
     )
 
     compaction_audit.set_records_deduped(total_dd_record_count.item())
-
+    compaction_audit.set_records_deleted(total_deleted_record_count.item())
     mat_results = []
     for merge_result in merge_results:
         mat_results.extend(merge_result.materialize_results)
@@ -466,6 +532,7 @@ def _execute_compaction(
     record_info_msg = (
         f"Hash bucket records: {total_hb_record_count},"
         f" Deduped records: {total_dd_record_count}, "
+        f" Deleted records: {total_deleted_record_count}, "
         f" Materialized records: {merged_delta.meta.record_count}"
     )
     logger.info(record_info_msg)
@@ -526,7 +593,7 @@ def _execute_compaction(
     )
 
     # After all incremental delta related calculations, we update
-    # the input sizes to accomodate the compacted table
+    # the input sizes to accommodate the compacted table
     if round_completion_info:
         compaction_audit.set_input_file_count(
             (compaction_audit.input_file_count or 0)
@@ -565,7 +632,6 @@ def _execute_compaction(
         f"partition-{params.source_partition_locator.partition_values},"
         f"compacted at: {params.last_stream_position_to_compact},"
     )
-
     return (
         compacted_partition,
         new_round_completion_info,

deltacat/compute/compactor_v2/deletes/delete_file_envelope.py

@@ -0,0 +1,83 @@
+# Allow classes to use self-referencing Type hints in Python 3.7.
+from __future__ import annotations
+from typing import Any, List, Optional
+from deltacat.storage import DeltaType, LocalTable
+from deltacat.compute.compactor import (
+    DeltaFileEnvelope,
+)
+import numpy as np
+import pyarrow as pa
+
+from deltacat.compute.compactor.model.table_object_store import (
+    LocalTableStorageStrategy,
+    LocalTableRayObjectStoreReferenceStorageStrategy,
+)
+
+
+class DeleteFileEnvelope(DeltaFileEnvelope):
+    @staticmethod
+    def of(
+        stream_position: int,
+        delta_type: DeltaType,
+        table: LocalTable,
+        delete_columns: List[str],
+        file_index: int = None,
+        is_src_delta: np.bool_ = True,
+        file_record_count: Optional[int] = None,
+        table_storage_strategy: LocalTableStorageStrategy = LocalTableRayObjectStoreReferenceStorageStrategy(),
+    ) -> DeleteFileEnvelope:
+        """
+        Static factory builder for a DeleteFileEnvelope. Subclasses from DeltaFileEnvelope
+        `
+        Args:
+            stream_position: Stream position of a delta.
+            delta_type: A delta type.
+            table: The table object that represents the delta file.
+            delete_columns: delete column_names needed for equality-based deletes,
+            file_index: Manifest file index number of a delta.
+            is_src_delta: True if this Delta File Locator is
+                pointing to a file from the uncompacted source table, False if
+                this Locator is pointing to a file in the compacted destination
+                table.
+            table_storage_strategy: The way the table object is stored in the delta file envelope. Defaults to LocalTableRayObjectStoreReferenceStorageStrategy
+        Returns:
+            A delete file envelope.
+
+        """
+        delete_file_envelope = DeltaFileEnvelope.of(
+            stream_position,
+            delta_type,
+            table,
+            file_index,
+            is_src_delta,
+            file_record_count,
+            table_storage_strategy,
+        )
+        assert len(delete_columns) > 0, "At least 1 delete column is expected"
+        delete_file_envelope["delete_columns"] = delete_columns
+        if isinstance(table, pa.Table):
+            delete_file_envelope["table_size_bytes"] = table.nbytes
+        return DeleteFileEnvelope(**delete_file_envelope)
+
+    @property
+    def table_size_bytes(self) -> int:
+        val = self.get("table_size_bytes")
+        if val is not None:
+            return val
+        else:
+            raise ValueError(
+                f"Table type: {type(self.table)} not for supported for size method."
+            )
+
+    @property
+    def delete_columns(self) -> List[str]:
+        return self["delete_columns"]
+
+    @property
+    def table_reference(self) -> Optional[Any]:
+        if self.table_storage_strategy is not None and isinstance(
+            self.table_storage_strategy,
+            LocalTableRayObjectStoreReferenceStorageStrategy,
+        ):
+            return self.table_storage_strategy.get_table_reference(self["table"])
+        return None

deltacat/compute/compactor_v2/deletes/delete_strategy.py

@@ -0,0 +1,82 @@
+from typing import List, Optional
+
+import pyarrow as pa
+from abc import ABC, abstractmethod
+
+from typing import Tuple
+from deltacat.compute.compactor_v2.deletes.delete_file_envelope import (
+    DeleteFileEnvelope,
+)
+
+
+class DeleteStrategy(ABC):
+    """
+    Encapsulates a strategy for applying row-level deletes on tables during compaction
+
+    This abstract base class defines the interface for applying delete operations
+    on intermediate in-memory pyarrow tables during compaction. Concrete subclasses must implement the `apply_deletes` and
+    `apply_many_deletes` methods, as well as the `name` property.
+
+    Example:
+        >>> class MyDeleteStrategy(DeleteStrategy):
+        ...     @property
+        ...     def name(self) -> str:
+        ...         return "MyDeleteStrategy"
+        ...
+        ...     def apply_deletes(self, table: Optional[pa.Table], delete_file_envelope: DeleteFileEnvelope) -> ReturnTuple[pa.Table, int]:
+        ...         # Implement delete logic here
+        ...         pass
+        ...
+        ...     def apply_many_deletes(self, table: Optional[pa.Table], delete_file_envelopes: List[DeleteFileEnvelope]) -> ReturnTuple[pa.Table, int]:
+        ...         # Implement delete logic here
+        ...         pass
+    """
+
+    @property
+    def name(self) -> str:
+        """
+        The name of the delete strategy.
+        """
+        pass
+
+    @abstractmethod
+    def apply_deletes(
+        self,
+        table: Optional[pa.Table],
+        delete_file_envelope: DeleteFileEnvelope,
+        *args,
+        **kwargs,
+    ) -> Tuple[pa.Table, int]:
+        """
+        Apply delete operations on the given table using the provided delete file envelope.
+
+        Args:
+            table (Optional[pa.Table]): The pyarrow table to apply deletes on.
+            delete_file_envelope (DeleteFileEnvelope): The delete file envelope containing delete parameters.
+
+        Returns:
+            Tuple[pa.Table, int]: A tuple containing the updated Arrow table after applying deletes,
+                and the number of rows deleted.
+        """
+        pass
+
+    @abstractmethod
+    def apply_many_deletes(
+        self,
+        table: Optional[pa.Table],
+        delete_file_envelopes: List[DeleteFileEnvelope],
+        *args,
+        **kwargs,
+    ) -> Tuple[pa.Table, int]:
+        """
+        Apply delete operations on the given table using all provided delete file envelopes.
+
+        Args:
+            table (Optional[pa.Table]): The Arrow table to apply deletes on.
+            delete_file_envelopes (List[DeleteFileEnvelope]): A list of delete file envelopes containing delete parameters.
+
+        Returns:
+            Tuple[pa.Table, int]: A tuple containing the updated Arrow table after applying all deletes,
+                and the total number of rows deleted.
+        """
+        pass