deltacat 1.0.2__py3-none-any.whl → 1.1.1__py3-none-any.whl

deltacat/logs.py

@@ -2,7 +2,7 @@ import logging
 import os
 import pathlib
 from logging import FileHandler, Handler, Logger, LoggerAdapter, handlers
-from typing import Union
+from typing import Any, Dict, Optional, Union
 
 import ray
 from ray.runtime_context import RuntimeContext
@@ -26,7 +26,32 @@ DEFAULT_MAX_BYTES_PER_LOG = 2 ^ 20 * 256  # 256 MiB
 DEFAULT_BACKUP_COUNT = 0
 
 
-class RayRuntimeContextLoggerAdapter(logging.LoggerAdapter):
+class DeltaCATLoggerAdapter(logging.LoggerAdapter):
+    """
+    Logger Adapter class with additional functionality
+    """
+
+    def __init__(self, logger: Logger, extra: Optional[Dict[str, Any]] = {}):
+        super().__init__(logger, extra)
+
+    def debug_conditional(self, msg, do_print: bool, *args, **kwargs):
+        if do_print:
+            self.debug(msg, *args, **kwargs)
+
+    def info_conditional(self, msg, do_print: bool, *args, **kwargs):
+        if do_print:
+            self.info(msg, *args, **kwargs)
+
+    def warning_conditional(self, msg, do_print: bool, *args, **kwargs):
+        if do_print:
+            self.warning(msg, *args, **kwargs)
+
+    def error_conditional(self, msg, do_print: bool, *args, **kwargs):
+        if do_print:
+            self.error(msg, *args, **kwargs)
+
+
+class RayRuntimeContextLoggerAdapter(DeltaCATLoggerAdapter):
     """
     Logger Adapter for injecting Ray Runtime Context into logging messages.
     """
@@ -147,6 +172,8 @@ def _configure_logger(
         ray_runtime_ctx = ray.get_runtime_context()
         if ray_runtime_ctx.worker.connected:
             logger = RayRuntimeContextLoggerAdapter(logger, ray_runtime_ctx)
+    else:
+        logger = DeltaCATLoggerAdapter(logger)
 
     return logger
 

deltacat/storage/__init__.py

@@ -13,6 +13,8 @@ from deltacat.storage.model.partition import Partition, PartitionLocator
 from deltacat.storage.model.stream import Stream, StreamLocator
 from deltacat.storage.model.table import Table, TableLocator
 from deltacat.storage.model.table_version import TableVersion, TableVersionLocator
+from deltacat.storage.model.delete_parameters import DeleteParameters
+
 from deltacat.storage.model.types import (
     CommitState,
     DeltaType,
@@ -29,6 +31,7 @@ __all__ = [
     "Delta",
     "DeltaLocator",
     "Partition",
+    "DeleteParameters",
     "DeltaType",
     "DistributedDataset",
     "LifecycleState",

deltacat/storage/interface.py

@@ -3,6 +3,7 @@ from typing import Any, Callable, Dict, List, Optional, Set, Union
 import pyarrow as pa
 
 from deltacat.storage import (
+    DeleteParameters,
     Delta,
     DeltaLocator,
     DeltaType,
@@ -468,6 +469,7 @@ def stage_delta(
     properties: Optional[Dict[str, str]] = None,
     s3_table_writer_kwargs: Optional[Dict[str, Any]] = None,
     content_type: ContentType = ContentType.PARQUET,
+    delete_parameters: Optional[DeleteParameters] = None,
     *args,
     **kwargs
 ) -> Delta:

deltacat/storage/model/delete_parameters.py

@@ -0,0 +1,40 @@
+# Allow classes to use self-referencing Type hints in Python 3.7.
+from __future__ import annotations
+
+from typing import List, Optional
+
+
+class DeleteParameters(dict):
+    """
+    Contains all parameters required to support DELETEs
+    equality_column_names: List of column names that would be used to determine row equality for equality deletes.  Relevant only to equality deletes
+    """
+
+    @staticmethod
+    def of(
+        equality_column_names: Optional[List[str]] = None,
+    ) -> DeleteParameters:
+        delete_parameters = DeleteParameters()
+        if equality_column_names is not None:
+            delete_parameters["equality_column_names"] = equality_column_names
+        return delete_parameters
+
+    @property
+    def equality_column_names(self) -> Optional[List[str]]:
+        return self.get("equality_column_names")
+
+    @staticmethod
+    def merge_delete_parameters(
+        delete_parameters: List[DeleteParameters],
+    ) -> Optional[DeleteParameters]:
+        if len(delete_parameters) < 2:
+            return delete_parameters
+        equality_column_names = delete_parameters[0].equality_column_names
+        assert all(
+            delete_prev.equality_column_names == delete_curr.equality_column_names
+            for delete_prev, delete_curr in zip(
+                delete_parameters, delete_parameters[1:]
+            )
+        ), "We cannot merge two delete parameters if their equality column names are different."
+        merge_delete_parameters = DeleteParameters.of(equality_column_names)
+        return merge_delete_parameters

deltacat/storage/model/delta.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 from typing import Any, Dict, List, Optional
 
 from deltacat.aws.redshift import Manifest, ManifestAuthor, ManifestMeta
+from deltacat.storage.model.delete_parameters import DeleteParameters
 from deltacat.storage.model.locator import Locator
 from deltacat.storage.model.namespace import NamespaceLocator
 from deltacat.storage.model.partition import PartitionLocator
@@ -22,6 +23,7 @@ class Delta(dict):
         properties: Optional[Dict[str, str]],
         manifest: Optional[Manifest],
         previous_stream_position: Optional[int] = None,
+        delete_parameters: Optional[DeleteParameters] = None,
     ) -> Delta:
         """
         Creates a Delta metadata model with the given Delta Locator, Delta Type,
@@ -35,6 +37,7 @@ class Delta(dict):
         delta.properties = properties
         delta.manifest = manifest
         delta.previous_stream_position = previous_stream_position
+        delta.delete_parameters = delete_parameters
         return delta
 
     @staticmethod
@@ -68,7 +71,7 @@ class Delta(dict):
             raise ValueError("No deltas given to merge.")
         manifests = [d.manifest for d in deltas]
         if any(not m for m in manifests):
-            raise ValueError(f"Deltas to merge must have non-empty manifests.")
+            raise ValueError("Deltas to merge must have non-empty manifests.")
         distinct_storage_types = set([d.storage_type for d in deltas])
         if len(distinct_storage_types) > 1:
             raise NotImplementedError(
@@ -91,6 +94,15 @@ class Delta(dict):
             manifests,
             manifest_author,
         )
+        distinct_delta_type = list(distinct_delta_types)[0]
+        merged_delete_parameters = None
+        if distinct_delta_type is DeltaType.DELETE:
+            delete_parameters: List[DeleteParameters] = [
+                d.delete_parameters for d in deltas if d.delete_parameters
+            ]
+            merged_delete_parameters: Optional[
+                DeleteParameters
+            ] = DeleteParameters.merge_delete_parameters(delete_parameters)
         partition_locator = deltas[0].partition_locator
         prev_positions = [d.previous_stream_position for d in deltas]
         prev_position = None if None in prev_positions else max(prev_positions)
@@ -101,6 +113,7 @@ class Delta(dict):
             properties,
             merged_manifest,
             prev_position,
+            merged_delete_parameters,
         )
 
     @property
@@ -252,6 +265,17 @@ class Delta(dict):
             return delta_locator.stream_position
         return None
 
+    @property
+    def delete_parameters(self) -> Optional[DeleteParameters]:
+        delete_parameters = self.get("delete_parameters")
+        return (
+            None if delete_parameters is None else DeleteParameters(delete_parameters)
+        )
+
+    @delete_parameters.setter
+    def delete_parameters(self, delete_parameters: Optional[DeleteParameters]) -> None:
+        self["delete_parameters"] = delete_parameters
+
 
 class DeltaLocator(Locator, dict):
     @staticmethod