deltacat 0.1.10.dev0__py3-none-any.whl → 0.1.12__py3-none-any.whl

deltacat/compute/compactor/model/delta_annotated.py

@@ -2,21 +2,26 @@
 from __future__ import annotations
 
 import logging
-from deltacat import logs
-from deltacat.storage import DeltaType, Manifest, ManifestEntry, \
-    ManifestEntryList
-from typing import List, Optional, Callable, Union
 from types import FunctionType
-from deltacat.storage import Delta
+from typing import Callable, List, Optional, Union
+
+from deltacat import logs
+from deltacat.storage import (
+    Delta,
+    DeltaType,
+    Manifest,
+    ManifestEntry,
+    ManifestEntryList,
+)
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 class DeltaAnnotation(tuple):
     @staticmethod
-    def of(file_index: int,
-           delta_type: DeltaType,
-           stream_position: int) -> DeltaAnnotation:
+    def of(
+        file_index: int, delta_type: DeltaType, stream_position: int
+    ) -> DeltaAnnotation:
         return DeltaAnnotation((file_index, delta_type, stream_position))
 
     @property
@@ -48,17 +53,19 @@ class DeltaAnnotated(Delta):
         if entries:
             dtype = delta.type
             pos = delta.stream_position
-            _annotations = [DeltaAnnotation.of(i, dtype, pos) for i in
-                            range(len(entries))]
+            _annotations = [
+                DeltaAnnotation.of(i, dtype, pos) for i in range(len(entries))
+            ]
         delta_annotated.annotations = _annotations
         return delta_annotated
 
     @staticmethod
     def rebatch(
-            annotated_deltas: List[DeltaAnnotated],
-            min_delta_bytes,
-            min_file_counts: Optional[Union[int, float]]=float("inf"),
-            estimation_function: Optional[Callable]=None) -> List[DeltaAnnotated]:
+        annotated_deltas: List[DeltaAnnotated],
+        min_delta_bytes,
+        min_file_counts: Optional[Union[int, float]] = float("inf"),
+        estimation_function: Optional[Callable] = None,
+    ) -> List[DeltaAnnotated]:
         """
         Simple greedy algorithm to split/merge 1 or more annotated deltas into
         size-limited annotated deltas. All ordered manifest entries in the input
@@ -75,30 +82,38 @@ class DeltaAnnotated(Delta):
         for src_da in annotated_deltas:
             src_da_annotations = src_da.annotations
             src_da_entries = src_da.manifest.entries
-            assert(len(src_da_annotations) == len(src_da_entries),
-                   f"Unexpected Error: Length of delta annotations "
-                   f"({len(src_da_annotations)}) doesn't mach the length of "
-                   f"delta manifest entries ({len(src_da_entries)}).")
+            assert (
+                len(src_da_annotations) == len(src_da_entries),
+                f"Unexpected Error: Length of delta annotations "
+                f"({len(src_da_annotations)}) doesn't mach the length of "
+                f"delta manifest entries ({len(src_da_entries)}).",
+            )
             for i, src_entry in enumerate(src_da_entries):
                 DeltaAnnotated._append_annotated_entry(
-                    src_da,
-                    new_da,
-                    src_entry,
-                    src_da_annotations[i])
+                    src_da, new_da, src_entry, src_da_annotations[i]
+                )
                 # TODO: Fetch s3_obj["Size"] if entry content length undefined?
-                estimated_new_da_bytes = estimation_function(src_entry.meta.content_length) if type(
-                    estimation_function) is FunctionType else src_entry.meta.content_length
+                estimated_new_da_bytes = (
+                    estimation_function(src_entry.meta.content_length)
+                    if type(estimation_function) is FunctionType
+                    else src_entry.meta.content_length
+                )
                 new_da_bytes += estimated_new_da_bytes
                 da_group_entry_count += 1
-                if new_da_bytes >= min_delta_bytes or da_group_entry_count >= min_file_counts:
+                if (
+                    new_da_bytes >= min_delta_bytes
+                    or da_group_entry_count >= min_file_counts
+                ):
                     if new_da_bytes >= min_delta_bytes:
                         logger.info(
                             f"Appending group of {da_group_entry_count} elements "
-                            f"and {new_da_bytes} bytes to meet file size limit")
+                            f"and {new_da_bytes} bytes to meet file size limit"
+                        )
                     if da_group_entry_count >= min_file_counts:
                         logger.info(
                             f"Appending group of {da_group_entry_count} elements "
-                            f"and {da_group_entry_count} files to meet file count limit")
+                            f"and {da_group_entry_count} files to meet file count limit"
+                        )
                     groups.append(new_da)
                     new_da = DeltaAnnotated()
                     new_da_bytes = 0
@@ -108,32 +123,31 @@ class DeltaAnnotated(Delta):
         return groups
 
     @staticmethod
-    def split(
-            src_da: DeltaAnnotated,
-            pieces: int) -> List[DeltaAnnotated]:
+    def split(src_da: DeltaAnnotated, pieces: int) -> List[DeltaAnnotated]:
         groups = []
         new_da = DeltaAnnotated()
         da_group_entry_count = 0
         src_da_annotations = src_da.annotations
         src_da_entries = src_da.manifest.entries
-        assert (len(src_da_annotations) == len(src_da_entries),
-                f"Unexpected Error: Length of delta annotations "
-                f"({len(src_da_annotations)}) doesn't mach the length of "
-                f"delta manifest entries ({len(src_da_entries)}).")
+        assert (
+            len(src_da_annotations) == len(src_da_entries),
+            f"Unexpected Error: Length of delta annotations "
+            f"({len(src_da_annotations)}) doesn't mach the length of "
+            f"delta manifest entries ({len(src_da_entries)}).",
+        )
         src_da_entries_length = len(src_da_entries)
         equal_length = src_da_entries_length // pieces
         for i in range(len(src_da_entries)):
             DeltaAnnotated._append_annotated_entry(
-                src_da,
-                new_da,
-                src_da_entries[i],
-                src_da_annotations[i])
+                src_da, new_da, src_da_entries[i], src_da_annotations[i]
+            )
             # TODO: Fetch s3_obj["Size"] if entry content length undefined?
             da_group_entry_count += 1
             if da_group_entry_count >= equal_length and i < equal_length * (pieces - 1):
                 logger.info(
                     f"Splitting {da_group_entry_count} manifest files "
-                    f"to {pieces} pieces of {equal_length} size.")
+                    f"to {pieces} pieces of {equal_length} size."
+                )
                 groups.append(new_da)
                 new_da = DeltaAnnotated()
                 da_group_entry_count = 0
@@ -141,7 +155,8 @@ class DeltaAnnotated(Delta):
                 groups.append(new_da)
                 logger.info(
                     f"Splitting {da_group_entry_count} manifest files "
-                    f"to {pieces} pieces of {equal_length} size.")
+                    f"to {pieces} pieces of {equal_length} size."
+                )
                 new_da = DeltaAnnotated()
         if new_da:
             groups.append(new_da)
@@ -157,10 +172,11 @@ class DeltaAnnotated(Delta):
 
     @staticmethod
     def _append_annotated_entry(
-            src_da: DeltaAnnotated,
-            dst_da: DeltaAnnotated,
-            src_entry: ManifestEntry,
-            src_annotation: DeltaAnnotation):
+        src_da: DeltaAnnotated,
+        dst_da: DeltaAnnotated,
+        src_entry: ManifestEntry,
+        src_annotation: DeltaAnnotation,
+    ):
 
         if not dst_da:
             # copy all extended properties from the source delta manifest impl

deltacat/compute/compactor/model/delta_file_envelope.py

@@ -2,19 +2,18 @@
 from __future__ import annotations
 
 import numpy as np
-from deltacat.storage import DeltaType, LocalTable
 
+from deltacat.storage import DeltaType, LocalTable
 
 DeltaFileEnvelopeGroups = np.ndarray
 
 
 class DeltaFileEnvelope(dict):
     @staticmethod
-    def of(stream_position: int,
-           file_index: int,
-           delta_type: DeltaType,
-           table: LocalTable) -> DeltaFileEnvelope:
-        """ Static factory builder for a Delta File Envelope
+    def of(
+        stream_position: int, file_index: int, delta_type: DeltaType, table: LocalTable
+    ) -> DeltaFileEnvelope:
+        """Static factory builder for a Delta File Envelope
         `
         Args:
             stream_position: Stream position of a delta.

deltacat/compute/compactor/model/delta_file_locator.py

@@ -8,9 +8,9 @@ from deltacat.storage import Locator
 
 class DeltaFileLocator(Locator, tuple):
     @staticmethod
-    def of(is_src_delta: np.bool_,
-           stream_position: np.int64,
-           file_index: np.int32) -> DeltaFileLocator:
+    def of(
+        is_src_delta: np.bool_, stream_position: np.int64, file_index: np.int32
+    ) -> DeltaFileLocator:
         """
         Create a Delta File Locator tuple that can be used to uniquely identify
         and retrieve a file from any compaction job run input Delta.
@@ -30,11 +30,13 @@ class DeltaFileLocator(Locator, tuple):
             delta_file_locator: The Delta File Locator Tuple as
             (is_source_delta, stream_position, file_index).
         """
-        return DeltaFileLocator((
-            is_src_delta,
-            stream_position,
-            file_index,
-        ))
+        return DeltaFileLocator(
+            (
+                is_src_delta,
+                stream_position,
+                file_index,
+            )
+        )
 
     @property
     def is_source_delta(self) -> np.bool_:

deltacat/compute/compactor/model/materialize_result.py

@@ -1,18 +1,17 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-from deltacat.storage import Delta
-from deltacat.compute.compactor.model.pyarrow_write_result import \
-    PyArrowWriteResult
-
 from typing import Any, Dict
 
+from deltacat.compute.compactor.model.pyarrow_write_result import PyArrowWriteResult
+from deltacat.storage import Delta
+
 
 class MaterializeResult(dict):
     @staticmethod
-    def of(delta: Delta,
-           task_index: int,
-           pyarrow_write_result: PyArrowWriteResult) -> MaterializeResult:
+    def of(
+        delta: Delta, task_index: int, pyarrow_write_result: PyArrowWriteResult
+    ) -> MaterializeResult:
         materialize_result = MaterializeResult()
         materialize_result["delta"] = delta
         materialize_result["taskIndex"] = task_index

deltacat/compute/compactor/model/primary_key_index.py

@@ -1,19 +1,17 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
+from typing import Any, Dict, List
 from uuid import uuid4
 
 from deltacat.compute.compactor.model.sort_key import SortKey
 from deltacat.storage import Locator, PartitionLocator
 from deltacat.utils.common import sha1_hexdigest
 
-from typing import Any, Dict, List
-
 
 class PrimaryKeyIndexLocator(Locator, dict):
     @staticmethod
-    def of(primary_key_index_meta: PrimaryKeyIndexMeta) \
-            -> PrimaryKeyIndexLocator:
+    def of(primary_key_index_meta: PrimaryKeyIndexMeta) -> PrimaryKeyIndexLocator:
         """
         Creates a Primary Key Index Locator from the given Primary Key
         Index Metadata. A Primary Key Index Locator consists of a Primary Key
@@ -33,13 +31,16 @@ class PrimaryKeyIndexLocator(Locator, dict):
 
     @staticmethod
     def _root_path(
-            compacted_partition_locator: PartitionLocator,
-            primary_keys: List[str],
-            sort_keys: List[SortKey],
-            primary_key_index_algorithm_version: str) -> str:
+        compacted_partition_locator: PartitionLocator,
+        primary_keys: List[str],
+        sort_keys: List[SortKey],
+        primary_key_index_algorithm_version: str,
+    ) -> str:
         pl_hexdigest = compacted_partition_locator.hexdigest()
-        pki_version_str = f"{pl_hexdigest}|{primary_keys}|{sort_keys}|" \
-                          f"{primary_key_index_algorithm_version}"
+        pki_version_str = (
+            f"{pl_hexdigest}|{primary_keys}|{sort_keys}|"
+            f"{primary_key_index_algorithm_version}"
+        )
         return sha1_hexdigest(pki_version_str.encode("utf-8"))
 
     @property
@@ -76,10 +77,12 @@ class PrimaryKeyIndexLocator(Locator, dict):
 
 class PrimaryKeyIndexMeta(dict):
     @staticmethod
-    def of(compacted_partition_locator: PartitionLocator,
-           primary_keys: List[str],
-           sort_keys: List[SortKey],
-           primary_key_index_algo_version: str) -> PrimaryKeyIndexMeta:
+    def of(
+        compacted_partition_locator: PartitionLocator,
+        primary_keys: List[str],
+        sort_keys: List[SortKey],
+        primary_key_index_algo_version: str,
+    ) -> PrimaryKeyIndexMeta:
         """
         Creates Primary Key Index Metadata from the given compacted
         Partition Locator, primary keys, sort keys, and primary key index
@@ -114,8 +117,10 @@ class PrimaryKeyIndexMeta(dict):
 
 class PrimaryKeyIndexVersionLocator(Locator, dict):
     @staticmethod
-    def of(primary_key_index_version_meta: PrimaryKeyIndexVersionMeta,
-           pki_version_root_path: str) -> PrimaryKeyIndexVersionLocator:
+    def of(
+        primary_key_index_version_meta: PrimaryKeyIndexVersionMeta,
+        pki_version_root_path: str,
+    ) -> PrimaryKeyIndexVersionLocator:
         """
         Creates a primary key index version locator from the given primary key
         index version metadata and version root path. Note that, while this is
@@ -129,8 +134,9 @@ class PrimaryKeyIndexVersionLocator(Locator, dict):
         return pkivl
 
     @staticmethod
-    def generate(pki_version_meta: PrimaryKeyIndexVersionMeta) \
-            -> PrimaryKeyIndexVersionLocator:
+    def generate(
+        pki_version_meta: PrimaryKeyIndexVersionMeta,
+    ) -> PrimaryKeyIndexVersionLocator:
         """
         Creates a new primary key index version locator from the given primary
         key index version metadata. A primary key index version locator
@@ -142,11 +148,12 @@ class PrimaryKeyIndexVersionLocator(Locator, dict):
         deterministically from the compacted partition locator, primary keys,
         sort keys, and primary key index algorithm version.
         """
-        pki_version_root_path = PrimaryKeyIndexVersionLocator.\
-            _generate_version_root_path(
+        pki_version_root_path = (
+            PrimaryKeyIndexVersionLocator._generate_version_root_path(
                 PrimaryKeyIndexVersionLocator._pki_root_path(pki_version_meta),
                 pki_version_meta.hash_bucket_count,
             )
+        )
         pkivl = PrimaryKeyIndexVersionLocator()
         pkivl["primaryKeyIndexVersionMeta"] = pki_version_meta
         pkivl["primaryKeyIndexVersionRootPath"] = pki_version_root_path
@@ -159,17 +166,14 @@ class PrimaryKeyIndexVersionLocator(Locator, dict):
         return pki_locator.primary_key_index_root_path
 
     @staticmethod
-    def _generate_version_root_path(
-            pki_root_path: str,
-            hash_bucket_count: int) -> str:
+    def _generate_version_root_path(pki_root_path: str, hash_bucket_count: int) -> str:
         return f"{pki_root_path}/{hash_bucket_count}/{str(uuid4())}"
 
     @property
     def primary_key_index_version_meta(self) -> PrimaryKeyIndexVersionMeta:
         val: Dict[str, Any] = self.get("primaryKeyIndexVersionMeta")
         if val is not None and not isinstance(val, PrimaryKeyIndexVersionMeta):
-            self["primaryKeyIndexVersionMeta"] = val = \
-                PrimaryKeyIndexVersionMeta(val)
+            self["primaryKeyIndexVersionMeta"] = val = PrimaryKeyIndexVersionMeta(val)
         return val
 
     @property
@@ -179,7 +183,8 @@ class PrimaryKeyIndexVersionLocator(Locator, dict):
         index version locator.
         """
         return PrimaryKeyIndexVersionLocator._pki_root_path(
-            self.primary_key_index_version_meta)
+            self.primary_key_index_version_meta
+        )
 
     @property
     def primary_key_index_version_root_path(self) -> str:
@@ -206,9 +211,8 @@ class PrimaryKeyIndexVersionLocator(Locator, dict):
         return f"{pkiv_root_path}/{hb_index}"
 
     def get_pkiv_hb_index_s3_url_base(
-            self,
-            s3_bucket: str,
-            hash_bucket_index: int) -> str:
+        self, s3_bucket: str, hash_bucket_index: int
+    ) -> str:
         """
         Gets the base S3 URL of a single hash bucket of the given primary key
         index version locator.
@@ -217,9 +221,8 @@ class PrimaryKeyIndexVersionLocator(Locator, dict):
         return f"s3://{s3_bucket}/{hbi_root_path}"
 
     def get_pkiv_hb_index_manifest_s3_url(
-            self,
-            s3_bucket: str,
-            hash_bucket_index: int) -> str:
+        self, s3_bucket: str, hash_bucket_index: int
+    ) -> str:
         """
         Gets the S3 URL of the manifest for a single primary key index version
         hash bucket.
@@ -241,8 +244,9 @@ class PrimaryKeyIndexVersionLocator(Locator, dict):
 
 class PrimaryKeyIndexVersionMeta(dict):
     @staticmethod
-    def of(primary_key_index_meta: PrimaryKeyIndexMeta,
-           hash_bucket_count: int) -> PrimaryKeyIndexVersionMeta:
+    def of(
+        primary_key_index_meta: PrimaryKeyIndexMeta, hash_bucket_count: int
+    ) -> PrimaryKeyIndexVersionMeta:
         """
         Creates Primary Key Index Version Metadata from the given Primary Key
         Index Metadata and hash bucket count.

deltacat/compute/compactor/model/pyarrow_write_result.py

@@ -6,10 +6,9 @@ from typing import List
 
 class PyArrowWriteResult(dict):
     @staticmethod
-    def of(file_count: int,
-           pyarrow_bytes: int,
-           file_bytes: int,
-           record_count: int) -> PyArrowWriteResult:
+    def of(
+        file_count: int, pyarrow_bytes: int, file_bytes: int, record_count: int
+    ) -> PyArrowWriteResult:
         pawr = PyArrowWriteResult()
         pawr["files"] = file_count
         pawr["paBytes"] = pyarrow_bytes

deltacat/compute/compactor/model/round_completion_info.py

@@ -1,24 +1,26 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-from deltacat.storage import DeltaLocator
-from deltacat.compute.compactor.model.pyarrow_write_result import \
-    PyArrowWriteResult
-from deltacat.compute.compactor.model.primary_key_index import \
-    PrimaryKeyIndexVersionLocator
+from typing import Any, Dict, Optional
 
-from typing import Any, Dict
+from deltacat.compute.compactor.model.primary_key_index import (
+    PrimaryKeyIndexVersionLocator,
+)
+from deltacat.compute.compactor.model.pyarrow_write_result import PyArrowWriteResult
+from deltacat.storage import DeltaLocator, PartitionLocator
 
 
 class RoundCompletionInfo(dict):
     @staticmethod
-    def of(high_watermark: int,
-           compacted_delta_locator: DeltaLocator,
-           compacted_pyarrow_write_result: PyArrowWriteResult,
-           pk_index_pyarrow_write_result: PyArrowWriteResult,
-           sort_keys_bit_width: int,
-           primary_key_index_version_locator: PrimaryKeyIndexVersionLocator) \
-            -> RoundCompletionInfo:
+    def of(
+        high_watermark: int,
+        compacted_delta_locator: DeltaLocator,
+        compacted_pyarrow_write_result: PyArrowWriteResult,
+        pk_index_pyarrow_write_result: PyArrowWriteResult,
+        sort_keys_bit_width: int,
+        primary_key_index_version_locator: PrimaryKeyIndexVersionLocator,
+        rebase_source_partition_locator: Optional[PartitionLocator],
+    ) -> RoundCompletionInfo:
 
         rci = RoundCompletionInfo()
         rci["highWatermark"] = high_watermark
@@ -27,6 +29,7 @@ class RoundCompletionInfo(dict):
         rci["pkIndexPyarrowWriteResult"] = pk_index_pyarrow_write_result
         rci["sortKeysBitWidth"] = sort_keys_bit_width
         rci["primaryKeyIndexVersionLocator"] = primary_key_index_version_locator
+        rci["rebaseSourcePartitionLocator"] = rebase_source_partition_locator
         return rci
 
     @property
@@ -59,11 +62,14 @@ class RoundCompletionInfo(dict):
         return self["sortKeysBitWidth"]
 
     @property
-    def primary_key_index_version_locator(self) \
-            -> PrimaryKeyIndexVersionLocator:
+    def primary_key_index_version_locator(self) -> PrimaryKeyIndexVersionLocator:
         val: Dict[str, Any] = self.get("primaryKeyIndexVersionLocator")
-        if val is not None \
-                and not isinstance(val, PrimaryKeyIndexVersionLocator):
-            self["primaryKeyIndexVersionLocator"] = val = \
-                PrimaryKeyIndexVersionLocator(val)
+        if val is not None and not isinstance(val, PrimaryKeyIndexVersionLocator):
+            self["primaryKeyIndexVersionLocator"] = val = PrimaryKeyIndexVersionLocator(
+                val
+            )
         return val
+
+    @property
+    def rebase_source_partition_locator(self) -> Optional[PartitionLocator]:
+        return self.get("rebaseSourcePartitionLocator")

deltacat/compute/compactor/model/sort_key.py

@@ -2,14 +2,13 @@
 from __future__ import annotations
 
 import logging
-import pyarrow as pa
-
 from enum import Enum
+from typing import List
 
-from deltacat.storage import PartitionLocator
-from deltacat import logs
+import pyarrow as pa
 
-from typing import List
+from deltacat import logs
+from deltacat.storage import PartitionLocator
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
@@ -23,8 +22,7 @@ class SortOrder(str, Enum):
 
 class SortKey(tuple):
     @staticmethod
-    def of(key_name: str, sort_order: SortOrder = SortOrder.ASCENDING) \
-            -> SortKey:
+    def of(key_name: str, sort_order: SortOrder = SortOrder.ASCENDING) -> SortKey:
         """
         Create a sort key from a field name to use as the sort key, and
         the sort order for this key. If no sort order is specified, then the
@@ -39,9 +37,10 @@ class SortKey(tuple):
 
     @staticmethod
     def validate_sort_keys(
-            source_partition_locator: PartitionLocator,
-            sort_keys: List[SortKey],
-            deltacat_storage) -> int:
+        source_partition_locator: PartitionLocator,
+        sort_keys: List[SortKey],
+        deltacat_storage,
+    ) -> int:
         """
         Validates the input sort keys to ensure that they are unique, are using
         a valid sort key model, are all fixed-width data types, and that the
@@ -51,8 +50,9 @@ class SortKey(tuple):
         total_sort_keys_bit_width = 0
         if sort_keys:
             sort_key_names = [key.key_name for key in sort_keys]
-            assert len(sort_key_names) == len(set(sort_key_names)), \
-                f"Sort key names must be unique: {sort_key_names}"
+            assert len(sort_key_names) == len(
+                set(sort_key_names)
+            ), f"Sort key names must be unique: {sort_key_names}"
             stream_locator = source_partition_locator.stream_locator
             table_version_schema = deltacat_storage.get_table_version_schema(
                 stream_locator.namespace,
@@ -70,19 +70,22 @@ class SortKey(tuple):
                                 f"Total length of sort keys "
                                 f"({total_sort_keys_bit_width}) is greater "
                                 f"than the max supported bit width for all "
-                                f"sort keys ({MAX_SORT_KEYS_BIT_WIDTH})")
+                                f"sort keys ({MAX_SORT_KEYS_BIT_WIDTH})"
+                            )
                     except ValueError as e:
                         raise ValueError(
                             f"Unable to get bit width of sort key: {pa_field}. "
                             f"Please ensure that all sort keys are fixed-size "
-                            f"PyArrow data types.") from e
+                            f"PyArrow data types."
+                        ) from e
             else:
                 logger.warning(
                     f"Unable to estimate sort key bit width for schema type "
                     f"{type(table_version_schema)}. This compaction job run "
                     f"may run out of memory, run more slowly, or underutilize "
                     f"available resources. To fix this, either remove the "
-                    f"sort keys or provide a PyArrow schema.")
+                    f"sort keys or provide a PyArrow schema."
+                )
                 total_sort_keys_bit_width = MAX_SORT_KEYS_BIT_WIDTH
         return total_sort_keys_bit_width