deltacat 0.1.10.dev0__py3-none-any.whl → 0.1.12__py3-none-any.whl

deltacat/compute/stats/basic.py

@@ -1,30 +1,33 @@
-import ray
-from typing import Dict, Set, Tuple, List, Optional
+from typing import Dict, List, Optional, Set, Tuple
 
-from deltacat.compute.stats.models.delta_stats import DeltaStats
+import ray
 from ray.types import ObjectRef
 
+from deltacat.compute.stats.models.delta_stats import DeltaStats
 from deltacat.compute.stats.models.delta_stats_cache_result import DeltaStatsCacheResult
 from deltacat.compute.stats.models.stats_result import StatsResult
 from deltacat.compute.stats.types import StatsType
-from deltacat.compute.stats.utils.io import read_cached_delta_stats, cache_delta_column_stats, get_delta_stats, \
-    get_deltas_from_range
-from deltacat.compute.stats.utils.intervals import merge_intervals, DeltaRange
-
-from deltacat.storage import PartitionLocator, DeltaLocator, Delta
+from deltacat.compute.stats.utils.intervals import DeltaRange, merge_intervals
+from deltacat.compute.stats.utils.io import (
+    cache_delta_column_stats,
+    get_delta_stats,
+    get_deltas_from_range,
+    read_cached_delta_stats,
+)
+from deltacat.storage import Delta, DeltaLocator, PartitionLocator
 from deltacat.storage import interface as unimplemented_deltacat_storage
 
-
 # TODO (ricmiyam): Decouple DeltaCAT from S3-based paths
 # TODO (ricmiyam): Determine cache eviction policy
 
 
 def collect(
-        source_partition_locator: PartitionLocator,
-        delta_stream_position_range_set: Optional[Set[DeltaRange]] = None,
-        columns: Optional[List[str]] = None,
-        stat_results_s3_bucket: Optional[str] = None,
-        deltacat_storage=unimplemented_deltacat_storage) -> Dict[int, DeltaStats]:
+    source_partition_locator: PartitionLocator,
+    delta_stream_position_range_set: Optional[Set[DeltaRange]] = None,
+    columns: Optional[List[str]] = None,
+    stat_results_s3_bucket: Optional[str] = None,
+    deltacat_storage=unimplemented_deltacat_storage,
+) -> Dict[int, DeltaStats]:
     """Collects statistics on deltas, given a set of delta stream position ranges.
 
     Example:
@@ -52,109 +55,139 @@ def collect(
     delta_range_lookup_pending: List[ObjectRef[List[Delta]]] = []
 
     if not columns:
-        columns = deltacat_storage.get_table_version_column_names(source_partition_locator.namespace,
-                                                                  source_partition_locator.table_name,
-                                                                  source_partition_locator.table_version)
+        columns = deltacat_storage.get_table_version_column_names(
+            source_partition_locator.namespace,
+            source_partition_locator.table_name,
+            source_partition_locator.table_version,
+        )
     for range_pair in merge_intervals(delta_stream_position_range_set):
         begin, end = range_pair
-        promise: ObjectRef[List[Delta]] = get_deltas_from_range.remote(source_partition_locator, begin, end,
-                                                                       deltacat_storage)
+        promise: ObjectRef[List[Delta]] = get_deltas_from_range.remote(
+            source_partition_locator, begin, end, deltacat_storage
+        )
         delta_range_lookup_pending.append(promise)
 
     delta_list_by_ranges: List[List[Delta]] = ray.get(delta_range_lookup_pending)
     deltas = [delta for delta_list in delta_list_by_ranges for delta in delta_list]
 
-    delta_stats_processed_list: List[DeltaStats] = _collect_stats_from_deltas(deltas,
-                                                                              columns,
-                                                                              stat_results_s3_bucket,
-                                                                              deltacat_storage)
+    delta_stats_processed_list: List[DeltaStats] = _collect_stats_from_deltas(
+        deltas, columns, stat_results_s3_bucket, deltacat_storage
+    )
 
     for delta_column_stats in delta_stats_processed_list:
-        assert len(delta_column_stats.column_stats) > 0, \
-            f"Expected columns of `{delta_column_stats}` to be non-empty"
-        stream_position = delta_column_stats.column_stats[0].manifest_stats.delta_locator.stream_position
+        assert (
+            len(delta_column_stats.column_stats) > 0
+        ), f"Expected columns of `{delta_column_stats}` to be non-empty"
+        stream_position = delta_column_stats.column_stats[
+            0
+        ].manifest_stats.delta_locator.stream_position
         delta_stream_range_stats[stream_position] = delta_column_stats
 
     return delta_stream_range_stats
 
 
 def collect_from_deltas(
-        deltas: List[Delta],
-        stat_types: Set[StatsType],
-        columns: Optional[List[str]] = None,
-        stat_results_s3_bucket: Optional[str] = None,
-        deltacat_storage=unimplemented_deltacat_storage) -> StatsResult:
+    deltas: List[Delta],
+    stat_types: Set[StatsType],
+    columns: Optional[List[str]] = None,
+    stat_results_s3_bucket: Optional[str] = None,
+    deltacat_storage=unimplemented_deltacat_storage,
+) -> StatsResult:
     """
     Variant of the `collect` function that takes a list of deltas and computes
     the aggregate of all the delta stats.
     """
     if columns is None and deltas:
         delta_locator: DeltaLocator = deltas[0].locator
-        columns = deltacat_storage.get_table_version_column_names(delta_locator.namespace,
-                                                                  delta_locator.table_name,
-                                                                  delta_locator.table_version)
+        columns = deltacat_storage.get_table_version_column_names(
+            delta_locator.namespace,
+            delta_locator.table_name,
+            delta_locator.table_version,
+        )
 
-    delta_stats_processed_list: List[DeltaStats] = \
-        _collect_stats_from_deltas(deltas, columns, stat_results_s3_bucket, deltacat_storage)
+    delta_stats_processed_list: List[DeltaStats] = _collect_stats_from_deltas(
+        deltas, columns, stat_results_s3_bucket, deltacat_storage
+    )
 
-    return StatsResult.merge([delta_ds.stats for delta_ds in delta_stats_processed_list], stat_types)
+    return StatsResult.merge(
+        [delta_ds.stats for delta_ds in delta_stats_processed_list], stat_types
+    )
 
 
 def _collect_stats_from_deltas(
-        deltas: List[Delta],
-        columns: Optional[List[str]] = None,
-        stat_results_s3_bucket: Optional[str] = None,
-        deltacat_storage=unimplemented_deltacat_storage) -> List[DeltaStats]:
+    deltas: List[Delta],
+    columns: Optional[List[str]] = None,
+    stat_results_s3_bucket: Optional[str] = None,
+    deltacat_storage=unimplemented_deltacat_storage,
+) -> List[DeltaStats]:
     delta_cache_lookup_pending: List[ObjectRef[DeltaStatsCacheResult]] = []
     delta_stats_compute_pending: List[ObjectRef[DeltaStats]] = []
 
     for delta in deltas:
         if stat_results_s3_bucket:
-            promise: ObjectRef[DeltaStatsCacheResult] = \
-                read_cached_delta_stats.remote(delta, columns, stat_results_s3_bucket)
+            promise: ObjectRef[DeltaStatsCacheResult] = read_cached_delta_stats.remote(
+                delta, columns, stat_results_s3_bucket
+            )
             delta_cache_lookup_pending.append(promise)
             continue
 
-        delta_stats_compute_pending.append(get_delta_stats.remote(delta.locator, columns, deltacat_storage))
+        delta_stats_compute_pending.append(
+            get_delta_stats.remote(delta.locator, columns, deltacat_storage)
+        )
 
-    return _process_stats(delta_cache_lookup_pending, delta_stats_compute_pending,
-                          stat_results_s3_bucket, deltacat_storage)
+    return _process_stats(
+        delta_cache_lookup_pending,
+        delta_stats_compute_pending,
+        stat_results_s3_bucket,
+        deltacat_storage,
+    )
 
 
 def _process_stats(
-        delta_cache_lookup_pending: List[ObjectRef[DeltaStatsCacheResult]],
-        delta_stats_compute_pending: List[ObjectRef[DeltaStats]],
-        stat_results_s3_bucket: Optional[str] = None,
-        deltacat_storage=unimplemented_deltacat_storage) -> List[DeltaStats]:
+    delta_cache_lookup_pending: List[ObjectRef[DeltaStatsCacheResult]],
+    delta_stats_compute_pending: List[ObjectRef[DeltaStats]],
+    stat_results_s3_bucket: Optional[str] = None,
+    deltacat_storage=unimplemented_deltacat_storage,
+) -> List[DeltaStats]:
     if stat_results_s3_bucket:
-        delta_stats_processed_list: List[DeltaStats] = _resolve_pending_stats_and_cache(delta_cache_lookup_pending,
-                                                                                        stat_results_s3_bucket,
-                                                                                        deltacat_storage)
+        delta_stats_processed_list: List[DeltaStats] = _resolve_pending_stats_and_cache(
+            delta_cache_lookup_pending, stat_results_s3_bucket, deltacat_storage
+        )
     else:
-        delta_stats_processed_list: List[DeltaStats] = _resolve_pending_stats(delta_stats_compute_pending)
+        delta_stats_processed_list: List[DeltaStats] = _resolve_pending_stats(
+            delta_stats_compute_pending
+        )
 
     return delta_stats_processed_list
 
 
-def _resolve_pending_stats_and_cache(delta_cache_lookup_pending: List[ObjectRef[DeltaStatsCacheResult]],
-                                     stat_results_s3_bucket: str,
-                                     deltacat_storage) -> List[DeltaStats]:
-    delta_stats_cached_list, delta_stats_pending_list = \
-        _get_cached_and_pending_stats(delta_cache_lookup_pending, deltacat_storage)
-    delta_stats_resolved_list: List[DeltaStats] = _resolve_pending_stats(delta_stats_pending_list)
+def _resolve_pending_stats_and_cache(
+    delta_cache_lookup_pending: List[ObjectRef[DeltaStatsCacheResult]],
+    stat_results_s3_bucket: str,
+    deltacat_storage,
+) -> List[DeltaStats]:
+    delta_stats_cached_list, delta_stats_pending_list = _get_cached_and_pending_stats(
+        delta_cache_lookup_pending, deltacat_storage
+    )
+    delta_stats_resolved_list: List[DeltaStats] = _resolve_pending_stats(
+        delta_stats_pending_list
+    )
 
     # Cache the stats into the file store
-    delta_stats_to_cache: List[ObjectRef] = [cache_delta_column_stats.remote(stat_results_s3_bucket, dcs)
-                                             for dataset_stats in delta_stats_resolved_list
-                                             for dcs in dataset_stats.column_stats]
+    delta_stats_to_cache: List[ObjectRef] = [
+        cache_delta_column_stats.remote(stat_results_s3_bucket, dcs)
+        for dataset_stats in delta_stats_resolved_list
+        for dcs in dataset_stats.column_stats
+    ]
     ray.get(delta_stats_to_cache)
 
     return [*delta_stats_cached_list, *delta_stats_resolved_list]
 
 
-def _get_cached_and_pending_stats(discover_deltas_pending: List[ObjectRef[DeltaStatsCacheResult]],
-                                  deltacat_storage=unimplemented_deltacat_storage) \
-        -> Tuple[List[DeltaStats], List[ObjectRef[DeltaStats]]]:
+def _get_cached_and_pending_stats(
+    discover_deltas_pending: List[ObjectRef[DeltaStatsCacheResult]],
+    deltacat_storage=unimplemented_deltacat_storage,
+) -> Tuple[List[DeltaStats], List[ObjectRef[DeltaStats]]]:
     """
     Returns a tuple of a list of delta stats fetched from the cache, and a list of Ray tasks which will
     calculate the stats for deltas on cache miss.
@@ -172,13 +205,18 @@ def _get_cached_and_pending_stats(discover_deltas_pending: List[ObjectRef[DeltaS
             if cached_result.misses:
                 missed_column_names: List[str] = cached_result.misses.column_names
                 delta_locator: DeltaLocator = cached_result.misses.delta_locator
-                delta_stats_pending.append(get_delta_stats.remote(delta_locator, missed_column_names, deltacat_storage))
+                delta_stats_pending.append(
+                    get_delta_stats.remote(
+                        delta_locator, missed_column_names, deltacat_storage
+                    )
+                )
 
     return delta_stats_processed, delta_stats_pending
 
 
-def _resolve_pending_stats(delta_stats_pending_list: List[ObjectRef[DeltaStats]]) \
-        -> List[DeltaStats]:
+def _resolve_pending_stats(
+    delta_stats_pending_list: List[ObjectRef[DeltaStats]],
+) -> List[DeltaStats]:
     delta_stats_processed_list: List[DeltaStats] = []
     while delta_stats_pending_list:
         ready, delta_stats_pending_list = ray.wait(delta_stats_pending_list)

deltacat/compute/stats/models/delta_column_stats.py

@@ -1,7 +1,7 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-from typing import Optional, Dict, Any, List
+from typing import Any, Dict, List, Optional
 
 from deltacat.compute.stats.models.manifest_entry_stats import ManifestEntryStats
 from deltacat.compute.stats.models.stats_result import StatsResult
@@ -42,6 +42,7 @@ class DeltaColumnStats(dict):
                 StatsResult([I, L]),     #  Manifest Entry 2
             ]))
     """
+
     @staticmethod
     def of(column: str, manifest_stats: ManifestEntryStats) -> DeltaColumnStats:
         """
@@ -59,13 +60,14 @@ class DeltaColumnStats(dict):
 
     @staticmethod
     def build_from_dict(delta_column_stats: List[str, Any]) -> List[DeltaColumnStats]:
-        return DeltaColumnStats.of(delta_column_stats["column"],
-                                   ManifestEntryStats.build_from_dict(delta_column_stats["manifestStats"]))
+        return DeltaColumnStats.of(
+            delta_column_stats["column"],
+            ManifestEntryStats.build_from_dict(delta_column_stats["manifestStats"]),
+        )
 
     @property
     def column(self) -> str:
-        """Returns the column name.
-        """
+        """Returns the column name."""
         return self.get("column")
 
     @property
@@ -81,8 +83,7 @@ class DeltaColumnStats(dict):
 
     @property
     def stats(self) -> Optional[StatsResult]:
-        """ Combines the numerical stats for every manifest entry and returns it.
-        """
+        """Combines the numerical stats for every manifest entry and returns it."""
         val: Dict[str, Any] = self.get("stats")
         if val is not None and not isinstance(val, StatsResult):
             self["stats"] = val = StatsResult(val)
@@ -92,4 +93,6 @@ class DeltaColumnStats(dict):
         return val
 
     def _merge_manifest_stats(self) -> StatsResult:
-        return StatsResult.merge(self.manifest_stats.stats, {StatsType.PYARROW_TABLE_BYTES})
+        return StatsResult.merge(
+            self.manifest_stats.stats, {StatsType.PYARROW_TABLE_BYTES}
+        )

deltacat/compute/stats/models/delta_stats.py

@@ -2,7 +2,7 @@
 from __future__ import annotations
 
 from collections import defaultdict
-from typing import List, Dict, Optional, Set, Any, NamedTuple
+from typing import Any, Dict, List, NamedTuple, Optional, Set
 
 from deltacat.compute.stats.models.delta_column_stats import DeltaColumnStats
 from deltacat.compute.stats.models.manifest_entry_stats import ManifestEntryStats
@@ -76,8 +76,7 @@ class DeltaStats(dict):
 
     @property
     def stats(self) -> Optional[StatsResult]:
-        """Returns a StatsResult object that represents this delta, aggregated by the column stats of this delta.
-        """
+        """Returns a StatsResult object that represents this delta, aggregated by the column stats of this delta."""
         val: Dict[str, Any] = self.get("stats")
         if val is not None and not isinstance(val, StatsResult):
             self["stats"] = val = StatsResult(val)
@@ -104,8 +103,12 @@ class DeltaStats(dict):
         Returns:
             Stats for the manifest entry.
         """
-        return StatsResult.merge(DeltaStats.get_manifest_entry_column_stats(self.column_stats, manifest_entry_idx),
-                                 record_row_count_once=True)
+        return StatsResult.merge(
+            DeltaStats.get_manifest_entry_column_stats(
+                self.column_stats, manifest_entry_idx
+            ),
+            record_row_count_once=True,
+        )
 
     def manifest_entry_column_stats(self, manifest_entry_idx: int) -> List[StatsResult]:
         """Fetch a list of stats for each column in a manifest entry.
@@ -116,23 +119,34 @@ class DeltaStats(dict):
         Returns:
             A list of columnar stats for the manifest entry
         """
-        return DeltaStats.get_manifest_entry_column_stats(self.column_stats, manifest_entry_idx)
+        return DeltaStats.get_manifest_entry_column_stats(
+            self.column_stats, manifest_entry_idx
+        )
 
     @staticmethod
-    def get_manifest_entry_column_stats(columns: List[DeltaColumnStats], manifest_entry_idx: int) -> List[StatsResult]:
+    def get_manifest_entry_column_stats(
+        columns: List[DeltaColumnStats], manifest_entry_idx: int
+    ) -> List[StatsResult]:
         """Helper method to provide a list of columnar stats for a specific manifest entry.
 
         Returns:
             A list of columnar stats for the manifest entry
         """
-        dataset_columnar_stats_list: List[ManifestEntryStats] = [column.manifest_stats for column in columns
-                                                                 if column.manifest_stats is not None]
+        dataset_columnar_stats_list: List[ManifestEntryStats] = [
+            column.manifest_stats
+            for column in columns
+            if column.manifest_stats is not None
+        ]
         try:
-            return [stats.stats[manifest_entry_idx] for stats in dataset_columnar_stats_list]
+            return [
+                stats.stats[manifest_entry_idx] for stats in dataset_columnar_stats_list
+            ]
         except IndexError:
             sci: ManifestEntryStats = dataset_columnar_stats_list[0]
-            raise ValueError(f"Table index {manifest_entry_idx} is not present in this dataset of {sci.delta_locator} "
-                             f"with manifest table count of {len(sci.stats)}")
+            raise ValueError(
+                f"Table index {manifest_entry_idx} is not present in this dataset of {sci.delta_locator} "
+                f"with manifest table count of {len(sci.stats)}"
+            )
 
     @staticmethod
     def get_column_names(columns: List[DeltaColumnStats]) -> List[str]:
@@ -147,8 +161,9 @@ class DeltaStats(dict):
         return [column_stats.column for column_stats in columns] if columns else []
 
     @staticmethod
-    def get_delta_stats(columns: List[DeltaColumnStats],
-                        stat_types: Optional[Set[StatsType]] = None) -> Optional[StatsResult]:
+    def get_delta_stats(
+        columns: List[DeltaColumnStats], stat_types: Optional[Set[StatsType]] = None
+    ) -> Optional[StatsResult]:
         """Calculate the sum of provided column stats and return it
 
         Args:
@@ -157,41 +172,52 @@ class DeltaStats(dict):
         Returns:
             Stats for the calculated sum
         """
-        assert columns and len(columns) > 0, \
-            f"Expected columns `{columns}` of type `{type(columns)}` " \
+        assert columns and len(columns) > 0, (
+            f"Expected columns `{columns}` of type `{type(columns)}` "
             f"to be a non-empty list of DeltaColumnStats"
+        )
 
-        assert all([col.manifest_stats for col in columns]), \
-            f"Expected stats completion info to be present in each item of {columns} "
+        assert all(
+            [col.manifest_stats for col in columns]
+        ), f"Expected stats completion info to be present in each item of {columns} "
 
         manifest_entry_count = len(columns[0].manifest_stats.stats)
-        column_stats_map: Dict[str, List[Optional[StatsResult]]] = \
-            defaultdict(lambda: [None] * manifest_entry_count)
+        column_stats_map: Dict[str, List[Optional[StatsResult]]] = defaultdict(
+            lambda: [None] * manifest_entry_count
+        )
 
         for column_stats in columns:
             for file_idx, entry_stats in enumerate(column_stats.manifest_stats.stats):
                 column_stats_map[column_stats.column][file_idx] = entry_stats
 
-        return DeltaStats._merge_stats_from_columns_to_dataset(DeltaStats.get_column_names(columns),
-                                                               column_stats_map,
-                                                               manifest_entry_count,
-                                                               stat_types)
+        return DeltaStats._merge_stats_from_columns_to_dataset(
+            DeltaStats.get_column_names(columns),
+            column_stats_map,
+            manifest_entry_count,
+            stat_types,
+        )
 
     @staticmethod
-    def _merge_stats_from_columns_to_dataset(column_names: List[str],
-                                             column_stats: Dict[str, List[Optional[StatsResult]]],
-                                             manifest_entries_size: int,
-                                             stat_types: Optional[Set[StatsType]] = None) -> StatsResult:
+    def _merge_stats_from_columns_to_dataset(
+        column_names: List[str],
+        column_stats: Dict[str, List[Optional[StatsResult]]],
+        manifest_entries_size: int,
+        stat_types: Optional[Set[StatsType]] = None,
+    ) -> StatsResult:
         manifest_entry_stats_summary_list: List[StatsResult] = []
         for manifest_entry_idx in range(manifest_entries_size):
             curr_manifest_entry_column_stats_list: List[StatsResult] = []
             for column_name in column_names:
-                current_table_column_stats: StatsResult = column_stats[column_name][manifest_entry_idx]
+                current_table_column_stats: StatsResult = column_stats[column_name][
+                    manifest_entry_idx
+                ]
                 curr_manifest_entry_column_stats_list.append(current_table_column_stats)
 
-            curr_manifest_entry_stats_summary = StatsResult.merge(curr_manifest_entry_column_stats_list,
-                                                                  stat_types,
-                                                                  record_row_count_once=True)
+            curr_manifest_entry_stats_summary = StatsResult.merge(
+                curr_manifest_entry_column_stats_list,
+                stat_types,
+                record_row_count_once=True,
+            )
             manifest_entry_stats_summary_list.append(curr_manifest_entry_stats_summary)
         return StatsResult.merge(manifest_entry_stats_summary_list, stat_types)
 
@@ -202,5 +228,6 @@ class DeltaStatsCacheMiss(NamedTuple):
     `column_names` represents missing dataset column names from the file system (ex: S3).
     delta_locator` is tied to the missing dataset columns and provided for future calculations.
     """
+
     column_names: List[str]
     delta_locator: DeltaLocator

deltacat/compute/stats/models/delta_stats_cache_result.py

@@ -12,8 +12,11 @@ class DeltaStatsCacheResult(dict):
     Stats are fetched and cached at the column level, and each column may represent one
     or more manifest entries.
     """
+
     @staticmethod
-    def of(hits: Optional[DeltaStats], misses: Optional[DeltaStatsCacheMiss]) -> DeltaStatsCacheResult:
+    def of(
+        hits: Optional[DeltaStats], misses: Optional[DeltaStatsCacheMiss]
+    ) -> DeltaStatsCacheResult:
         cds = DeltaStatsCacheResult()
         cds["hits"] = hits
         cds["misses"] = misses

deltacat/compute/stats/models/manifest_entry_stats.py

@@ -1,22 +1,24 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
+from typing import Any, Dict, List
+
 import pyarrow as pa
 
 from deltacat.compute.stats.models.stats_result import StatsResult
 from deltacat.storage import DeltaLocator
 
-from typing import Any, Dict, List
-
 
 class ManifestEntryStats(dict):
     """Holds computed statistics for one or more manifest entries (tables) and their corresponding delta locator.
 
     To be stored/retrieved from a file system (ex: S3).
     """
+
     @staticmethod
-    def of(manifest_entries_stats: List[StatsResult],
-           delta_locator: DeltaLocator) -> ManifestEntryStats:
+    def of(
+        manifest_entries_stats: List[StatsResult], delta_locator: DeltaLocator
+    ) -> ManifestEntryStats:
         """
         Creates a stats container that represents a particular manifest.
 
@@ -35,8 +37,12 @@ class ManifestEntryStats(dict):
     def build_from_dict(manifest_entries_stats: dict) -> ManifestEntryStats:
         stats_res_list = []
         for stats_res in manifest_entries_stats["stats"]:
-            stats_res_list.append(StatsResult.of(stats_res["rowCount"], stats_res["pyarrowTableBytes"]))
-        return ManifestEntryStats.of(stats_res_list, manifest_entries_stats["deltaLocator"])
+            stats_res_list.append(
+                StatsResult.of(stats_res["rowCount"], stats_res["pyarrowTableBytes"])
+            )
+        return ManifestEntryStats.of(
+            stats_res_list, manifest_entries_stats["deltaLocator"]
+        )
 
     @property
     def delta_locator(self) -> DeltaLocator:

deltacat/compute/stats/models/stats_result.py

@@ -1,17 +1,19 @@
 # Allow classes to use self-referencing Type hints in Python 3.7.
 from __future__ import annotations
 
-from typing import Optional, List, Set, Dict, Any
-
 from collections import defaultdict
-from deltacat.compute.stats.types import StatsType, ALL_STATS_TYPES
+from typing import Any, Dict, List, Optional, Set
+
+from deltacat.compute.stats.types import ALL_STATS_TYPES, StatsType
+
 
 class StatsResult(dict):
-    """A generic container that holds stats for a single manifest entry file.
-    """
+    """A generic container that holds stats for a single manifest entry file."""
+
     @staticmethod
-    def of(row_count: Optional[int] = 0,
-           pyarrow_table_bytes: Optional[int] = 0) -> StatsResult:
+    def of(
+        row_count: Optional[int] = 0, pyarrow_table_bytes: Optional[int] = 0
+    ) -> StatsResult:
         """Static factory for building a stats result object
 
         Args:
@@ -54,13 +56,20 @@ class StatsResult(dict):
         Returns:
             A stats result object
         """
-        return StatsResult({k: v for k, v in stats_types.items()
-                            if k in [StatsType.ROW_COUNT, StatsType.PYARROW_TABLE_BYTES]})
+        return StatsResult(
+            {
+                k: v
+                for k, v in stats_types.items()
+                if k in [StatsType.ROW_COUNT, StatsType.PYARROW_TABLE_BYTES]
+            }
+        )
 
     @staticmethod
-    def merge(stats_list: List[StatsResult],
-              stat_types: Optional[Set[StatsType]] = None,
-              record_row_count_once: bool = False) -> StatsResult:
+    def merge(
+        stats_list: List[StatsResult],
+        stat_types: Optional[Set[StatsType]] = None,
+        record_row_count_once: bool = False,
+    ) -> StatsResult:
         """Helper method to merge any list of StatsResult objects into one.
 
         StatsResult objects are merged by adding up their numerical stats.
@@ -75,9 +84,10 @@ class StatsResult(dict):
         Returns:
             A stats result object
         """
-        assert isinstance(stats_list, list) and len(stats_list) > 0, \
-            f"Expected stats list: {stats_list} of type {type(stats_list)} to be a " \
+        assert isinstance(stats_list, list) and len(stats_list) > 0, (
+            f"Expected stats list: {stats_list} of type {type(stats_list)} to be a "
             f"non-empty list of StatsResult objects."
+        )
 
         # Fallback to all stat types if not provided
         stats_to_collect: Set = stat_types or ALL_STATS_TYPES

deltacat/compute/stats/utils/intervals.py

@@ -1,4 +1,4 @@
-from typing import Set, Tuple, List, Iterable, Union, Optional
+from typing import Iterable, List, Optional, Set, Tuple, Union
 
 DeltaPosition = Optional[int]
 NumericDeltaPosition = Union[int, float]  # float is added here to support math.inf
@@ -38,7 +38,9 @@ def merge_intervals(intervals: Set[DeltaRange]) -> Set[DeltaRange]:
     for interval in intervals_list:
         start, end = interval
         if start > end:
-            raise ValueError(f"Invalid stream position range interval: ({start}, {end})")
+            raise ValueError(
+                f"Invalid stream position range interval: ({start}, {end})"
+            )
 
         if merge_start is None and merge_end is None:
             merge_start, merge_end = start, end
@@ -57,7 +59,9 @@ def merge_intervals(intervals: Set[DeltaRange]) -> Set[DeltaRange]:
     return merged
 
 
-def _add_merged_interval(result_set: set, start: NumericDeltaPosition, end: NumericDeltaPosition):
+def _add_merged_interval(
+    result_set: set, start: NumericDeltaPosition, end: NumericDeltaPosition
+):
     start_pos: DeltaPosition = start if isinstance(start, int) else None
     end_pos: DeltaPosition = end if isinstance(end, int) else None
     result_set.add((start_pos, end_pos))
@@ -67,9 +71,9 @@ def _to_numeric_values(intervals_list: List[DeltaRange]):
     for i, interval in enumerate(intervals_list):
         start, end = _get_validated_interval(interval)
         if start is None:
-            start = float('-inf')
+            start = float("-inf")
         if end is None:
-            end = float('inf')
+            end = float("inf")
 
         intervals_list[i] = (start, end)
 
@@ -79,9 +83,12 @@ def _get_validated_interval(interval: DeltaRange) -> DeltaRange:
         raise ValueError(f"Interval {interval} must be a tuple of size 2")
 
     start, end = interval
-    if not (isinstance(start, int) or start is None) \
-            or not (isinstance(end, int) or end is None):
-        raise ValueError(f"Invalid stream position value types: "
-                         f"({start}, {end}) - ({type(start), type(end)})")
+    if not (isinstance(start, int) or start is None) or not (
+        isinstance(end, int) or end is None
+    ):
+        raise ValueError(
+            f"Invalid stream position value types: "
+            f"({start}, {end}) - ({type(start), type(end)})"
+        )
 
     return start, end