deltacat 0.1.10.dev0__py3-none-any.whl → 0.1.12__py3-none-any.whl

deltacat/compute/stats/utils/io.py

@@ -1,31 +1,31 @@
 import logging
+from collections import defaultdict
+from typing import Dict, List, Optional
 
 import pyarrow
 import ray
-from collections import defaultdict
 
-from deltacat.compute.stats.utils.manifest_stats_file import read_manifest_stats_by_columns, write_manifest_stats_file
-from deltacat.compute.stats.models.delta_stats_cache_result import DeltaStatsCacheResult
-from deltacat.compute.stats.models.manifest_entry_stats import ManifestEntryStats
+from deltacat import LocalTable, TableType, logs
 from deltacat.compute.stats.models.delta_column_stats import DeltaColumnStats
 from deltacat.compute.stats.models.delta_stats import DeltaStats, DeltaStatsCacheMiss
-
+from deltacat.compute.stats.models.delta_stats_cache_result import DeltaStatsCacheResult
+from deltacat.compute.stats.models.manifest_entry_stats import ManifestEntryStats
 from deltacat.compute.stats.models.stats_result import StatsResult
 from deltacat.compute.stats.utils.intervals import DeltaRange
-from deltacat.storage import PartitionLocator, Delta, DeltaLocator
-from deltacat import logs, LocalTable, TableType
+from deltacat.compute.stats.utils.manifest_stats_file import (
+    read_manifest_stats_by_columns,
+    write_manifest_stats_file,
+)
+from deltacat.storage import Delta, DeltaLocator, PartitionLocator
 from deltacat.storage import interface as unimplemented_deltacat_storage
-from deltacat.compute.compactor import DeltaAnnotated
-
-from typing import Dict, List, Optional, Any
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 @ray.remote
-def read_cached_delta_stats(delta: Delta,
-                            columns_to_fetch: List[str],
-                            stat_results_s3_bucket: str):
+def read_cached_delta_stats(
+    delta: Delta, columns_to_fetch: List[str], stat_results_s3_bucket: str
+):
     """Read delta stats that are cached in S3
 
     This Ray distributed task reads delta stats from a file system (i.e. S3) based on specified columns.
@@ -39,8 +39,11 @@ def read_cached_delta_stats(delta: Delta,
     """
 
     delta_locator = DeltaLocator.of(delta.partition_locator, delta.stream_position)
-    column_stats_completion_info: List[DeltaColumnStats] = \
-        read_manifest_stats_by_columns(stat_results_s3_bucket, columns_to_fetch, delta_locator)
+    column_stats_completion_info: List[
+        DeltaColumnStats
+    ] = read_manifest_stats_by_columns(
+        stat_results_s3_bucket, columns_to_fetch, delta_locator
+    )
 
     found_columns_stats: List[DeltaColumnStats] = []
     missed_columns: List[str] = []
@@ -50,29 +53,37 @@ def read_cached_delta_stats(delta: Delta,
         else:
             missed_columns.append(column_stats.column)
 
-    found_stats: Optional[DeltaStats] = DeltaStats.of(found_columns_stats) if found_columns_stats else None
-    missed_stats: Optional[DeltaStatsCacheMiss] = DeltaStatsCacheMiss(missed_columns, delta.locator) \
-        if missed_columns else None
+    found_stats: Optional[DeltaStats] = (
+        DeltaStats.of(found_columns_stats) if found_columns_stats else None
+    )
+    missed_stats: Optional[DeltaStatsCacheMiss] = (
+        DeltaStatsCacheMiss(missed_columns, delta.locator) if missed_columns else None
+    )
 
     return DeltaStatsCacheResult.of(found_stats, missed_stats)
 
 
 @ray.remote
-def cache_delta_column_stats(stat_results_s3_bucket: str,
-                             dataset_column: DeltaColumnStats) -> None:
+def cache_delta_column_stats(
+    stat_results_s3_bucket: str, dataset_column: DeltaColumnStats
+) -> None:
     """Ray distributed task to cache the delta column stats into a file system (i.e. S3).
 
     Args:
         stat_results_s3_bucket: The S3 bucket name
         dataset_column: Column-oriented stats for a given delta
     """
-    write_manifest_stats_file(stat_results_s3_bucket, dataset_column.column, dataset_column.manifest_stats)
+    write_manifest_stats_file(
+        stat_results_s3_bucket, dataset_column.column, dataset_column.manifest_stats
+    )
 
 
 @ray.remote
-def get_delta_stats(delta_locator: DeltaLocator,
-                    columns: Optional[List[str]] = None,
-                    deltacat_storage=unimplemented_deltacat_storage) -> DeltaStats:
+def get_delta_stats(
+    delta_locator: DeltaLocator,
+    columns: Optional[List[str]] = None,
+    deltacat_storage=unimplemented_deltacat_storage,
+) -> DeltaStats:
     """Ray distributed task to compute and collect stats for a requested delta.
     If no columns are requested, stats will be computed for all columns.
     Args:
@@ -90,10 +101,11 @@ def get_delta_stats(delta_locator: DeltaLocator,
 
 @ray.remote
 def get_deltas_from_range(
-        source_partition_locator: PartitionLocator,
-        start_position_inclusive: DeltaRange,
-        end_position_inclusive: DeltaRange,
-        deltacat_storage=unimplemented_deltacat_storage) -> List[Delta]:
+    source_partition_locator: PartitionLocator,
+    start_position_inclusive: DeltaRange,
+    end_position_inclusive: DeltaRange,
+    deltacat_storage=unimplemented_deltacat_storage,
+) -> List[Delta]:
     """Looks up deltas in the specified partition using Ray, given both starting and ending delta stream positions.
 
     Args:
@@ -108,8 +120,14 @@ def get_deltas_from_range(
         a list of delta objects
     """
 
-    namespace, partition_values = source_partition_locator.namespace, source_partition_locator.partition_values
-    table_name, table_version = source_partition_locator.table_name, source_partition_locator.table_version
+    namespace, partition_values = (
+        source_partition_locator.namespace,
+        source_partition_locator.partition_values,
+    )
+    table_name, table_version = (
+        source_partition_locator.table_name,
+        source_partition_locator.table_version,
+    )
     deltas_list_result = deltacat_storage.list_deltas(
         namespace,
         table_name,
@@ -118,14 +136,16 @@ def get_deltas_from_range(
         start_position_inclusive,
         end_position_inclusive,
         ascending_order=True,
-        include_manifest=False
+        include_manifest=False,
     )
     return deltas_list_result.all_items()
 
 
-def _collect_stats_by_columns(delta: Delta,
-                              columns_to_compute: Optional[List[str]] = None,
-                              deltacat_storage=unimplemented_deltacat_storage) -> DeltaStats:
+def _collect_stats_by_columns(
+    delta: Delta,
+    columns_to_compute: Optional[List[str]] = None,
+    deltacat_storage=unimplemented_deltacat_storage,
+) -> DeltaStats:
     """Materializes one manifest entry at a time to save memory usage and calculate stats from each of its columns.
     Args:
         delta: A delta object to calculate stats for
@@ -134,47 +154,62 @@ def _collect_stats_by_columns(delta: Delta,
     Returns:
         A delta wide stats container
     """
-    assert delta.manifest is not None, f"Manifest should not be missing from delta for stats calculation: {delta}"
+    assert (
+        delta.manifest is not None
+    ), f"Manifest should not be missing from delta for stats calculation: {delta}"
 
     # Mapping of column_name -> [stats_file_idx_1, stats_file_idx_2, ... stats_file_idx_n]
-    column_stats_map: Dict[str, List[Optional[StatsResult]]] = defaultdict(lambda: [None] * len(delta.manifest.entries))
+    column_stats_map: Dict[str, List[Optional[StatsResult]]] = defaultdict(
+        lambda: [None] * len(delta.manifest.entries)
+    )
 
     total_tables_size = 0
     for file_idx, manifest in enumerate(delta.manifest.entries):
-        entry_pyarrow_table: LocalTable = \
-            deltacat_storage.download_delta_manifest_entry(delta, file_idx, TableType.PYARROW, columns_to_compute)
-        assert isinstance(entry_pyarrow_table, pyarrow.Table), \
-            f"Stats collection is only supported for PyArrow tables, but received a table of " \
+        entry_pyarrow_table: LocalTable = (
+            deltacat_storage.download_delta_manifest_entry(
+                delta, file_idx, TableType.PYARROW, columns_to_compute
+            )
+        )
+        assert isinstance(entry_pyarrow_table, pyarrow.Table), (
+            f"Stats collection is only supported for PyArrow tables, but received a table of "
             f"type '{type(entry_pyarrow_table)}' for manifest entry {file_idx} of delta: {delta.locator}."
+        )
         total_tables_size += entry_pyarrow_table.nbytes
         if not columns_to_compute:
             columns_to_compute = entry_pyarrow_table.column_names
 
         for column_idx, pyarrow_column in enumerate(entry_pyarrow_table.columns):
             column_name = columns_to_compute[column_idx]
-            column_stats_map[column_name][file_idx] = StatsResult.of(len(pyarrow_column), pyarrow_column.nbytes)
+            column_stats_map[column_name][file_idx] = StatsResult.of(
+                len(pyarrow_column), pyarrow_column.nbytes
+            )
 
     # Add column-wide stats for a list of tables, these will be used for caching and retrieving later
-    delta_ds_column_stats: List[DeltaColumnStats] = \
-        _to_dataset_column_stats(delta.locator, columns_to_compute, column_stats_map)
+    delta_ds_column_stats: List[DeltaColumnStats] = _to_dataset_column_stats(
+        delta.locator, columns_to_compute, column_stats_map
+    )
 
     dataset_stats: DeltaStats = DeltaStats.of(delta_ds_column_stats)
 
     # Quick validation for calculations
-    assert dataset_stats.stats.pyarrow_table_bytes == total_tables_size, \
-        f"Expected the size of all PyArrow tables ({total_tables_size} bytes) " \
+    assert dataset_stats.stats.pyarrow_table_bytes == total_tables_size, (
+        f"Expected the size of all PyArrow tables ({total_tables_size} bytes) "
         f"to match the sum of each of its columns ({dataset_stats.stats.pyarrow_table_bytes} bytes)"
+    )
 
     return dataset_stats
 
 
-def _to_dataset_column_stats(delta_locator: DeltaLocator,
-                             column_names: List[str],
-                             column_manifest_map: Dict[str, List[Optional[StatsResult]]]) \
-        -> List[DeltaColumnStats]:
+def _to_dataset_column_stats(
+    delta_locator: DeltaLocator,
+    column_names: List[str],
+    column_manifest_map: Dict[str, List[Optional[StatsResult]]],
+) -> List[DeltaColumnStats]:
     dataset_stats: List[DeltaColumnStats] = []
     for column_name in column_names:
-        column_manifest_stats = ManifestEntryStats.of(column_manifest_map[column_name], delta_locator)
+        column_manifest_stats = ManifestEntryStats.of(
+            column_manifest_map[column_name], delta_locator
+        )
         dataset_column_stats = DeltaColumnStats.of(column_name, column_manifest_stats)
         dataset_stats.append(dataset_column_stats)
-    return dataset_stats
+    return dataset_stats

deltacat/compute/stats/utils/manifest_stats_file.py

@@ -1,21 +1,20 @@
-import logging
 import json
+import logging
 from typing import List
 
-from deltacat.compute.stats.models.manifest_entry_stats import ManifestEntryStats
-from deltacat.compute.stats.models.delta_column_stats import DeltaColumnStats
-from deltacat.storage import DeltaLocator
 from deltacat import logs
 from deltacat.aws import s3u as s3_utils
+from deltacat.compute.stats.models.delta_column_stats import DeltaColumnStats
+from deltacat.compute.stats.models.manifest_entry_stats import ManifestEntryStats
+from deltacat.storage import DeltaLocator
 from deltacat.utils.common import sha1_hexdigest
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 def get_manifest_stats_s3_url(
-        bucket: str,
-        column_name: str,
-        delta_locator: DeltaLocator) -> str:
+    bucket: str, column_name: str, delta_locator: DeltaLocator
+) -> str:
     """Returns the S3 URL path to the column-oriented delta stats
 
     Args:
@@ -33,9 +32,8 @@ def get_manifest_stats_s3_url(
 
 
 def read_manifest_stats_by_columns(
-        bucket: str,
-        column_names: List[str],
-        delta_locator: DeltaLocator) -> List[DeltaColumnStats]:
+    bucket: str, column_names: List[str], delta_locator: DeltaLocator
+) -> List[DeltaColumnStats]:
     """Fetch a list of delta column stats by reading each column-oriented delta stats file from S3
 
     Args:
@@ -46,14 +44,17 @@ def read_manifest_stats_by_columns(
     Returns:
         A list of delta column stats
     """
-    return [DeltaColumnStats.of(column, read_manifest_stats_file(bucket, column, delta_locator))
-            for column in column_names]
+    return [
+        DeltaColumnStats.of(
+            column, read_manifest_stats_file(bucket, column, delta_locator)
+        )
+        for column in column_names
+    ]
 
 
 def read_manifest_stats_file(
-        bucket: str,
-        column_name: str,
-        delta_locator: DeltaLocator) -> ManifestEntryStats:
+    bucket: str, column_name: str, delta_locator: DeltaLocator
+) -> ManifestEntryStats:
     """Read a manifest entry stats from S3
 
     Args:
@@ -66,12 +67,9 @@ def read_manifest_stats_file(
     """
 
     stats_completion_file_url = get_manifest_stats_s3_url(
-        bucket,
-        column_name,
-        delta_locator
+        bucket, column_name, delta_locator
     )
-    logger.info(
-        f"reading stats completion file from: {stats_completion_file_url}")
+    logger.info(f"reading stats completion file from: {stats_completion_file_url}")
     stats_completion_info_file = None
     result = s3_utils.download(stats_completion_file_url, fail_if_not_found=False)
     if result:
@@ -82,9 +80,8 @@ def read_manifest_stats_file(
 
 
 def write_manifest_stats_file(
-        bucket: str,
-        column_name: str,
-        manifest_entry_stats: ManifestEntryStats) -> None:
+    bucket: str, column_name: str, manifest_entry_stats: ManifestEntryStats
+) -> None:
     """Write a manifest entry stats into S3
 
     Args:
@@ -92,18 +89,12 @@ def write_manifest_stats_file(
         column_name: The name of the column which represents this manifest entry stats
         manifest_entry_stats: The manifest entry stats to serialize and store into S3
     """
-    logger.info(
-        f"writing stats completion file contents: {manifest_entry_stats}")
+    logger.info(f"writing stats completion file contents: {manifest_entry_stats}")
     stats_completion_file_s3_url = get_manifest_stats_s3_url(
         bucket,
         column_name,
         manifest_entry_stats.delta_locator,
     )
-    logger.info(
-        f"writing stats completion file to: {stats_completion_file_s3_url}")
-    s3_utils.upload(
-        stats_completion_file_s3_url,
-        str(json.dumps(manifest_entry_stats))
-    )
-    logger.info(
-        f"stats completion file written to: {stats_completion_file_s3_url}")
+    logger.info(f"writing stats completion file to: {stats_completion_file_s3_url}")
+    s3_utils.upload(stats_completion_file_s3_url, str(json.dumps(manifest_entry_stats)))
+    logger.info(f"stats completion file written to: {stats_completion_file_s3_url}")

deltacat/constants.py

@@ -1,14 +1,8 @@
 from deltacat.utils.common import env_string
 
 # Environment variables
-DELTACAT_LOG_LEVEL = env_string(
-    "DELTACAT_LOG_LEVEL",
-    "DEBUG"
-)
-APPLICATION_LOG_LEVEL = env_string(
-    "APPLICATION_LOG_LEVEL",
-    "DEBUG"
-)
+DELTACAT_LOG_LEVEL = env_string("DELTACAT_LOG_LEVEL", "DEBUG")
+APPLICATION_LOG_LEVEL = env_string("APPLICATION_LOG_LEVEL", "DEBUG")
 
 # Byte Units
 BYTES_PER_KIBIBYTE = 2**10
@@ -31,8 +25,5 @@ PYARROW_INFLATION_MULTIPLIER = 2.5
 PYARROW_INFLATION_MULTIPLIER_ALL_COLUMNS = 6
 
 PRIMARY_KEY_INDEX_WRITE_BOTO3_CONFIG = {
-   "retries": {
-      'max_attempts': 25,
-      'mode': 'standard'
-   }
-}
+    "retries": {"max_attempts": 25, "mode": "standard"}
+}

deltacat/io/__init__.py

@@ -1,7 +1,7 @@
-from deltacat.io.read_api import read_redshift
 from deltacat.io.dataset import DeltacatDataset
+from deltacat.io.read_api import read_redshift
 
 __all__ = [
     "DeltacatDataset",
     "read_redshift",
-]
+]