deltacat 1.1.17__py3-none-any.whl → 1.1.19__py3-none-any.whl

deltacat/compute/stats/models/stats_result.py

@@ -1,104 +0,0 @@
-# Allow classes to use self-referencing Type hints in Python 3.7.
-from __future__ import annotations
-
-from collections import defaultdict
-from typing import Any, Dict, List, Optional, Set
-
-from deltacat.compute.stats.types import ALL_STATS_TYPES, StatsType
-
-
-class StatsResult(dict):
-    """A generic container that holds stats for a single manifest entry file."""
-
-    @staticmethod
-    def of(
-        row_count: Optional[int] = 0, pyarrow_table_bytes: Optional[int] = 0
-    ) -> StatsResult:
-        """Static factory for building a stats result object
-
-        Args:
-            row_count: The total number of rows of a manifest entry
-            pyarrow_table_bytes: The total number of bytes when loaded into memory as a PyArrow Table
-
-        Returns:
-            A stats result object
-        """
-        sr = StatsResult()
-        sr[StatsType.ROW_COUNT.value] = row_count
-        sr[StatsType.PYARROW_TABLE_BYTES.value] = pyarrow_table_bytes
-        return sr
-
-    @property
-    def row_count(self) -> int:
-        """Represents the row count of a manifest entry file.
-
-        Returns:
-            The total number of rows of a manifest entry
-        """
-        return self[StatsType.ROW_COUNT.value]
-
-    @property
-    def pyarrow_table_bytes(self) -> int:
-        """Represents the size of a manifest entry file (in bytes) as it was loaded into a PyArrow table.
-
-        Returns:
-            The total number of bytes when loaded into memory as a PyArrow Table
-        """
-        return self[StatsType.PYARROW_TABLE_BYTES.value]
-
-    @staticmethod
-    def from_stats_types(stats_types: Dict[StatsType, Any]) -> StatsResult:
-        """A helper method to filter a dictionary by supported stats and returns a stats result object.
-
-        Args:
-            stats_types: Stats that should be included for constructing a stats result
-
-        Returns:
-            A stats result object
-        """
-        return StatsResult(
-            {
-                k: v
-                for k, v in stats_types.items()
-                if k in [StatsType.ROW_COUNT, StatsType.PYARROW_TABLE_BYTES]
-            }
-        )
-
-    @staticmethod
-    def merge(
-        stats_list: List[StatsResult],
-        stat_types: Optional[Set[StatsType]] = None,
-        record_row_count_once: bool = False,
-    ) -> StatsResult:
-        """Helper method to merge any list of StatsResult objects into one.
-
-        StatsResult objects are merged by adding up their numerical stats.
-        TODO (ricmiyam): Handle non-numerical stats when they are added
-
-        Args:
-            stat_types: If provided, the calculation will only include the requested stats.
-            record_row_count_once: If optionally set to `True`, then row counts are only added
-                from the first stats entry. One use case for this is merging table-centric stats
-                by columns, since the row count is expected to be the same across different columns.
-
-        Returns:
-            A stats result object
-        """
-        assert isinstance(stats_list, list) and len(stats_list) > 0, (
-            f"Expected stats list: {stats_list} of type {type(stats_list)} to be a "
-            f"non-empty list of StatsResult objects."
-        )
-
-        # Fallback to all stat types if not provided
-        stats_to_collect: Set = stat_types or ALL_STATS_TYPES
-
-        merged_stats: Dict[StatsType, int] = defaultdict(int)
-        for stats_result in stats_list:
-            for stat_type in stats_to_collect:
-                if stats_result:
-                    merged_stats[stat_type.value] += stats_result[stat_type.value]
-
-        if record_row_count_once and StatsType.ROW_COUNT in stats_to_collect:
-            merged_stats[StatsType.ROW_COUNT.value] = stats_list[0].row_count
-
-        return StatsResult.from_stats_types(merged_stats)

deltacat/compute/stats/utils/intervals.py

@@ -1,94 +0,0 @@
-from typing import Iterable, List, Optional, Set, Tuple, Union
-
-DeltaPosition = Optional[int]
-NumericDeltaPosition = Union[int, float]  # float is added here to support math.inf
-DeltaRange = Tuple[DeltaPosition, DeltaPosition]
-
-
-def merge_intervals(intervals: Set[DeltaRange]) -> Set[DeltaRange]:
-    """Merges a set of N input intervals into a minimal number of output intervals.
-
-    All input intervals will be merged into a minimal number of output intervals in O(N log N) time.
-
-    Example:
-        >>> merge_intervals((3, 9), (8, 12), (15, 19))
-        ((3, 12), (15, 19))
-
-    Example:
-        >>> merge_intervals((3, 9), (None, 15), (13, 30))
-        ((None, 30))
-
-    Args:
-        intervals: A list of intervals with an int type representing finite, closed bounded values and
-            a None type representing infinity.
-
-    Returns:
-        A minimal number of output intervals
-    """
-    merged: Set[DeltaRange] = set()
-    intervals_list: List[DeltaRange] = list(intervals)
-
-    if len(intervals_list) == 0:
-        return merged
-
-    _to_numeric_values(intervals_list)
-    intervals_list.sort()  # sort by starting range numbers
-
-    merge_start, merge_end = None, None
-    for interval in intervals_list:
-        start, end = interval
-        if start > end:
-            raise ValueError(
-                f"Invalid stream position range interval: ({start}, {end})"
-            )
-
-        if merge_start is None and merge_end is None:
-            merge_start, merge_end = start, end
-            continue
-
-        if merge_end < start:
-            # add current merge interval if no overlap, begin new interval
-            _add_merged_interval(merged, merge_start, merge_end)
-            merge_start, merge_end = start, end
-        elif merge_end < end:
-            merge_end = end  # expand current merge interval if there is an overlap
-
-    # add final merge interval
-    _add_merged_interval(merged, merge_start, merge_end)
-
-    return merged
-
-
-def _add_merged_interval(
-    result_set: set, start: NumericDeltaPosition, end: NumericDeltaPosition
-):
-    start_pos: DeltaPosition = start if isinstance(start, int) else None
-    end_pos: DeltaPosition = end if isinstance(end, int) else None
-    result_set.add((start_pos, end_pos))
-
-
-def _to_numeric_values(intervals_list: List[DeltaRange]):
-    for i, interval in enumerate(intervals_list):
-        start, end = _get_validated_interval(interval)
-        if start is None:
-            start = float("-inf")
-        if end is None:
-            end = float("inf")
-
-        intervals_list[i] = (start, end)
-
-
-def _get_validated_interval(interval: DeltaRange) -> DeltaRange:
-    if not isinstance(interval, Iterable) or len(interval) != 2:
-        raise ValueError(f"Interval {interval} must be a tuple of size 2")
-
-    start, end = interval
-    if not (isinstance(start, int) or start is None) or not (
-        isinstance(end, int) or end is None
-    ):
-        raise ValueError(
-            f"Invalid stream position value types: "
-            f"({start}, {end}) - ({type(start), type(end)})"
-        )
-
-    return start, end

deltacat/compute/stats/utils/io.py

@@ -1,230 +0,0 @@
-import logging
-from collections import defaultdict
-from typing import Any, Dict, List, Optional
-
-import pyarrow
-import ray
-
-from deltacat import LocalTable, TableType, logs
-from deltacat.compute.stats.models.delta_column_stats import DeltaColumnStats
-from deltacat.compute.stats.models.delta_stats import DeltaStats, DeltaStatsCacheMiss
-from deltacat.compute.stats.models.delta_stats_cache_result import DeltaStatsCacheResult
-from deltacat.compute.stats.models.manifest_entry_stats import ManifestEntryStats
-from deltacat.compute.stats.models.stats_result import StatsResult
-from deltacat.compute.stats.utils.intervals import DeltaRange
-from deltacat.compute.stats.utils.manifest_stats_file import (
-    read_manifest_stats_by_columns,
-    write_manifest_stats_file,
-)
-from deltacat.storage import Delta, DeltaLocator, PartitionLocator
-from deltacat.storage import interface as unimplemented_deltacat_storage
-
-logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
-
-
-@ray.remote
-def read_cached_delta_stats(
-    delta: Delta, columns_to_fetch: List[str], stat_results_s3_bucket: str
-):
-    """Read delta stats that are cached in S3
-
-    This Ray distributed task reads delta stats from a file system (i.e. S3) based on specified columns.
-    Stats are extracted from columns that are found (cache hit), while columns that are missing
-    from the file system will record their column names and the delta locator as a cache miss.
-
-    Args:
-        delta: The delta object to look up
-        columns_to_fetch: Columns to look up for this delta
-        stat_results_s3_bucket: The S3 bucket name
-    """
-
-    delta_locator = DeltaLocator.of(delta.partition_locator, delta.stream_position)
-    column_stats_completion_info: List[
-        DeltaColumnStats
-    ] = read_manifest_stats_by_columns(
-        stat_results_s3_bucket, columns_to_fetch, delta_locator
-    )
-
-    found_columns_stats: List[DeltaColumnStats] = []
-    missed_columns: List[str] = []
-    for column_stats in column_stats_completion_info:
-        if column_stats.manifest_stats:
-            found_columns_stats.append(column_stats)
-        else:
-            missed_columns.append(column_stats.column)
-
-    found_stats: Optional[DeltaStats] = (
-        DeltaStats.of(found_columns_stats) if found_columns_stats else None
-    )
-    missed_stats: Optional[DeltaStatsCacheMiss] = (
-        DeltaStatsCacheMiss(missed_columns, delta.locator) if missed_columns else None
-    )
-
-    return DeltaStatsCacheResult.of(found_stats, missed_stats)
-
-
-@ray.remote
-def cache_delta_column_stats(
-    stat_results_s3_bucket: str, dataset_column: DeltaColumnStats
-) -> None:
-    """Ray distributed task to cache the delta column stats into a file system (i.e. S3).
-
-    Args:
-        stat_results_s3_bucket: The S3 bucket name
-        dataset_column: Column-oriented stats for a given delta
-    """
-    write_manifest_stats_file(
-        stat_results_s3_bucket, dataset_column.column, dataset_column.manifest_stats
-    )
-
-
-@ray.remote
-def get_delta_stats(
-    delta_locator: DeltaLocator,
-    columns: Optional[List[str]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
-    deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
-) -> DeltaStats:
-    """Ray distributed task to compute and collect stats for a requested delta.
-    If no columns are requested, stats will be computed for all columns.
-    Args:
-        delta_locator: A reference to the delta
-        columns: Column names to specify for this delta. If not provided, all columns are considered.
-        deltacat_storage: Client implementation of the DeltaCAT storage interface
-    Returns:
-        A delta wide stats container
-    """
-    if deltacat_storage_kwargs is None:
-        deltacat_storage_kwargs = {}
-    manifest = deltacat_storage.get_delta_manifest(
-        delta_locator, **deltacat_storage_kwargs
-    )
-    delta = Delta.of(delta_locator, None, None, None, manifest)
-    return _collect_stats_by_columns(
-        delta, columns, deltacat_storage, deltacat_storage_kwargs
-    )
-
-
-@ray.remote
-def get_deltas_from_range(
-    source_partition_locator: PartitionLocator,
-    start_position_inclusive: DeltaRange,
-    end_position_inclusive: DeltaRange,
-    deltacat_storage=unimplemented_deltacat_storage,
-    **kwargs,
-) -> List[Delta]:
-    """Looks up deltas in the specified partition using Ray, given both starting and ending delta stream positions.
-
-    Args:
-        source_partition_locator: Reference to the partition locator tied to the given delta stream positions
-        start_position_inclusive: Starting stream position of a range interval.
-            Can be an int type to represent a closed bounded range, or a None type to represent unbounded infinity.
-        end_position_inclusive: Ending stream position of a range interval.
-            Can be an int type to represent a closed bounded range, or a None type to represent unbounded infinity.
-        deltacat_storage: Client implementation of the DeltaCAT storage interface
-
-    Returns:
-        a list of delta objects
-    """
-
-    namespace, partition_values = (
-        source_partition_locator.namespace,
-        source_partition_locator.partition_values,
-    )
-    table_name, table_version = (
-        source_partition_locator.table_name,
-        source_partition_locator.table_version,
-    )
-    deltas_list_result = deltacat_storage.list_deltas(
-        namespace,
-        table_name,
-        partition_values,
-        table_version,
-        start_position_inclusive,
-        end_position_inclusive,
-        ascending_order=True,
-        include_manifest=False,
-        **kwargs,
-    )
-    return deltas_list_result.all_items()
-
-
-def _collect_stats_by_columns(
-    delta: Delta,
-    columns_to_compute: Optional[List[str]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
-    deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
-) -> DeltaStats:
-    """Materializes one manifest entry at a time to save memory usage and calculate stats from each of its columns.
-    Args:
-        delta: A delta object to calculate stats for
-        columns_to_compute: Columns to calculate stats for. If not provided, all columns are considered.
-        deltacat_storage: Client implementation of the DeltaCAT storage interface
-    Returns:
-        A delta wide stats container
-    """
-    if deltacat_storage_kwargs is None:
-        deltacat_storage_kwargs = {}
-    assert (
-        delta.manifest is not None
-    ), f"Manifest should not be missing from delta for stats calculation: {delta}"
-
-    # Mapping of column_name -> [stats_file_idx_1, stats_file_idx_2, ... stats_file_idx_n]
-    column_stats_map: Dict[str, List[Optional[StatsResult]]] = defaultdict(
-        lambda: [None] * len(delta.manifest.entries)
-    )
-
-    total_tables_size = 0
-    for file_idx, manifest in enumerate(delta.manifest.entries):
-        entry_pyarrow_table: LocalTable = (
-            deltacat_storage.download_delta_manifest_entry(
-                delta,
-                file_idx,
-                TableType.PYARROW,
-                columns_to_compute,
-                **deltacat_storage_kwargs,
-            )
-        )
-        assert isinstance(entry_pyarrow_table, pyarrow.Table), (
-            f"Stats collection is only supported for PyArrow tables, but received a table of "
-            f"type '{type(entry_pyarrow_table)}' for manifest entry {file_idx} of delta: {delta.locator}."
-        )
-        total_tables_size += entry_pyarrow_table.nbytes
-        if not columns_to_compute:
-            columns_to_compute = entry_pyarrow_table.column_names
-
-        for column_idx, pyarrow_column in enumerate(entry_pyarrow_table.columns):
-            column_name = columns_to_compute[column_idx]
-            column_stats_map[column_name][file_idx] = StatsResult.of(
-                len(pyarrow_column), pyarrow_column.nbytes
-            )
-
-    # Add column-wide stats for a list of tables, these will be used for caching and retrieving later
-    delta_ds_column_stats: List[DeltaColumnStats] = _to_dataset_column_stats(
-        delta.locator, columns_to_compute, column_stats_map
-    )
-
-    dataset_stats: DeltaStats = DeltaStats.of(delta_ds_column_stats)
-
-    # Quick validation for calculations
-    assert dataset_stats.stats.pyarrow_table_bytes == total_tables_size, (
-        f"Expected the size of all PyArrow tables ({total_tables_size} bytes) "
-        f"to match the sum of each of its columns ({dataset_stats.stats.pyarrow_table_bytes} bytes)"
-    )
-
-    return dataset_stats
-
-
-def _to_dataset_column_stats(
-    delta_locator: DeltaLocator,
-    column_names: List[str],
-    column_manifest_map: Dict[str, List[Optional[StatsResult]]],
-) -> List[DeltaColumnStats]:
-    dataset_stats: List[DeltaColumnStats] = []
-    for column_name in column_names:
-        column_manifest_stats = ManifestEntryStats.of(
-            column_manifest_map[column_name], delta_locator
-        )
-        dataset_column_stats = DeltaColumnStats.of(column_name, column_manifest_stats)
-        dataset_stats.append(dataset_column_stats)
-    return dataset_stats

deltacat/compute/stats/utils/manifest_stats_file.py

@@ -1,100 +0,0 @@
-import json
-import logging
-from typing import List
-
-from deltacat import logs
-from deltacat.aws import s3u as s3_utils
-from deltacat.compute.stats.models.delta_column_stats import DeltaColumnStats
-from deltacat.compute.stats.models.manifest_entry_stats import ManifestEntryStats
-from deltacat.storage import DeltaLocator
-from deltacat.utils.common import sha1_hexdigest
-
-logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
-
-
-def get_manifest_stats_s3_url(
-    bucket: str, column_name: str, delta_locator: DeltaLocator
-) -> str:
-    """Returns the S3 URL path to the column-oriented delta stats
-
-    Args:
-        bucket: The S3 bucket
-        column_name: The name of the column to look up stats for
-        delta_locator: The reference to the delta corresponding to the manifest entries
-
-    Returns:
-        A S3 URL path
-    """
-    stats_column_id = f"{delta_locator.canonical_string()}|{column_name}"
-    stats_column_hexdigest = sha1_hexdigest(stats_column_id.encode("utf-8"))
-    base_path = s3_utils.parse_s3_url(bucket).url
-    return f"{base_path}/{stats_column_hexdigest}.json"
-
-
-def read_manifest_stats_by_columns(
-    bucket: str, column_names: List[str], delta_locator: DeltaLocator
-) -> List[DeltaColumnStats]:
-    """Fetch a list of delta column stats by reading each column-oriented delta stats file from S3
-
-    Args:
-        bucket: The S3 bucket
-        column_names: A list of column names to look up stats for
-        delta_locator: The reference to the delta corresponding to the manifest entries
-
-    Returns:
-        A list of delta column stats
-    """
-    return [
-        DeltaColumnStats.of(
-            column, read_manifest_stats_file(bucket, column, delta_locator)
-        )
-        for column in column_names
-    ]
-
-
-def read_manifest_stats_file(
-    bucket: str, column_name: str, delta_locator: DeltaLocator
-) -> ManifestEntryStats:
-    """Read a manifest entry stats from S3
-
-    Args:
-        bucket: The S3 bucket
-        column_name: The name of the column to look up stats for
-        delta_locator: The reference to the delta corresponding to the manifest entries
-
-    Returns:
-        A container that holds a list of manifest entry stats for the given column name
-    """
-
-    stats_completion_file_url = get_manifest_stats_s3_url(
-        bucket, column_name, delta_locator
-    )
-    logger.info(f"reading stats completion file from: {stats_completion_file_url}")
-    stats_completion_info_file = None
-    result = s3_utils.download(stats_completion_file_url, fail_if_not_found=False)
-    if result:
-        json_str = result["Body"].read().decode("utf-8")
-        stats_completion_info_file = ManifestEntryStats(json.loads(json_str))
-        logger.info(f"read stats completion info: {stats_completion_info_file}")
-    return stats_completion_info_file
-
-
-def write_manifest_stats_file(
-    bucket: str, column_name: str, manifest_entry_stats: ManifestEntryStats
-) -> None:
-    """Write a manifest entry stats into S3
-
-    Args:
-        bucket: The S3 bucket
-        column_name: The name of the column which represents this manifest entry stats
-        manifest_entry_stats: The manifest entry stats to serialize and store into S3
-    """
-    logger.info(f"writing stats completion file contents: {manifest_entry_stats}")
-    stats_completion_file_s3_url = get_manifest_stats_s3_url(
-        bucket,
-        column_name,
-        manifest_entry_stats.delta_locator,
-    )
-    logger.info(f"writing stats completion file to: {stats_completion_file_s3_url}")
-    s3_utils.upload(stats_completion_file_s3_url, str(json.dumps(manifest_entry_stats)))
-    logger.info(f"stats completion file written to: {stats_completion_file_s3_url}")

deltacat/tests/stats/test_intervals.py

@@ -1,49 +0,0 @@
-import unittest
-from typing import Tuple
-
-from deltacat.compute.stats.utils.intervals import DeltaRange, merge_intervals
-
-
-class TestMergeIntervals(unittest.TestCase):
-    def test_unbounded_start_range(self):
-        intervals = sorted(merge_intervals({(3, 9), (None, 15), (13, 30)}))
-        interval: Tuple[DeltaRange, DeltaRange] = intervals[0]
-        self.assertEqual(interval[0], None)
-        self.assertEqual(interval[1], 30)
-
-    def test_unbounded_end_range(self):
-        intervals = sorted(merge_intervals({(3, 9), (2, None), (13, 30)}))
-        interval: Tuple[DeltaRange, DeltaRange] = intervals[0]
-        self.assertEqual(interval[0], 2)
-        self.assertEqual(interval[1], None)
-
-    def test_unbounded_start_end_range(self):
-        intervals = sorted(merge_intervals({(None, None)}))
-        interval: Tuple[DeltaRange, DeltaRange] = intervals[0]
-        self.assertEqual(interval[0], None)
-        self.assertEqual(interval[1], None)
-
-    def test_no_overlap_range(self):
-        intervals = sorted(merge_intervals({(3, 9), (11, 14), (19, 30)}))
-        interval1: Tuple[DeltaRange, DeltaRange] = intervals[0]
-        interval2: Tuple[DeltaRange, DeltaRange] = intervals[1]
-        interval3: Tuple[DeltaRange, DeltaRange] = intervals[2]
-        self.assertEqual(interval1, (3, 9))
-        self.assertEqual(interval2, (11, 14))
-        self.assertEqual(interval3, (19, 30))
-
-    def test_overlap_range(self):
-        intervals = sorted(merge_intervals({(3, 9), (9, 14), (14, 30)}))
-        interval1: Tuple[DeltaRange, DeltaRange] = intervals[0]
-        self.assertEqual(interval1, (3, 30))
-
-    def test_invalid_range(self):
-        self.assertRaises(ValueError, merge_intervals, {(3, 9), (9, 3)})
-
-    def test_invalid_type(self):
-        self.assertRaises(ValueError, merge_intervals, {(3, 9), (1.2, 3)})
-        self.assertRaises(ValueError, merge_intervals, {(3, 9), ("1", 3)})
-
-
-if __name__ == "__main__":
-    unittest.main()