deltacat 1.1.17__py3-none-any.whl → 1.1.19__py3-none-any.whl

deltacat/compute/metastats/stats.py

@@ -1,182 +0,0 @@
-import logging
-from collections import defaultdict
-from typing import Dict, List, Optional
-
-import ray
-from ray.types import ObjectRef
-
-from deltacat import logs
-from deltacat.aws import s3u as s3_utils
-from deltacat.aws.clients import client_cache
-from deltacat.aws.constants import AWS_REGION
-from deltacat.compute.compactor import DeltaAnnotated
-from deltacat.compute.metastats.utils.io import (
-    cache_inflation_rate_data_for_delta_stats_ready,
-    cache_partition_stats_to_s3,
-    collect_stats_by_columns,
-)
-from deltacat.compute.stats.models.delta_column_stats import DeltaColumnStats
-from deltacat.compute.stats.models.delta_stats import DeltaStats
-from deltacat.compute.stats.models.manifest_entry_stats import ManifestEntryStats
-from deltacat.compute.stats.models.stats_result import StatsResult
-from deltacat.storage import DeltaLocator, PartitionLocator
-from deltacat.storage import interface as unimplemented_deltacat_storage
-
-logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
-
-# TODO: get cpu info from ray.nodes() resource key
-DEFAULT_CPUS_STATS_CLUSTER_INSTANCE = 32
-
-
-def start_stats_collection(
-    batched_delta_stats_compute_list: List[DeltaAnnotated],
-    columns: List[str],
-    stat_results_s3_bucket: Optional[str] = None,
-    metastats_results_s3_bucket: Optional[str] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
-    **kwargs,
-) -> Dict[str, List[DeltaStats]]:
-    """Collects statistics on deltas, given a set of delta stream position ranges.
-    Example:
-        >>> collect(locator, set((1, 5), (4, 8), (13, 16)))
-        {
-            1: DeltaStats(),  # DeltaStats for stream positions 1 - 8
-            13: DeltaStats()  # DeltaStats for stream positions 13 - 16
-        }
-    Args:
-        source_partition_locator: Reference to the partition locator tied to the given delta stream positions
-        delta_stream_position_range_set: A set of intervals with an int type representing finite,
-            closed bounded values, and a None type representing unbounded infinity.
-        columns: Columns can be optionally included to collect stats on specific columns.
-            By default, all columns will be calculated.
-        stat_results_s3_bucket: Used as a cache file storage for computed delta stats
-        metastats_results_s3_bucket: Used as cache file storage for inflation rate meta stats
-        deltacat_storage: Client implementation of the DeltaCAT storage interface
-    Returns:
-        A mapping of stream positions to their corresponding delta stats.
-    """
-    # TODO: Add CompactionEventDispatcher for stats collection started event
-    delta_stats_compute_pending: List[ObjectRef[Dict[str, List[StatsResult, int]]]] = []
-
-    for batched_deltas in batched_delta_stats_compute_list:
-        splitted_annotated_deltas = DeltaAnnotated.split(
-            batched_deltas, DEFAULT_CPUS_STATS_CLUSTER_INSTANCE
-        )
-        for splitted_annotated_delta in splitted_annotated_deltas:
-            delta_stats_compute_pending.append(
-                collect_stats_by_columns.remote(
-                    splitted_annotated_delta, columns, deltacat_storage
-                )
-            )
-
-    column_stats_map = _process_stats(delta_stats_compute_pending)
-
-    if not batched_delta_stats_compute_list:
-        logger.info("No new delta need stats collection")
-    else:
-        (
-            delta_stream_range_stats,
-            partition_canonical_string,
-        ) = resolve_annotated_delta_stats_to_original_deltas_stats(
-            column_stats_map, columns, batched_delta_stats_compute_list[0]
-        )
-
-        _cache_stats_res_to_s3(
-            stat_results_s3_bucket, delta_stream_range_stats, partition_canonical_string
-        )
-
-        base_path = s3_utils.parse_s3_url(metastats_results_s3_bucket).url
-        inflation_rate_stats_s3_url = f"{base_path}/inflation-rates.json"
-        cache_inflation_rate_data_for_delta_stats_ready(
-            delta_stream_range_stats, inflation_rate_stats_s3_url, deltacat_storage
-        )
-        # TODO: Add CompactionEventDispatcher for stats collection completed event
-        return delta_stream_range_stats
-
-
-def _get_account_id() -> str:
-    client = client_cache("sts", region_name=AWS_REGION)
-    account_id = client.get_caller_identity()["Account"]
-    return account_id
-
-
-def _process_stats(
-    delta_stats_compute_pending: List[ObjectRef[DeltaStats]],
-) -> List[DeltaStats]:
-    delta_stats_processed_list: List[DeltaStats] = _resolve_pending_stats(
-        delta_stats_compute_pending
-    )
-
-    return delta_stats_processed_list
-
-
-def _resolve_pending_stats(
-    delta_stats_pending_list: List[ObjectRef[DeltaStats]],
-) -> List[DeltaStats]:
-    delta_stats_processed_list: List[DeltaStats] = []
-
-    while delta_stats_pending_list:
-        ready, delta_stats_pending_list = ray.wait(delta_stats_pending_list)
-        processed_stats_batch: List[DeltaStats] = ray.get(ready)
-        delta_stats_processed_list.extend(processed_stats_batch)
-
-    return delta_stats_processed_list
-
-
-def _cache_stats_res_to_s3(
-    stat_results_s3_bucket, delta_stream_range_stats, partition_canonical_string
-):
-    if stat_results_s3_bucket:
-        # Cache the stats into the file store
-        cache_partition_stats_to_s3(
-            stat_results_s3_bucket, delta_stream_range_stats, partition_canonical_string
-        )
-
-
-def resolve_annotated_delta_stats_to_original_deltas_stats(
-    column_stats_map, column_names, delta_annotated
-) -> Dict[int, DeltaStats]:
-
-    partition_values = delta_annotated["deltaLocator"]["partitionLocator"][
-        "partitionValues"
-    ]
-    partition_id = delta_annotated["deltaLocator"]["partitionLocator"]["partitionId"]
-    stream_locator = delta_annotated["deltaLocator"]["partitionLocator"][
-        "streamLocator"
-    ]
-    partition_locator = PartitionLocator.of(
-        stream_locator, partition_values, partition_id
-    )
-
-    # Dict[stream_position: List[StatsResult]]
-    manifest_column_stats_list = defaultdict(lambda: [])
-    for i in range(len(column_stats_map)):
-        for column_name in column_names:
-            for j in range(len(column_stats_map[i][column_name])):
-                manifest_column_stats_list[
-                    column_stats_map[i][column_name][j][1]
-                ].append([column_stats_map[i][column_name][j][0], column_name])
-
-    stats_res: Dict[int, List[DeltaStats]] = {}
-    for key, value in manifest_column_stats_list.items():
-        delta_locator = DeltaLocator.of(partition_locator, key)
-
-        # Dict[column_name: List[StatsResult]]
-        manifest_stats_list = defaultdict(lambda: [])
-        for manifest_stat in value:
-            manifest_stats_list[manifest_stat[1]].append(manifest_stat[0])
-        delta_ds_column_stats: List[DeltaColumnStats] = []
-        for column_name, column_manifest_stats_list in manifest_stats_list.items():
-
-            column_manifest_stats = ManifestEntryStats.of(
-                column_manifest_stats_list, delta_locator
-            )
-            dataset_column_stats = DeltaColumnStats.of(
-                column_name, column_manifest_stats
-            )
-            delta_ds_column_stats.append(dataset_column_stats)
-
-        dataset_stats: DeltaStats = DeltaStats.of(delta_ds_column_stats)
-        stats_res[key] = dataset_stats
-
-    return stats_res, partition_locator.canonical_string()

deltacat/compute/metastats/utils/constants.py

@@ -1,16 +0,0 @@
-# Default to use r5.8xlarge instance type for stats collection cluster
-STATS_CLUSTER_R5_INSTANCE_TYPE = 8
-# Using R5 instance type, 8GiB memory is available per cpu
-R5_MEMORY_PER_CPU = 8
-# Default to use r5.8xlarge instance type for stats collection cluster
-DEFAULT_CPUS_PER_INSTANCE_R5_8XLARGE = 32
-# memory reserved for head node object store
-HEAD_NODE_OBJECT_STORE_MEMORY_RESERVE_RATIO = 0.3
-# memory reserved for worker node object store
-WORKER_NODE_OBJECT_STORE_MEMORY_RESERVE_RATIO = 0.1
-# each cpu should not be processing more than this number of files to avoid unreasonable S3 I/O latency
-MANIFEST_FILE_COUNT_PER_CPU = 200
-# MAX_WORKER_MULTIPLIER * min_workers = max_workers to determine max workers based on min workers given
-MAX_WORKER_MULTIPLIER = 2
-# default trace id used for metastats collection triggered without trace id
-DEFAULT_JOB_RUN_TRACE_ID = "0"

deltacat/compute/metastats/utils/io.py

@@ -1,223 +0,0 @@
-import json
-import logging
-from collections import defaultdict
-from typing import Any, Dict, List, Optional
-
-import pyarrow
-import ray
-
-from deltacat import LocalTable, TableType, logs
-from deltacat.aws import s3u as s3_utils
-from deltacat.compute.compactor import DeltaAnnotated
-from deltacat.compute.metastats.model.partition_stats_dict import PartitionStats
-from deltacat.compute.stats.models.delta_column_stats import DeltaColumnStats
-from deltacat.compute.stats.models.delta_stats import DeltaStats, DeltaStatsCacheMiss
-from deltacat.compute.stats.models.delta_stats_cache_result import DeltaStatsCacheResult
-from deltacat.compute.stats.models.stats_result import StatsResult
-from deltacat.storage import Delta
-from deltacat.storage import interface as unimplemented_deltacat_storage
-from deltacat.utils.common import sha1_hexdigest
-
-logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
-
-
-def cache_inflation_rate_data_for_delta_stats_ready(
-    delta_stats_processed_list, inflation_rate_stats_s3_url, deltacat_storage
-):
-    meta_stats_processed_list: Dict[int, int] = {}
-
-    for key, value in delta_stats_processed_list.items():
-        delta_locator = value.column_stats[0].manifest_stats.delta_locator
-        delta_meta_count = 0
-        manifest = deltacat_storage.get_delta_manifest(delta_locator)
-        delta = Delta.of(delta_locator, None, None, None, manifest)
-        for entry in delta.manifest.entries:
-            delta_meta_count += entry.meta.content_length
-        meta_stats_processed_list[delta.stream_position] = delta_meta_count
-
-    cache_inflation_rate_res = dict()
-
-    for key, value in delta_stats_processed_list.items():
-        delta_stats_pyarrow_bytes_sum = 0
-        delta_stats_row_count = 0
-        for column_stats in (
-            delta_stats_processed_list[key].column_stats[0].manifest_stats.stats
-        ):
-            delta_stats_row_count += column_stats.get("rowCount")
-        for stats in delta_stats_processed_list[key].get("column_stats"):
-
-            delta_stats_pyarrow_bytes_sum += stats.get("stats").get("pyarrowTableBytes")
-        cache_inflation_rate_res[key] = [
-            meta_stats_processed_list[key],
-            delta_stats_row_count,
-            delta_stats_pyarrow_bytes_sum,
-        ]
-
-    if inflation_rate_stats_s3_url:
-        logger.warning(
-            f"reading previous inflation rate stats from: {inflation_rate_stats_s3_url}"
-        )
-
-        result = s3_utils.download(inflation_rate_stats_s3_url, fail_if_not_found=False)
-
-        prev_inflation_rate_stats = dict()
-        if result:
-            json_str = result["Body"].read().decode("utf-8")
-            prev_inflation_rate_stats_read = json.loads(json_str)
-            prev_inflation_rate_stats = (
-                prev_inflation_rate_stats_read
-                if prev_inflation_rate_stats_read
-                else dict()
-            )
-            logger.debug(
-                f"read stats completion info: {prev_inflation_rate_stats_read}"
-            )
-        logger.debug(
-            f"writing inflation rate info to S3: {inflation_rate_stats_s3_url}"
-        )
-        prev_inflation_rate_stats.update(cache_inflation_rate_res)
-        logger.debug(
-            f"writing current inflation rate info to S3: {prev_inflation_rate_stats}"
-        )
-        s3_utils.upload(
-            inflation_rate_stats_s3_url, json.dumps(prev_inflation_rate_stats)
-        )
-    else:
-        logger.warning(
-            f"No valid s3 url received to cache inflation rate stats, got {inflation_rate_stats_s3_url}"
-        )
-
-
-def read_cached_partition_stats(
-    partition_canonical_string: str, stat_results_s3_bucket: str
-):
-    partition_stats_url = get_partition_stats_s3_url(
-        partition_canonical_string, stat_results_s3_bucket
-    )
-    logger.info(f"reading partition stats completion file from: {partition_stats_url}")
-
-    result = s3_utils.download(partition_stats_url, fail_if_not_found=False)
-    delta_stats_cache_res_map: Dict[int, List[DeltaStatsCacheResult]] = {}
-    if result:
-        json_str = result["Body"].read().decode("utf-8")
-        partition_stats_str = json.loads(json_str)
-        delta_stats_cache_res_map = get_delta_stats_from_partition_stats(
-            partition_stats_str
-        )
-
-    return delta_stats_cache_res_map
-
-
-def get_partition_stats_s3_url(
-    partition_canonical_string: str, stat_results_s3_bucket: str
-):
-    stats_partition_canonical_string = f"{partition_canonical_string}"
-    stats_partition_hexdigest = sha1_hexdigest(
-        stats_partition_canonical_string.encode("utf-8")
-    )
-    base_path = s3_utils.parse_s3_url(stat_results_s3_bucket).url
-
-    return f"{base_path}/{stats_partition_hexdigest}.json"
-
-
-def get_delta_stats_from_partition_stats(partition_stats_str: str):
-
-    partition_stats = PartitionStats.build_from_dict(partition_stats_str)
-
-    found_columns_stats_map: Dict[int, List[DeltaStatsCacheResult]] = {}
-    for stream_position, delta_stats in partition_stats.delta_stats.items():
-        found_columns_stats: List[DeltaColumnStats] = []
-        missed_columns: List[str] = []
-        for cs in delta_stats.column_stats:
-            if cs.manifest_stats:
-                found_columns_stats.append(cs)
-            else:
-                missed_columns.append(cs.column)
-
-        delta_locator = delta_stats.column_stats[0].manifest_stats.delta_locator
-        found_stats: Optional[DeltaStats] = (
-            DeltaStats.of(found_columns_stats) if found_columns_stats else None
-        )
-        missed_stats: Optional[DeltaStatsCacheMiss] = (
-            DeltaStatsCacheMiss(missed_columns, delta_locator)
-            if missed_columns
-            else None
-        )
-        delta_stats_cache_res = DeltaStatsCacheResult.of(found_stats, missed_stats)
-        found_columns_stats_map[int(stream_position)] = delta_stats_cache_res
-    return found_columns_stats_map
-
-
-def cache_partition_stats_to_s3(
-    stat_results_s3_bucket, delta_stream_range_stats, partition_canonical_string
-):
-    partition_stats = PartitionStats.of(
-        delta_stream_range_stats, partition_canonical_string
-    )
-    logger.info(f"writing partition stats completion for {partition_canonical_string}")
-    partition_stats_completion_file_s3_url = get_partition_stats_s3_url(
-        partition_canonical_string, stat_results_s3_bucket
-    )
-    s3_utils.upload(
-        partition_stats_completion_file_s3_url, str(json.dumps(partition_stats))
-    )
-    logger.debug(
-        f"stats completion file written to: {partition_stats_completion_file_s3_url}"
-    )
-
-
-@ray.remote
-def collect_stats_by_columns(
-    delta_annotated: DeltaAnnotated,
-    columns_to_compute: Optional[List[str]] = None,
-    deltacat_storage=unimplemented_deltacat_storage,
-    deltacat_storage_kwargs: Optional[Dict[str, Any]] = None,
-) -> Dict[str, Any]:
-    """Materializes one manifest entry at a time to save memory usage and calculate stats from each of its columns.
-
-    Args:
-        delta: A delta object to calculate stats for
-        columns_to_compute: Columns to calculate stats for. If not provided, all columns are considered.
-        deltacat_storage: Client implementation of the DeltaCAT storage interface
-
-    Returns:
-        A delta wide stats container
-    """
-    if deltacat_storage_kwargs is None:
-        deltacat_storage_kwargs = {}
-    total_tables_size = 0
-
-    # Mapping of column_name -> [stats_file_idx_1, stats_file_idx_2, ... stats_file_idx_n]
-    column_stats_map = defaultdict(
-        lambda: [[None, None]] * len(delta_annotated["manifest"].get("entries"))
-    )
-    src_da_entries = delta_annotated["manifest"].get("entries")
-    manifest_annotations = delta_annotated["annotations"]
-    for file_idx, manifest in enumerate(src_da_entries):
-        entry_pyarrow_table: LocalTable = (
-            deltacat_storage.download_delta_manifest_entry(
-                delta_annotated,
-                file_idx,
-                TableType.PYARROW,
-                columns_to_compute,
-                equivalent_table_types="uncompacted",
-                **deltacat_storage_kwargs,
-            )
-        )
-        assert isinstance(entry_pyarrow_table, pyarrow.Table), (
-            f"Stats collection is only supported for PyArrow tables, but received a table of "
-            f"type '{type(entry_pyarrow_table)}' for manifest entry {file_idx} of delta: {delta_annotated.locator}."
-        )
-        total_tables_size += entry_pyarrow_table.nbytes
-        if not columns_to_compute:
-            columns_to_compute = entry_pyarrow_table.column_names
-
-        for column_idx, pyarrow_column in enumerate(entry_pyarrow_table.columns):
-            column_name = columns_to_compute[column_idx]
-            origin_delta_stream_position = manifest_annotations[file_idx][-1]
-            column_stats_map[column_name][file_idx] = [
-                StatsResult.of(len(pyarrow_column), pyarrow_column.nbytes),
-                origin_delta_stream_position,
-            ]
-
-    return column_stats_map

deltacat/compute/metastats/utils/pyarrow_memory_estimation_function.py

@@ -1,18 +0,0 @@
-from deltacat.constants import PYARROW_INFLATION_MULTIPLIER_ALL_COLUMNS
-
-
-def estimation_function(
-    content_length, content_type, content_encoding, *args, **kwargs
-):
-    # TODO(zyiqin): update the estimation here to be consistent with number of required worker nodes estimate.
-    #  Current implementation is only a rough guess using the PYARROW_INFLATION_MULTIPLIER(content_length to pyarrow butes(all columns).
-    #  The full implementation logic should be:
-    #      1. liner regression with a confidence level: pull metastats data for all deltas for this partition if len(datapoints) > 30.
-    #      2. if not enough previous stats collected for same partition: Fall back to datapoints for all paritions for same table.
-    #      3. If not enough stats collected for this table: use average content length to each content_type and content_encoding inflation rates
-    #      4. If not enough stats for this content_type and content_encoding combination: use the basic PYARROW_INFLATION_MULTIPLIER instead.
-
-    if content_length:
-        return content_length * PYARROW_INFLATION_MULTIPLIER_ALL_COLUMNS
-    else:
-        return 0

deltacat/compute/metastats/utils/ray_utils.py

@@ -1,129 +0,0 @@
-import errno
-import logging
-import os
-import subprocess
-from subprocess import run
-from typing import Any
-
-import ray
-from tenacity import RetryError, Retrying, stop_after_attempt, wait_fixed
-
-from deltacat import logs
-from deltacat.compute.metastats.utils.constants import (
-    MAX_WORKER_MULTIPLIER,
-    WORKER_NODE_OBJECT_STORE_MEMORY_RESERVE_RATIO,
-)
-
-logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
-
-RAY_DOWN_DEFAULT_RETRY_ATTEMPTS = 3
-
-
-def run_cmd_exit_code(cmd: str) -> int:
-    logger.info(f"running command {cmd}")
-    exit_code = int(os.system(cmd))
-    logger.info(f"Got {exit_code} when running {cmd}")
-
-
-def run_cmd_with_retry(cmd: str) -> None:
-    retrying = Retrying(
-        wait=wait_fixed(2), stop=stop_after_attempt(RAY_DOWN_DEFAULT_RETRY_ATTEMPTS)
-    )
-    try:
-        retrying(run_cmd_exit_code(cmd))
-    except RetryError:
-        logger.info(f"{cmd} failed after {RAY_DOWN_DEFAULT_RETRY_ATTEMPTS} retries.")
-
-
-def run_cmd(cmd: str) -> None:
-    result = run(cmd, shell=True, capture_output=True)
-    exit_code = int(result.returncode)
-    assert exit_code == 0, (
-        f"`{cmd}` failed. Exit code: {exit_code} " f"Error Trace: {result.stderr}"
-    )
-
-
-def ray_up(cluster_cfg: str) -> None:
-    logger.info(f"Starting Ray cluster '{cluster_cfg}'")
-    run_cmd(f"ray up {cluster_cfg} -y --no-config-cache --no-restart")
-    logger.info(f"Started Ray cluster '{cluster_cfg}'")
-
-
-def ray_down(cluster_cfg: str) -> None:
-    logger.info(f"Destroying Ray cluster '{cluster_cfg}'")
-    run_cmd_with_retry(f"ray down {cluster_cfg} -y")
-    logger.info(f"Destroyed Ray cluster '{cluster_cfg}'")
-
-
-def clean_up_cluster_cfg_file(cluster_cfg) -> None:
-    logger.info(f"Removing stats cluster config at: '{cluster_cfg}'")
-    run_cmd(f"rm -f {cluster_cfg}")
-    logger.info(f"Removed stats cluster config at: '{cluster_cfg}'")
-
-
-def get_head_node_ip(cluster_cfg: str) -> str:
-    logger.info(f"Getting Ray cluster head node IP for '{cluster_cfg}'")
-    proc = subprocess.run(
-        f"ray get-head-ip {cluster_cfg}",
-        shell=True,
-        capture_output=True,
-        text=True,
-        check=True,
-    )
-    # the head node IP should be the last line printed to stdout
-    head_node_ip = proc.stdout.splitlines()[-1]
-    logger.info(f"Ray cluster head node IP for '{cluster_cfg}': {head_node_ip}")
-    return head_node_ip
-
-
-def ray_init(host, port) -> Any:
-    ray_init_uri = f"ray://{host}:{port}"
-    logger.info(f"Connecting Ray Client to '{ray_init_uri}'")
-    client = ray.init(ray_init_uri, allow_multiple=True)
-    logger.info(f"Connected Ray Client to '{ray_init_uri}'")
-    return client
-
-
-def replace_cluster_cfg_vars(
-    partition_canonical_string: str,
-    trace_id: str,
-    file_path: str,
-    min_workers: int,
-    head_type: str,
-    worker_type: str,
-    head_object_store_memory_pct: int,
-    worker_object_store_memory_pct: int,
-) -> str:
-
-    head_object_store_memory_pct = head_object_store_memory_pct if not None else 30
-    worker_object_store_memory_pct = WORKER_NODE_OBJECT_STORE_MEMORY_RESERVE_RATIO * 100
-
-    max_workers = int(min_workers * MAX_WORKER_MULTIPLIER)
-    with open(file_path, "r+") as file:
-        contents = file.read().replace("{{use-internal-ips}}", "True")
-        contents = contents.replace(
-            "{{partition_canonical_string}}", partition_canonical_string
-        )
-        contents = contents.replace("'{{trace_id}}'", trace_id)
-        contents = contents.replace("'{{min-workers}}'", str(min_workers))
-        contents = contents.replace("'{{max-workers}}'", str(max_workers))
-        contents = contents.replace("'{{head-instance-type}}'", head_type)
-        contents = contents.replace("'{{worker-instance-type}}'", worker_type)
-        contents = contents.replace(
-            "'{{head-object-store-memory-pct}}'", str(head_object_store_memory_pct)
-        )
-        contents = contents.replace(
-            "'{{worker-object-store-memory-pct}}'", str(worker_object_store_memory_pct)
-        )
-    partition_id = partition_canonical_string.split("|")[-1]
-    out_file_name = f"{trace_id}-{partition_id}.{os.path.basename(file_path)}"
-    out_file_dir = os.path.join(os.path.dirname(file_path), "tmp")
-    out_file_path = os.path.join(out_file_dir, out_file_name)
-    try:
-        os.makedirs(os.path.dirname(out_file_path))
-    except OSError as e:
-        if e.errno != errno.EEXIST:
-            raise
-    with open(out_file_path, "w") as output:
-        output.write(contents)
-    return out_file_path