deltacat 0.1.8__py3-none-any.whl → 0.1.11__py3-none-any.whl

deltacat/utils/ray_utils/dataset.py

@@ -1,26 +1,25 @@
 import logging
+from typing import Callable, Dict, List, Optional
 
-from pyarrow import csv as pacsv
 from fsspec import AbstractFileSystem
-
+from pyarrow import csv as pacsv
 from ray.data import Dataset
 from ray.data.datasource import BlockWritePathProvider
 
 from deltacat import logs
-from deltacat.types.media import ContentType, ContentEncoding
-
-from typing import Callable, Dict, List, Optional
+from deltacat.types.media import ContentEncoding, ContentType
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 def write_parquet(
-        dataset: Dataset,
-        base_path: str,
-        *,
-        filesystem: AbstractFileSystem,
-        block_path_provider: BlockWritePathProvider,
-        **kwargs) -> None:
+    dataset: Dataset,
+    base_path: str,
+    *,
+    filesystem: AbstractFileSystem,
+    block_path_provider: BlockWritePathProvider,
+    **kwargs,
+) -> None:
 
     dataset.write_parquet(
         base_path,
@@ -32,12 +31,13 @@ def write_parquet(
 
 
 def write_csv(
-        dataset: Dataset,
-        base_path: str,
-        *,
-        filesystem: AbstractFileSystem,
-        block_path_provider: BlockWritePathProvider,
-        **kwargs) -> None:
+    dataset: Dataset,
+    base_path: str,
+    *,
+    filesystem: AbstractFileSystem,
+    block_path_provider: BlockWritePathProvider,
+    **kwargs,
+) -> None:
 
     # column names are kept in table metadata, so omit header
     arrow_csv_args_fn = lambda: {
@@ -61,9 +61,7 @@ CONTENT_TYPE_TO_DATASET_WRITE_FUNC: Dict[str, Callable] = {
 }
 
 
-def slice_dataset(
-        dataset: Dataset,
-        max_len: Optional[int]) -> List[Dataset]:
+def slice_dataset(dataset: Dataset, max_len: Optional[int]) -> List[Dataset]:
     """
     Returns equally-sized dataset slices of up to `max_len` records each.
     """
@@ -88,12 +86,13 @@ def dataset_size(dataset: Dataset) -> int:
 
 
 def dataset_to_file(
-        table: Dataset,
-        base_path: str,
-        file_system: AbstractFileSystem,
-        block_path_provider: BlockWritePathProvider,
-        content_type: str = ContentType.PARQUET.value,
-        **kwargs) -> None:
+    table: Dataset,
+    base_path: str,
+    file_system: AbstractFileSystem,
+    block_path_provider: BlockWritePathProvider,
+    content_type: str = ContentType.PARQUET.value,
+    **kwargs,
+) -> None:
     """
     Writes the given Distributed Dataset to one or more files.
     """
@@ -102,11 +101,12 @@ def dataset_to_file(
         raise NotImplementedError(
             f"Distributed Dataset writer for content type '{content_type}' not"
             f" implemented. Known content types: "
-            f"{CONTENT_TYPE_TO_DATASET_WRITE_FUNC.keys}")
+            f"{CONTENT_TYPE_TO_DATASET_WRITE_FUNC.keys}"
+        )
     writer(
         table,
         base_path,
         filesystem=file_system,
         block_path_provider=block_path_provider,
-        **kwargs
+        **kwargs,
     )

deltacat/utils/ray_utils/performance.py

@@ -1,20 +1,16 @@
 import time
-from deltacat.utils.ray_utils.collections import DistributedCounter
 from typing import Any, Callable, Tuple
 
+from deltacat.utils.ray_utils.collections import DistributedCounter
+
 
 def invoke_with_perf_counter(
-        counter: DistributedCounter,
-        counter_key: Any,
-        func: Callable,
-        *args,
-        **kwargs) -> Tuple[Any, float]:
+    counter: DistributedCounter, counter_key: Any, func: Callable, *args, **kwargs
+) -> Tuple[Any, float]:
 
     start = time.perf_counter()
     result = func(*args, **kwargs)
     stop = time.perf_counter()
     latency = stop - start
-    counter.increment.remote(
-        counter_key,
-        latency)
+    counter.increment.remote(counter_key, latency)
     return result, latency

deltacat/utils/ray_utils/runtime.py

@@ -1,17 +1,17 @@
-import ray
 import logging
 import time
+from typing import Any, Callable, Dict, List
 
-from deltacat import logs
+import ray
 
-from typing import Any, Callable, Dict, List
+from deltacat import logs
 
 logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
 
 
 def node_resource_keys(
-        filter_fn: Callable[[Dict[str, Any]], bool] = lambda n: True) \
-        -> List[str]:
+    filter_fn: Callable[[Dict[str, Any]], bool] = lambda n: True
+) -> List[str]:
     """Get all Ray resource keys for cluster nodes that pass the given filter
     as a list of strings of the form: "node:{node_resource_name}". The returned
     keys can be used to place tasks or actors on that node via:
@@ -39,8 +39,9 @@ def current_node_resource_key() -> str:
     """
     current_node_id = ray.get_runtime_context().node_id.hex()
     keys = node_resource_keys(lambda n: n["NodeID"] == current_node_id)
-    assert len(keys) <= 1, \
-        f"Expected <= 1 keys for the current node, but found {len(keys)}"
+    assert (
+        len(keys) <= 1
+    ), f"Expected <= 1 keys for the current node, but found {len(keys)}"
     return keys[0] if len(keys) == 1 else None
 
 
@@ -55,9 +56,7 @@ def live_node_count() -> int:
     return sum(1 for n in ray.nodes() if is_node_alive(n))
 
 
-def live_node_waiter(
-        min_live_nodes: int,
-        poll_interval_seconds: float = 0.5) -> None:
+def live_node_waiter(min_live_nodes: int, poll_interval_seconds: float = 0.5) -> None:
     """Waits until the given minimum number of live nodes are present in the
     cluster. Checks the current number of live nodes every
     `poll_interval_seconds`."""

{deltacat-0.1.8.dist-info → deltacat-0.1.11.dist-info}/METADATA

@@ -1,9 +1,11 @@
 Metadata-Version: 2.1
 Name: deltacat
-Version: 0.1.8
+Version: 0.1.11
 Summary: A scalable, fast, ACID-compliant Data Catalog powered by Ray.
 Home-page: https://github.com/ray-project/deltacat
 Author: Ray Team
+License: UNKNOWN
+Platform: UNKNOWN
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3 :: Only
@@ -13,25 +15,33 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Operating System :: OS Independent
 Requires-Python: >=3.7
 Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: s3fs (==2022.1.0)
-Requires-Dist: tenacity (==8.0.1)
-Requires-Dist: ray[default] (==2.0.0)
-Requires-Dist: pandas (>=1.3.1)
-Requires-Dist: pyarrow (==8.0.0)
-Requires-Dist: pydantic (==1.10.2)
-Requires-Dist: numpy (>=1.21.1)
 Requires-Dist: boto3 (==1.20.24)
+Requires-Dist: numpy (==1.21.5)
+Requires-Dist: pandas (==1.3.5)
+Requires-Dist: pyarrow (==10.0.1)
+Requires-Dist: pydantic (==1.10.4)
+Requires-Dist: ray[default] (==2.0.0)
+Requires-Dist: s3fs (==2022.2.0)
+Requires-Dist: tenacity (==8.1.0)
 Requires-Dist: typing-extensions (==4.4.0)
 
 # DeltaCAT
 
 DeltaCAT is a Pythonic Data Catalog powered by Ray.
 
-Its data storage model allows you to define and manage fast, scalable, 
-ACID-compliant data catalogs through git-like stage/commit APIs, and has been 
+Its data storage model allows you to define and manage fast, scalable,
+ACID-compliant data catalogs through git-like stage/commit APIs, and has been
 used to successfully host exabyte-scale enterprise data lakes.
 
 DeltaCAT uses the Ray distributed compute framework together with Apache Arrow
-for common table management tasks, including petabyte-scale 
+for common table management tasks, including petabyte-scale
 change-data-capture, data consistency checks, and table repair.
+
+## Getting Started
+---
+### Install
+```
+pip install deltacat
+```
+
+

deltacat-0.1.11.dist-info/RECORD

@@ -0,0 +1,110 @@
+deltacat/__init__.py,sha256=gF2hBR7_JIL4gQRa1JN-fDraLLLmwXisIOcZLpbwTCM,1808
+deltacat/constants.py,sha256=E_1hOQolyvJCWB8eIVWtlAMgk2dmXGyXBhW05czilwQ,1173
+deltacat/exceptions.py,sha256=x7qem7FLujXf-DzPsNcQ-XYkW3cF3A0YGIbxkcpz0Mw,146
+deltacat/logs.py,sha256=T_-_JwOZFRSV64-KvmhMf-dInvYvuO4CSN-1EDrJJsU,5808
+deltacat/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/aws/clients.py,sha256=gBwSjCUfzQyiq-empApD685S-FhVlTjdJtDIWpR_alg,1743
+deltacat/aws/constants.py,sha256=4OnwC1H12FPs2bpVN7tXkxn-DAEJS4MYRrFh5HKFv7s,204
+deltacat/aws/s3u.py,sha256=KTAG9uNCYpANG-rWNAByhF062bpEnCqFP-_Crp1y6dA,17371
+deltacat/aws/redshift/__init__.py,sha256=fjuv3jWdPE8IgF4uSrL0YEqV3XUfqDULX3xV27ICceo,266
+deltacat/aws/redshift/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/aws/redshift/model/manifest.py,sha256=E71avhRHQGEZ6It3-axCB5FdiieQhSMu9Wt8oZAdXro,9519
+deltacat/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/catalog/delegate.py,sha256=yuMh8vcXwYBcaMO9HYasbj4DHJIl6Y5xZ5Qd2kTT278,8755
+deltacat/catalog/interface.py,sha256=A3Mr5tOBEG4VgDJuzrt5XEwrbNxYZPWHq34TAJvnX5M,6566
+deltacat/catalog/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/catalog/model/catalog.py,sha256=-Ho7a3rV1hiOS9cSRCAor9AtXV9nJn9t_MDVql9pIxo,2212
+deltacat/catalog/model/table_definition.py,sha256=tKrM1mmaQlvxqXrLt3QJVZK5BZfaJnhjTZ6KjybYlhE,727
+deltacat/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/compactor/__init__.py,sha256=5wjMMS0-22weCtKZe76dQwT6YGFCYgLtvhsb2gd5a8M,1078
+deltacat/compute/compactor/compaction_session.py,sha256=JJHZZmsQ4bOXYnmazomlnd64kcDCEdsfN7TMXRIqYvs,21182
+deltacat/compute/compactor/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/compactor/model/delta_annotated.py,sha256=BFnyeoeLkSwCDsfzipENgRBU8x8GiudSqZj946H8agY,7703
+deltacat/compute/compactor/model/delta_file_envelope.py,sha256=SfmdEPbfCfUekV_NvlWcyQOYo1CEHcRCGXHWHqmYQYg,1835
+deltacat/compute/compactor/model/delta_file_locator.py,sha256=Cc-YzxxyrXK6FlY8ek2L92XzfT0qkMCxs6yrC_FsEwU,1766
+deltacat/compute/compactor/model/materialize_result.py,sha256=b1Pwa89fgvr7rX3uSWwIt2ld-ElmqOSu-BXkZ1wwXdA,1253
+deltacat/compute/compactor/model/primary_key_index.py,sha256=MT4zqwhzh3e9qZotWvZavT_MtWXm_81ojfcOCv1t17w,10459
+deltacat/compute/compactor/model/pyarrow_write_result.py,sha256=WYIa0DRcyaemR6yUS8_8RLQ2voTmCVNFUL99qxPmt70,1324
+deltacat/compute/compactor/model/round_completion_info.py,sha256=3s0rAjJoV_IZ9OBe6KxopOijte2cS4khS2Nuw-Q2NQ8,3041
+deltacat/compute/compactor/model/sort_key.py,sha256=XDIoYrV18FciomV5yWxu1OaDsD78trmUUtseyRurIKo,4124
+deltacat/compute/compactor/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/compactor/steps/dedupe.py,sha256=cSmgeTYm0lQvteNZhvRFJjX0zeBJd2-gvnY9smsVmzk,11044
+deltacat/compute/compactor/steps/hash_bucket.py,sha256=XIPudRw4a9l7cCjfIQhYz_szktaY4tD6pcHRNBIh-HM,5809
+deltacat/compute/compactor/steps/materialize.py,sha256=8xsKBhBxankFLas6Ay97KeYcwyFlCs82UieBTLneTTE,8786
+deltacat/compute/compactor/steps/rehash/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/compactor/steps/rehash/rehash_bucket.py,sha256=yh-sBuUI3hqw2vk_nK9o-KDrgSww4oSvAz2hBxTkv8s,1765
+deltacat/compute/compactor/steps/rehash/rewrite_index.py,sha256=-HVM08pk5ROHEgDP-FVty55-a_0dsGRiSnPlNJw7C6Q,1838
+deltacat/compute/compactor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/compactor/utils/io.py,sha256=QMyh-oh4EqpN-lnurxJAS50IvY5LNoWHT6o6KhEPDiw,9637
+deltacat/compute/compactor/utils/primary_key_index.py,sha256=taYw1AjGIFlD9c8OXyj9ps816a15B61aoV4I00EAUyo,12072
+deltacat/compute/compactor/utils/round_completion_file.py,sha256=F4HwEG3egg59w4eiSvwrYImcDWgk56oXOyprnuYZYdE,2078
+deltacat/compute/compactor/utils/system_columns.py,sha256=ge4cL0RVsZ-9vTyU0xErnB-ClVASF5_CxsOTnAXnpfc,7106
+deltacat/compute/metastats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/metastats/meta_stats.py,sha256=-Fb0yQAdUUgm2IShcWlPZto-qdivF-nK05sQqJu7K5s,18588
+deltacat/compute/metastats/stats.py,sha256=-aFFrh7b--PzvQWNJG5_PgdN7ZM1bmGMeha5khwxhNw,7285
+deltacat/compute/metastats/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/metastats/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/metastats/model/partition_stats_dict.py,sha256=FbfoOxmTZfjRT7iHwc_96gHmB_r6iUvVM9BoTldD5mY,1123
+deltacat/compute/metastats/model/stats_cluster_size_estimator.py,sha256=AfH2rsC1DdJ2R_CwOPgjGJ04h-yWROsMfTw83GdpGXM,2849
+deltacat/compute/metastats/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/metastats/utils/constants.py,sha256=bFUPKmR3FkjEnwpHuToQYZ9QcHqYpd4OMMSwVwnJcaA,869
+deltacat/compute/metastats/utils/io.py,sha256=F9bY0Wo-qeokBLn5eXN9zIV2duLTXO5aNUMbL3_Ae2U,8825
+deltacat/compute/metastats/utils/pyarrow_memory_estimation_function.py,sha256=-3utoiC9fP2UFiJ-u7KbESNiHCRVzh5NGtSld0xRXX0,1143
+deltacat/compute/metastats/utils/ray_utils.py,sha256=sEDzcA0K8DMbQ_i8axBCQiPRrySPM14piaTqzKqhkss,4516
+deltacat/compute/stats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/stats/basic.py,sha256=m_tDdtLbsyyky-UJ0UULBZDoAAjYr02O0sSvFCKyHGk,8837
+deltacat/compute/stats/types.py,sha256=cp0lT8nITTKbnkc03OysRjXfcfXzQml9a4wqCnR6kqs,215
+deltacat/compute/stats/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/stats/models/delta_column_stats.py,sha256=-wXjB2c0BC1RDheumjL_j5-DfRNql4WsK9GpMFQI1cg,3300
+deltacat/compute/stats/models/delta_stats.py,sha256=hBith8_hbF9TVr6HocLAt6RJ_kZZKO4zrGP8VOP05vA,8556
+deltacat/compute/stats/models/delta_stats_cache_result.py,sha256=mbJYxpZd5jaER_BWrCD2hROFy3p1nNdBrj66nUpc6io,1624
+deltacat/compute/stats/models/manifest_entry_stats.py,sha256=NCDAe2nPDEI4kOkuwNkRFgGPS-rqQaQqLuaLoKk20KQ,2419
+deltacat/compute/stats/models/stats_result.py,sha256=XQAlmzhUqRmg4jzEMUAOqcYn1HUOBTMryBH1CCVlet8,3820
+deltacat/compute/stats/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/compute/stats/utils/intervals.py,sha256=9ezOzIrBGU1fWBuAn1CorJ3uX5COU7vxrfA7kI1cB7I,3094
+deltacat/compute/stats/utils/io.py,sha256=ZXpntXqa41l5bxxAa2vcTW5mVpWeBIvd3QA9VWnX-aw,8573
+deltacat/compute/stats/utils/manifest_stats_file.py,sha256=PtqW5Zc5e09HcfiAgvoZHVMJ2gamGdwmynMXOJNJUaY,3693
+deltacat/io/__init__.py,sha256=5Al7BPSaQghEp1K3PfiKIJJ0HR6MUuaN7HTMyM_9lf4,154
+deltacat/io/dataset.py,sha256=8w9sPVDpGnjjGVDWB39YSKWxq4zRv9VEfDtj7PYwjqM,3755
+deltacat/io/read_api.py,sha256=BhkjL3xjY-fsa62AA9Yv20_88uTskn4_Bv2W6VmMXVA,7023
+deltacat/io/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/io/aws/redshift/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/io/aws/redshift/redshift_datasource.py,sha256=X183O4tgBqtaZOSFmMFvp-9mv8NX5kGvRvX0eoSX8rA,22599
+deltacat/storage/__init__.py,sha256=ElzZuG5zrX9nUIe7f0Sp21WDX7yBoclclq3TIL-doag,1371
+deltacat/storage/interface.py,sha256=pw8t0jCqPakw13wDpg_cW0eBGBpqG1GO0djg-ZNNW6Q,21133
+deltacat/storage/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/storage/model/delta.py,sha256=bmcG1rF6mwUdM3YHh6M9gLV6uqxbwHZVNS3WHkXFeDw,13734
+deltacat/storage/model/list_result.py,sha256=FgD6oYeKo0EPe8z7jC8T4pAFjBOuBwd4axxGrnYyBG4,2466
+deltacat/storage/model/locator.py,sha256=1S7szmDSx-R4Z3arFNILOvS4t7dF7_rJNV9fHyRc3G4,1296
+deltacat/storage/model/namespace.py,sha256=KI2umYWShXFTx1ykLwsQjuce078WYo_Hmavn3DDeBzE,2086
+deltacat/storage/model/partition.py,sha256=6Sknqi2GhtaSpkM--3oMjR9agRLHS4i7izFWM4iiGmY,11068
+deltacat/storage/model/stream.py,sha256=XZ-c4EQR89NWydEOEG5GCaT8zST10OmjLZBKHZPdrzA,7738
+deltacat/storage/model/table.py,sha256=IOu1ZOrdRkVDB-FOxYMRvnNf5TukIDfbdHWTqHYN_OY,4225
+deltacat/storage/model/table_version.py,sha256=j57er3zlN0_2kwVMpWZ3iouABO-Kl8_Txi0UWIZ0dtk,7034
+deltacat/storage/model/types.py,sha256=LQPe_CxcoW2N67Leu3fNbnSXhbl9ubDa8LVvBY0JUiY,1580
+deltacat/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/tests/stats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/tests/stats/test_intervals.py,sha256=S92DgkALQ1WmbLWcxtvS7RlVGvL-XoPJKUUbkdn9_CQ,1955
+deltacat/tests/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/tests/utils/test_record_batch_tables.py,sha256=yLExx5jZfi65uSjkdhOCGnP7Km6zWqKCzmULf1PEKA0,11322
+deltacat/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/types/media.py,sha256=py1BnfMqNpJlW1RKzHWwB0NmQ33oCk9qg1fz7alvi3E,2187
+deltacat/types/tables.py,sha256=yUzkzmUij8kssEYI_dfVDSLXf8HfMm_jpgWkPxDHAas,3893
+deltacat/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/utils/common.py,sha256=RG_-enXNpLKaYrqyx1ne2lL10lxN9vK7F631oJP6SE8,1375
+deltacat/utils/numpy.py,sha256=ZiGREobTVT6IZXgPxkSUpLJFN2Hn8KEZcrqybLDXCIA,2027
+deltacat/utils/pandas.py,sha256=eGOpiZE1zLznTtuwoN80j4PBp1_bUV8SE4c951r0a3o,9561
+deltacat/utils/performance.py,sha256=rC3CPfroZP3T5TbRNZXB9GRBr0F9i2KUeZYL45JBgCU,610
+deltacat/utils/placement.py,sha256=JE6OsW16VonlMhdH5B2IYuLJxItoYguaKpZNgbpMNLw,11066
+deltacat/utils/pyarrow.py,sha256=Xf7KKTlA6wPJXcd_Uopm6iTSM9IlZ0M6Ajr4tWJP8OU,18230
+deltacat/utils/ray_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+deltacat/utils/ray_utils/collections.py,sha256=hj20s4D2RF2jZETU_44r6mFbsczA0JI_I_4kWKTmqes,1951
+deltacat/utils/ray_utils/concurrency.py,sha256=AyL7hpvYjkmsz-KcpYjVgPpNsmu-x8-rlLyG0qXoV_c,5123
+deltacat/utils/ray_utils/dataset.py,sha256=SIljK3UkSqQ6Ntit_iSiYt9yYjN_gGrCTX6_72XdQ3w,3244
+deltacat/utils/ray_utils/performance.py,sha256=d7JFM7vTXHzkGx9qNQcZzUWajnqINvYRwaM088_FpsE,464
+deltacat/utils/ray_utils/runtime.py,sha256=pUCSt2Fo8iMrvjmbkQdFtgSxZW9PA05H1molItzr5Rc,4786
+deltacat-0.1.11.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+deltacat-0.1.11.dist-info/METADATA,sha256=Ce07iEuy13s5C0jOpFDUkcL8jhDWy8BvWTP4GHPWDdw,1493
+deltacat-0.1.11.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+deltacat-0.1.11.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
+deltacat-0.1.11.dist-info/RECORD,,

{deltacat-0.1.8.dist-info → deltacat-0.1.11.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.37.1)
+Generator: bdist_wheel (0.38.4)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

deltacat/autoscaler/events/compaction/cluster.py

@@ -1,82 +0,0 @@
-from typing import Dict, Any
-
-import math
-
-
-MB_IN_BYTES = 1_000_000
-
-
-class ClusterSizeSuggester:
-    def __init__(self,
-                 cluster_memory_bytes: float = None,
-                 heap_memory_alloc_ratio: float = 0.7,  # Ray defaults
-                 object_store_memory_alloc_ratio: float = 0.3,  # Ray defaults
-                 instance_type: str = "r5.8xlarge"):
-        """Given the total required memory constraints, constructs a helper class that
-        recommends an instance type, number of instance nodes, max usable heap and object store memory,
-
-        Args:
-            cluster_memory_bytes: Total memory needed for the cluster.
-                If not provided, defaults to memory size of instance type.
-            heap_memory_alloc_ratio: Optional. Set to 0.7 by default.
-            object_store_memory_alloc_ratio: Optional. Set to 0.3 by default.
-            instance_type: Optional. Set to r5.8xlarge by default, to allow for up to 8GB of memory per vCPU.
-                # TODO: suggest various r5 instance types based on memory input
-        """
-        self._instance_type = instance_type
-        self.cluster_memory_bytes = cluster_memory_bytes if cluster_memory_bytes else self.get_node_memory_size()
-        self.heap_memory_alloc_ratio = heap_memory_alloc_ratio
-        self.object_store_memory_alloc_ratio = object_store_memory_alloc_ratio
-
-    @property
-    def instance_type(self):
-        return self._instance_type
-
-    def get_instance_type_specifications(self) -> Dict[str, Any]:
-        """Assumes r5.8xlarge instances (for now)
-
-        Returns: a dict of hardware details
-
-        """
-        # TODO: call ec2 describe-instance-types to extract hardware details (vCPUs, memory, network bandwidth).
-        #  Current implementation assumes we only serve r5.8xlarge node types.
-        if self.instance_type == "r5.8xlarge":
-            return {
-                # Intentionally mimic the output format of describe-instance-types API
-                "VCpuInfo": {
-                    "DefaultVCpus": 32,
-                    "DefaultCores": 16,
-                    "DefaultThreadsPerCore": 2,
-                },
-                "MemoryInfo": {
-                    "SizeInMiB": 262144
-                }
-            }
-
-    def get_num_vcpu_per_node(self):
-        spec = self.get_instance_type_specifications()
-        return spec["VCpuInfo"]["DefaultVCpus"]
-
-    def get_node_memory_size(self):
-        spec = self.get_instance_type_specifications()
-        return spec["MemoryInfo"]["SizeInMiB"] * MB_IN_BYTES
-
-    def get_max_memory_per_vcpu(self):
-        return self.get_node_memory_size() / self.get_num_vcpu_per_node()
-
-    def get_node_max_object_store_memory(self):
-        return self.object_store_memory_alloc_ratio * self.get_node_memory_size()
-
-    def get_node_max_heap_memory(self):
-        return self.heap_memory_alloc_ratio * self.get_node_memory_size()
-
-    def get_suggested_vcpu_count(self):
-        return self.cluster_memory_bytes / self.get_max_memory_per_vcpu()
-
-    def get_suggested_node_size(self):
-        return math.ceil(self.cluster_memory_bytes / (self.get_num_vcpu_per_node() * self.get_max_memory_per_vcpu()))
-
-
-class InstanceTypeSuggester:
-    def __init__(self):
-        raise NotImplementedError("Instance Type Suggester is not implemented.")

deltacat/autoscaler/events/compaction/collections/partition_key_value.py

@@ -1,36 +0,0 @@
-from typing import List, Iterable
-
-
-class PartitionKeyValue(dict):
-    @staticmethod
-    def of(key_name: str,
-           key_type: str,
-           value: str):
-        pkv = PartitionKeyValue()
-        pkv["key"] = {}
-        pkv["key"]["keyName"] = key_name
-        pkv["key"]["keyType"] = key_type
-        pkv["value"] = value
-        return pkv
-
-    @property
-    def key(self):
-        return self["key"]
-
-    @property
-    def key_name(self):
-        return self["key"]["keyName"]
-
-    @property
-    def key_type(self):
-        return self["key"]["keyType"]
-
-    @property
-    def value(self):
-        return self["value"]
-
-
-class PartitionKeyValues(tuple):
-    @property
-    def id(self):
-        return ".".join([pkv.value for pkv in self if isinstance(pkv, PartitionKeyValue)])

deltacat/autoscaler/events/compaction/dispatcher.py

@@ -1,28 +0,0 @@
-import logging
-
-from deltacat.autoscaler.events.compaction.session_manager import CompactionSessionManager
-from deltacat.autoscaler.events.dispatcher import EventDispatcher
-from ray.autoscaler._private.aws.events import EventPublisher
-
-from deltacat import logs
-from deltacat.storage import interface as unimplemented_deltacat_storage
-
-logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
-
-
-class CompactionEventDispatcher(EventDispatcher):
-    def __init__(self,
-                 events_publisher: EventPublisher,
-                 deltacat_storage: unimplemented_deltacat_storage,
-                 session_manager: CompactionSessionManager = None):
-        """Constructor for the event dispatcher.
-
-        Intended for usage by Ray parent and child clusters running compaction jobs.
-
-        Args:
-            events_publisher: Events manager for publishing events through a cloud provider
-            deltacat_storage: Storage interface for deltacat
-            session_manager: Manager for launching child Ray sessions
-        """
-        super().__init__(events_publisher, deltacat_storage, session_manager)
-        self.session_manager = session_manager

deltacat/autoscaler/events/compaction/input.py

@@ -1,27 +0,0 @@
-from typing import Set, NamedTuple, List, Optional, Dict
-
-from deltacat import ContentType, SortKey
-from deltacat.autoscaler.events.compaction.collections.partition_key_value import PartitionKeyValues
-from deltacat.compute.stats.models.delta_stats import DeltaStats
-from deltacat.storage import PartitionLocator, interface as unimplemented_deltacat_storage
-
-import pyarrow as pa
-
-
-class CompactionInput(NamedTuple):
-    source_partition_locator: PartitionLocator
-    compacted_partition_locator: PartitionLocator
-    primary_keys: Set[str]
-    compaction_artifact_s3_bucket: str
-    last_stream_position_to_compact: int
-    hash_bucket_count: Optional[int] = None
-    sort_keys: List[SortKey] = None
-    records_per_primary_key_index_file: int = 38_000_000
-    records_per_compacted_file: int = 4_000_000
-    input_deltas_stats: Optional[Dict[int, DeltaStats]] = None
-    min_hash_bucket_chunk_size: int = 0
-    compacted_file_content_type: ContentType = ContentType.PARQUET
-    delete_prev_primary_key_index: bool = False
-    schema_on_read: Optional[pa.schema] = None
-    deltacat_storage = unimplemented_deltacat_storage
-    partition_key_values: PartitionKeyValues = None

deltacat/autoscaler/events/compaction/process.py

@@ -1,25 +0,0 @@
-from typing import Optional
-
-from deltacat.autoscaler.events.compaction.collections.partition_key_value import PartitionKeyValues
-from deltacat.storage import PartitionLocator
-
-
-class CompactionProcess:
-    def __init__(self,
-                 partition_locator: PartitionLocator,
-                 compaction_cluster_config_path: str,
-                 hash_bucket_count: Optional[int] = None,
-                 last_stream_position_to_compact: Optional[int] = None,
-                 partition_key_values: PartitionKeyValues = None,
-                 cluster_memory_bytes: Optional[int] = None,
-                 input_delta_total_bytes: Optional[int] = None):
-        self.partition_locator = partition_locator
-        self.compaction_cluster_config_path = compaction_cluster_config_path
-        self.hash_bucket_count = hash_bucket_count
-        self.last_stream_position_to_compact = last_stream_position_to_compact
-        self.partition_values = partition_key_values
-        self.cluster_memory_bytes = cluster_memory_bytes
-        self.input_delta_total_bytes = input_delta_total_bytes
-        self.id = ".".join([pkv.value for pkv in partition_key_values])
-
-

deltacat/autoscaler/events/compaction/session_manager.py

@@ -1,13 +0,0 @@
-from typing import List
-
-from deltacat.autoscaler.events.compaction.process import CompactionProcess
-from deltacat.autoscaler.events.session_manager import SessionManager
-from deltacat.storage import PartitionLocator
-
-
-class CompactionSessionManager(SessionManager):
-    def launch_stats_metadata_collection(self, source_partition_locators: List[PartitionLocator]):
-        raise NotImplementedError("Stats Metadata Collection is not implemented.")
-
-    def launch_compaction(self, compaction_processes: List[CompactionProcess]):
-        raise NotImplementedError("Compaction is not implemented.")