deltacat 0.1.8__py3-none-any.whl → 0.1.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deltacat/__init__.py +41 -15
- deltacat/aws/clients.py +12 -31
- deltacat/aws/constants.py +1 -1
- deltacat/aws/redshift/__init__.py +7 -2
- deltacat/aws/redshift/model/manifest.py +54 -50
- deltacat/aws/s3u.py +188 -218
- deltacat/catalog/delegate.py +151 -185
- deltacat/catalog/interface.py +78 -97
- deltacat/catalog/model/catalog.py +21 -21
- deltacat/catalog/model/table_definition.py +11 -9
- deltacat/compute/compactor/__init__.py +12 -16
- deltacat/compute/compactor/compaction_session.py +259 -316
- deltacat/compute/compactor/model/delta_annotated.py +60 -44
- deltacat/compute/compactor/model/delta_file_envelope.py +5 -6
- deltacat/compute/compactor/model/delta_file_locator.py +10 -8
- deltacat/compute/compactor/model/materialize_result.py +6 -7
- deltacat/compute/compactor/model/primary_key_index.py +38 -34
- deltacat/compute/compactor/model/pyarrow_write_result.py +3 -4
- deltacat/compute/compactor/model/round_completion_info.py +25 -19
- deltacat/compute/compactor/model/sort_key.py +18 -15
- deltacat/compute/compactor/steps/dedupe.py +152 -259
- deltacat/compute/compactor/steps/hash_bucket.py +57 -73
- deltacat/compute/compactor/steps/materialize.py +138 -99
- deltacat/compute/compactor/steps/rehash/rehash_bucket.py +13 -13
- deltacat/compute/compactor/steps/rehash/rewrite_index.py +11 -13
- deltacat/compute/compactor/utils/io.py +59 -47
- deltacat/compute/compactor/utils/primary_key_index.py +131 -90
- deltacat/compute/compactor/utils/round_completion_file.py +22 -23
- deltacat/compute/compactor/utils/system_columns.py +33 -42
- deltacat/compute/metastats/meta_stats.py +235 -157
- deltacat/compute/metastats/model/partition_stats_dict.py +7 -10
- deltacat/compute/metastats/model/stats_cluster_size_estimator.py +13 -5
- deltacat/compute/metastats/stats.py +95 -64
- deltacat/compute/metastats/utils/io.py +100 -53
- deltacat/compute/metastats/utils/pyarrow_memory_estimation_function.py +5 -2
- deltacat/compute/metastats/utils/ray_utils.py +38 -33
- deltacat/compute/stats/basic.py +107 -69
- deltacat/compute/stats/models/delta_column_stats.py +11 -8
- deltacat/compute/stats/models/delta_stats.py +59 -32
- deltacat/compute/stats/models/delta_stats_cache_result.py +4 -1
- deltacat/compute/stats/models/manifest_entry_stats.py +12 -6
- deltacat/compute/stats/models/stats_result.py +24 -14
- deltacat/compute/stats/utils/intervals.py +16 -9
- deltacat/compute/stats/utils/io.py +86 -51
- deltacat/compute/stats/utils/manifest_stats_file.py +24 -33
- deltacat/constants.py +8 -10
- deltacat/io/__init__.py +2 -2
- deltacat/io/aws/redshift/redshift_datasource.py +157 -143
- deltacat/io/dataset.py +14 -17
- deltacat/io/read_api.py +36 -33
- deltacat/logs.py +94 -42
- deltacat/storage/__init__.py +18 -8
- deltacat/storage/interface.py +196 -213
- deltacat/storage/model/delta.py +45 -51
- deltacat/storage/model/list_result.py +12 -8
- deltacat/storage/model/namespace.py +4 -5
- deltacat/storage/model/partition.py +42 -42
- deltacat/storage/model/stream.py +29 -30
- deltacat/storage/model/table.py +14 -14
- deltacat/storage/model/table_version.py +32 -31
- deltacat/storage/model/types.py +1 -0
- deltacat/tests/stats/test_intervals.py +11 -24
- deltacat/tests/utils/test_record_batch_tables.py +284 -0
- deltacat/types/media.py +3 -4
- deltacat/types/tables.py +31 -21
- deltacat/utils/common.py +5 -11
- deltacat/utils/numpy.py +20 -22
- deltacat/utils/pandas.py +73 -100
- deltacat/utils/performance.py +3 -9
- deltacat/utils/placement.py +276 -231
- deltacat/utils/pyarrow.py +302 -89
- deltacat/utils/ray_utils/collections.py +2 -1
- deltacat/utils/ray_utils/concurrency.py +38 -32
- deltacat/utils/ray_utils/dataset.py +28 -28
- deltacat/utils/ray_utils/performance.py +5 -9
- deltacat/utils/ray_utils/runtime.py +9 -10
- {deltacat-0.1.8.dist-info → deltacat-0.1.11.dist-info}/METADATA +22 -12
- deltacat-0.1.11.dist-info/RECORD +110 -0
- {deltacat-0.1.8.dist-info → deltacat-0.1.11.dist-info}/WHEEL +1 -1
- deltacat/autoscaler/events/__init__.py +0 -0
- deltacat/autoscaler/events/compaction/__init__.py +0 -0
- deltacat/autoscaler/events/compaction/cluster.py +0 -82
- deltacat/autoscaler/events/compaction/collections/__init__.py +0 -0
- deltacat/autoscaler/events/compaction/collections/partition_key_value.py +0 -36
- deltacat/autoscaler/events/compaction/dispatcher.py +0 -28
- deltacat/autoscaler/events/compaction/input.py +0 -27
- deltacat/autoscaler/events/compaction/process.py +0 -25
- deltacat/autoscaler/events/compaction/session_manager.py +0 -13
- deltacat/autoscaler/events/compaction/utils.py +0 -216
- deltacat/autoscaler/events/compaction/workflow.py +0 -303
- deltacat/autoscaler/events/dispatcher.py +0 -95
- deltacat/autoscaler/events/dynamodb/__init__.py +0 -0
- deltacat/autoscaler/events/dynamodb/event_store.py +0 -164
- deltacat/autoscaler/events/event_store.py +0 -55
- deltacat/autoscaler/events/exceptions.py +0 -6
- deltacat/autoscaler/events/processor.py +0 -177
- deltacat/autoscaler/events/session_manager.py +0 -25
- deltacat/autoscaler/events/states.py +0 -88
- deltacat/autoscaler/events/workflow.py +0 -54
- deltacat/autoscaler/node_group.py +0 -230
- deltacat/autoscaler/utils.py +0 -69
- deltacat-0.1.8.dist-info/RECORD +0 -131
- /deltacat/{autoscaler → tests/utils}/__init__.py +0 -0
- {deltacat-0.1.8.dist-info → deltacat-0.1.11.dist-info}/LICENSE +0 -0
- {deltacat-0.1.8.dist-info → deltacat-0.1.11.dist-info}/top_level.txt +0 -0
@@ -1,230 +0,0 @@
|
|
1
|
-
import ray
|
2
|
-
import re
|
3
|
-
import time
|
4
|
-
import yaml
|
5
|
-
import logging
|
6
|
-
from typing import Optional, Union, List, Dict, Any, Callable, Tuple
|
7
|
-
from ray.util.placement_group import (
|
8
|
-
placement_group,
|
9
|
-
placement_group_table,
|
10
|
-
remove_placement_group,
|
11
|
-
get_current_placement_group
|
12
|
-
)
|
13
|
-
|
14
|
-
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
|
15
|
-
from ray.experimental.state.api import list_placement_groups, get_node, get_placement_group
|
16
|
-
|
17
|
-
|
18
|
-
from deltacat import logs
|
19
|
-
logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
|
20
|
-
|
21
|
-
class NodeGroupManager():
|
22
|
-
|
23
|
-
def __init__(self,path: str, gname: str):
|
24
|
-
"""Node Group Manager
|
25
|
-
Args:
|
26
|
-
path: cluster yaml file
|
27
|
-
gname: node group prefix, e.g., 'partition'
|
28
|
-
"""
|
29
|
-
#cluster init status:
|
30
|
-
self.NODE_GROUP_PREFIX=gname
|
31
|
-
self.cluster_config=self._read_yaml(path)
|
32
|
-
self.init_groups = self._cluster_node_groups(self.cluster_config)
|
33
|
-
self.init_group_res = self._parse_node_resources()
|
34
|
-
|
35
|
-
def _cluster_node_groups(self, config: Dict[str, Any]) -> Dict[str, Any]:
|
36
|
-
"""Get Worker Groups
|
37
|
-
Args:
|
38
|
-
config: cluster yaml data
|
39
|
-
Returns:
|
40
|
-
worker groups: a dict of worker node group
|
41
|
-
|
42
|
-
"""
|
43
|
-
avail_node_types = list(config['available_node_types'].items())
|
44
|
-
#exclude head node type
|
45
|
-
head_node_types = [nt for nt in avail_node_types if 'resources' in nt[1] and 'CPU' in nt[1]['resources'] and nt[1]['resources']['CPU']==0][0]
|
46
|
-
worker_node_types = [x for x in avail_node_types if x !=head_node_types]
|
47
|
-
#assuming homogenous cluster
|
48
|
-
#in future, update with fleet resource
|
49
|
-
if len(worker_node_types)>0:
|
50
|
-
self.INSTANCE_TYPE = worker_node_types[0][1]['node_config']['InstanceType']
|
51
|
-
return worker_node_types
|
52
|
-
|
53
|
-
|
54
|
-
def _read_yaml(self, path: str) -> Dict[str, Any]:
|
55
|
-
with open(path, "rt") as f:
|
56
|
-
return yaml.safe_load(f)
|
57
|
-
|
58
|
-
def _update_groups(self) -> List[Tuple[str, float]]:
|
59
|
-
"""
|
60
|
-
Node groups can come and go during runtime, whenever a node group is needed, we need to check the current available groups
|
61
|
-
Returns:
|
62
|
-
current_groups: dict of custom resource groups
|
63
|
-
"""
|
64
|
-
#Add 1.1 second latency to avoid inconsistency issue between raylet and head
|
65
|
-
time.sleep(1.1)
|
66
|
-
all_available_res = ray.available_resources()
|
67
|
-
current_groups =[(k,all_available_res[k]) for k in all_available_res.keys() if self.NODE_GROUP_PREFIX in k]
|
68
|
-
return current_groups
|
69
|
-
|
70
|
-
def _parse_node_resources(self) -> Dict[str, Dict[str, float]]:
|
71
|
-
"""
|
72
|
-
Parse resources per node to get detailed resource tighted to each node group
|
73
|
-
Returns:
|
74
|
-
group_res: a dict of resources, e.g., {'CPU':0,'memory':0,'object_store_memory':0}
|
75
|
-
"""
|
76
|
-
all_available_resources= ray._private.state.state._available_resources_per_node()
|
77
|
-
group_keys = [x[0] for x in self.init_groups]
|
78
|
-
group_res={}
|
79
|
-
for k in group_keys:
|
80
|
-
group_res[k]={'CPU':0,'memory':0,'object_store_memory':0,'node_id':[]}
|
81
|
-
for v in all_available_resources.values():
|
82
|
-
keys =v.keys()
|
83
|
-
r = re.compile(self.NODE_GROUP_PREFIX)
|
84
|
-
partition=list(filter(r.match, list(keys)))
|
85
|
-
r = re.compile("node:")
|
86
|
-
node_id = list(filter(r.match, list(keys)))
|
87
|
-
if len(partition)>0:
|
88
|
-
partition = partition[0]
|
89
|
-
if len(node_id)>0:
|
90
|
-
node_id = node_id[0]
|
91
|
-
if self.NODE_GROUP_PREFIX in partition:
|
92
|
-
group_res[partition]['CPU']+=v['CPU']
|
93
|
-
group_res[partition]['memory']+=v['memory']
|
94
|
-
group_res[partition]['object_store_memory']+=v['object_store_memory']
|
95
|
-
group_res[partition]['node_id'].append(node_id)
|
96
|
-
return group_res
|
97
|
-
|
98
|
-
def _update_group_res(self, gname: str) -> Dict[str, Union[str, float]]:
|
99
|
-
"""
|
100
|
-
Get the realtime resource of a node group
|
101
|
-
Args:
|
102
|
-
gname: name of node group
|
103
|
-
Returns:
|
104
|
-
group_res: dict of updated resource(cpu, memory, object store memory) for a given group
|
105
|
-
"""
|
106
|
-
all_available_resources= ray._private.state.state._available_resources_per_node()
|
107
|
-
group_res={'CPU':0,'memory':0,'object_store_memory':0,'node_id':[]}
|
108
|
-
for v in all_available_resources.values():
|
109
|
-
keys =v.keys()
|
110
|
-
r = re.compile("node:")
|
111
|
-
node_id = list(filter(r.match, list(keys)))
|
112
|
-
if len(node_id)>0:
|
113
|
-
node_id = node_id[0]
|
114
|
-
if gname in v.keys():
|
115
|
-
group_res['CPU']+=v['CPU']
|
116
|
-
group_res['memory']+=v['memory']
|
117
|
-
group_res['object_store_memory']+=v['object_store_memory']
|
118
|
-
group_res['node_id'].append(node_id)
|
119
|
-
return group_res
|
120
|
-
|
121
|
-
def get_one_group(self) -> Optional[Dict[str, Union[str, float]]]:
|
122
|
-
"""
|
123
|
-
Pop up one node group
|
124
|
-
Returns:
|
125
|
-
group_res: dict of node group resource, {"group":"partition_1","CPU":2,...}
|
126
|
-
"""
|
127
|
-
current_groups = self._update_groups()
|
128
|
-
if len(current_groups)>0:
|
129
|
-
gname = current_groups[-1][0]
|
130
|
-
group_res=self._update_group_res(gname)
|
131
|
-
group_res['group']=gname
|
132
|
-
try:
|
133
|
-
group_res['group_res']=ray.available_resources()[gname]
|
134
|
-
except Exception as e:
|
135
|
-
print("There is no available resources for %s"%gname)
|
136
|
-
return None
|
137
|
-
return group_res
|
138
|
-
else:
|
139
|
-
return None
|
140
|
-
|
141
|
-
def get_group_by_name(self, gname: str) -> Optional[Dict[str, Union[str, float]]]:
|
142
|
-
"""
|
143
|
-
Get the specific node group given its pre-filled name
|
144
|
-
Args:
|
145
|
-
gname: name of the node group
|
146
|
-
Returns:
|
147
|
-
group_res: dict of node group resource
|
148
|
-
|
149
|
-
"""
|
150
|
-
group_res=self._update_group_res(gname)
|
151
|
-
group_res['group']=gname
|
152
|
-
try:
|
153
|
-
group_res['group_res']=ray.available_resources()[gname]
|
154
|
-
except Exception as e:
|
155
|
-
print("There is no available resources for %s"%gname)
|
156
|
-
return None
|
157
|
-
return group_res
|
158
|
-
|
159
|
-
|
160
|
-
class PlacementGroupManager():
|
161
|
-
"""Placement Group Manager
|
162
|
-
Create a list of placement group with the desired number of cpus
|
163
|
-
e.g., create a pg with 32 cpus, then this class will look for a node that has 32 cpus, and collect all
|
164
|
-
resources, including cpu, memory, and object store;
|
165
|
-
How to use:
|
166
|
-
```
|
167
|
-
from deltacat.autoscaler.node_group import PlacementGroupManager as pgm
|
168
|
-
pgm = pgm(10, 32)
|
169
|
-
pg_configs = pgm.pgs
|
170
|
-
opts = pg_configs[0][0]
|
171
|
-
fun.options(**opts).remote()
|
172
|
-
```
|
173
|
-
Args:
|
174
|
-
num_pgs: number of placement groups to be created
|
175
|
-
instance_cpus: number of cpus per instance
|
176
|
-
"""
|
177
|
-
def __init__(self, num_pgs: int, instance_cpus: int, time_out: Optional[float] = None):
|
178
|
-
#self._pg_configs =self._config(num_pgs, instance_cpus, time_out)
|
179
|
-
self._pg_configs = ray.get([_config.remote(instance_cpus) for _ in range(num_pgs)])
|
180
|
-
@property
|
181
|
-
def pgs(self):
|
182
|
-
return self._pg_configs
|
183
|
-
|
184
|
-
@ray.remote
|
185
|
-
def _config(instance_cpus: int, time_out: Optional[float] = None) -> Tuple[Dict[str,Any], Dict[str,Any]]:
|
186
|
-
pg_config = None
|
187
|
-
try:
|
188
|
-
opts ={}
|
189
|
-
cluster_resources={}
|
190
|
-
bundle1={'CPU': instance_cpus} # hardcode for now
|
191
|
-
pg = placement_group([bundle1], strategy="PACK")
|
192
|
-
ray.get(pg.ready(), timeout=time_out)
|
193
|
-
if not pg:
|
194
|
-
return
|
195
|
-
opts = {"scheduling_strategy":PlacementGroupSchedulingStrategy(
|
196
|
-
placement_group=pg, placement_group_capture_child_tasks=True)
|
197
|
-
}
|
198
|
-
pg_id = placement_group_table(pg)['placement_group_id']
|
199
|
-
#print("pg id{}".format(pg_id))
|
200
|
-
pg_details = get_placement_group(pg_id)
|
201
|
-
#print("pgs-{}".format(pgs))
|
202
|
-
bundles = pg_details['bundles']
|
203
|
-
node_ids =[]
|
204
|
-
for bd in bundles:
|
205
|
-
node_ids.append(bd['node_id'])
|
206
|
-
#query available resources given list of node id
|
207
|
-
all_nodes_available_res = ray._private.state.state._available_resources_per_node()
|
208
|
-
pg_res = {'CPU':0,'memory':0,'object_store_memory':0,'node_id':[]}
|
209
|
-
for node_id in node_ids:
|
210
|
-
if node_id in all_nodes_available_res:
|
211
|
-
v = all_nodes_available_res[node_id]
|
212
|
-
node_detail = get_node(node_id)
|
213
|
-
pg_res['CPU']+=node_detail['resources_total']['CPU']
|
214
|
-
pg_res['memory']+=v['memory']
|
215
|
-
pg_res['object_store_memory']+=v['object_store_memory']
|
216
|
-
#pg_res['node_id'].append(node_id)
|
217
|
-
cluster_resources['CPU'] = int(pg_res['CPU'])
|
218
|
-
cluster_resources['memory'] = float(pg_res['memory'])
|
219
|
-
cluster_resources['object_store_memory'] = float(pg_res['object_store_memory'])
|
220
|
-
#cluster_resources['node_id'] = pg_res['node_id']
|
221
|
-
#cluster_cpus = cluster_resources['CPU']
|
222
|
-
pg_config=[opts,cluster_resources]
|
223
|
-
print("pg has resources:{}".format(cluster_resources))
|
224
|
-
|
225
|
-
except Exception as e:
|
226
|
-
print("Error in creating pg:{}".format(e))
|
227
|
-
logger.error(f"placement group error{e}")
|
228
|
-
pass
|
229
|
-
return pg_config
|
230
|
-
|
deltacat/autoscaler/utils.py
DELETED
@@ -1,69 +0,0 @@
|
|
1
|
-
import os
|
2
|
-
import subprocess
|
3
|
-
from typing import Any, Optional, List
|
4
|
-
|
5
|
-
import ray
|
6
|
-
|
7
|
-
|
8
|
-
def run_cmd(cmd: str,
|
9
|
-
background_process: bool = False,
|
10
|
-
stdout: Optional[Any] = subprocess.DEVNULL,
|
11
|
-
stderr: Optional[Any] = subprocess.STDOUT) -> None:
|
12
|
-
cmd_list = cmd.split(" ")
|
13
|
-
if background_process:
|
14
|
-
cmd_list = ["nohup"] + cmd_list
|
15
|
-
subprocess.Popen(cmd_list,
|
16
|
-
stdout=stdout,
|
17
|
-
stderr=stderr)
|
18
|
-
return
|
19
|
-
else:
|
20
|
-
exit_code = subprocess.call(cmd_list, stdout=stdout, stderr=stderr)
|
21
|
-
assert exit_code == 0, f"`{cmd}` failed. Exit code: {exit_code}"
|
22
|
-
|
23
|
-
|
24
|
-
def ray_submit(cluster_cfg: str,
|
25
|
-
path_to_script: str,
|
26
|
-
script_arguments: Optional[List[str]] = None,
|
27
|
-
background_process: bool = False,
|
28
|
-
start_cluster: bool = True,
|
29
|
-
stop_cluster: bool = True,
|
30
|
-
stdout: Optional[Any] = subprocess.DEVNULL,
|
31
|
-
stderr: Optional[Any] = subprocess.STDOUT) -> None:
|
32
|
-
cluster_args = []
|
33
|
-
if start_cluster:
|
34
|
-
cluster_args.append("--start")
|
35
|
-
if stop_cluster:
|
36
|
-
cluster_args.append("--stop")
|
37
|
-
|
38
|
-
if script_arguments is None:
|
39
|
-
script_arguments = []
|
40
|
-
|
41
|
-
script_arguments_str = " ".join(script_arguments)
|
42
|
-
|
43
|
-
cmd = f"ray submit --log-style record --no-config-cache {' '.join(cluster_args)} {cluster_cfg} {path_to_script}"
|
44
|
-
if script_arguments:
|
45
|
-
cmd = f"{cmd} -- {script_arguments_str}"
|
46
|
-
|
47
|
-
run_cmd(cmd, background_process=background_process, stdout=stdout, stderr=stderr)
|
48
|
-
print(f"Submitted script on Ray cluster '{cluster_cfg}' with command '{cmd}'")
|
49
|
-
|
50
|
-
|
51
|
-
def ray_up(cluster_cfg: str) -> None:
|
52
|
-
print(f"Starting Ray cluster '{cluster_cfg}'")
|
53
|
-
run_cmd(f"ray up {cluster_cfg} -y --no-config-cache --no-restart")
|
54
|
-
print(f"Started Ray cluster '{cluster_cfg}'")
|
55
|
-
|
56
|
-
|
57
|
-
def get_head_node_ip(cluster_cfg: str) -> str:
|
58
|
-
print(f"Getting Ray cluster head node IP for '{cluster_cfg}'")
|
59
|
-
proc = subprocess.run(
|
60
|
-
f"ray get-head-ip {cluster_cfg}",
|
61
|
-
shell=True,
|
62
|
-
capture_output=True,
|
63
|
-
text=True,
|
64
|
-
check=True)
|
65
|
-
# the head node IP should be the last line printed to stdout
|
66
|
-
head_node_ip = proc.stdout.splitlines()[-1]
|
67
|
-
print(f"Ray cluster head node IP for '{cluster_cfg}': {head_node_ip}")
|
68
|
-
return head_node_ip
|
69
|
-
|
deltacat-0.1.8.dist-info/RECORD
DELETED
@@ -1,131 +0,0 @@
|
|
1
|
-
deltacat/__init__.py,sha256=PykA9TJ4ofPC4lwnnzyAgabk_POjYX8QZ-b1TOwe8IY,1675
|
2
|
-
deltacat/constants.py,sha256=ZGSRdqQQEIuQGO_Qwkl3tQxdZGVl4oLH01tCdXvyURo,1092
|
3
|
-
deltacat/exceptions.py,sha256=x7qem7FLujXf-DzPsNcQ-XYkW3cF3A0YGIbxkcpz0Mw,146
|
4
|
-
deltacat/logs.py,sha256=ZvMw7mhQZTNZH1Czwd_rW1OAnnFiQpogN4441UjWVQo,3879
|
5
|
-
deltacat/autoscaler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
6
|
-
deltacat/autoscaler/node_group.py,sha256=On7UffaEsl_iSFTY5biAakECRYCou8r6P661WzgeUqU,8026
|
7
|
-
deltacat/autoscaler/utils.py,sha256=76bcUt2tEu6fxUpKBQwyWr_Ea2Xnwvjff2OxvLhQ0sc,2368
|
8
|
-
deltacat/autoscaler/events/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
9
|
-
deltacat/autoscaler/events/dispatcher.py,sha256=T2oa9wmm3bcCmmoBmj8fjZojR9m0aTe2OVp8uKE5ZMQ,3792
|
10
|
-
deltacat/autoscaler/events/event_store.py,sha256=26uggtK8DbGv4tflW80z32KKwGzvfNYoNeYBfPqGgcw,1764
|
11
|
-
deltacat/autoscaler/events/exceptions.py,sha256=1vdhgrCaXd10-NNM5Etem-q-041ZJBzom0IXKSfJpQk,97
|
12
|
-
deltacat/autoscaler/events/processor.py,sha256=vW9K0paNBXaNQyJbRxjTzc4r4GbMmM7tyVIWXoUNWWs,7655
|
13
|
-
deltacat/autoscaler/events/session_manager.py,sha256=Hb1zpNrc3r2tPFTJMrPW8c5oDF0XtsxSpJM1OcK__Wk,598
|
14
|
-
deltacat/autoscaler/events/states.py,sha256=AAMcIuH9lNSZ3uj7UUQgE9CgCs-rph1dXy0fQ0r-bNE,2080
|
15
|
-
deltacat/autoscaler/events/workflow.py,sha256=4o6AWguNiZB2hO6H7vIqTvYWxoVIQKi8xwWIcU2n8NI,2151
|
16
|
-
deltacat/autoscaler/events/compaction/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
17
|
-
deltacat/autoscaler/events/compaction/cluster.py,sha256=CPVG4gOaQonUX4uXpM4Ax8F_Wz826yG3SesqPrYbfvM,3323
|
18
|
-
deltacat/autoscaler/events/compaction/dispatcher.py,sha256=8PcjIkR930-hMqP1kpJXbeMIj41ezX4gQop8DkQEvTU,1176
|
19
|
-
deltacat/autoscaler/events/compaction/input.py,sha256=4lYsUSmsxvB4DbkdEJ3SMFxO8avJW8qakhkEaN69Wkk,1176
|
20
|
-
deltacat/autoscaler/events/compaction/process.py,sha256=0iyqngVv-Ib0VIQRQH8U-Q29Zl90rpzwb3TxWNTtWFQ,1161
|
21
|
-
deltacat/autoscaler/events/compaction/session_manager.py,sha256=gO2nNPTAorQzCaQzpCYXcstlU8aZLOEPH95d_Et4ci8,598
|
22
|
-
deltacat/autoscaler/events/compaction/utils.py,sha256=k_x4wBo5WvzBMV21EwFbNnqp1NyirR9LZc9Ez9hG6k8,10073
|
23
|
-
deltacat/autoscaler/events/compaction/workflow.py,sha256=dYEJBpiJ66XIkinD-g28JM018UMR5wUN2DLXfW-VWHI,18182
|
24
|
-
deltacat/autoscaler/events/compaction/collections/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
25
|
-
deltacat/autoscaler/events/compaction/collections/partition_key_value.py,sha256=kc5regg9DIKmFAlEriGYf6QoIna63eIMPSBhcVCccFk,787
|
26
|
-
deltacat/autoscaler/events/dynamodb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
27
|
-
deltacat/autoscaler/events/dynamodb/event_store.py,sha256=_2_nHcpLOqml7pDowOy5hwbmEzdwzG_OERpCApS6I10,5686
|
28
|
-
deltacat/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
29
|
-
deltacat/aws/clients.py,sha256=5i2F-o3fRI4JzaYw-2015lwf1fYs_Zir6JnSPQK5Mjo,1936
|
30
|
-
deltacat/aws/constants.py,sha256=jbcUubyhkMOmDh_XpTljBX-0hnhYJduetJcLqRmpfl4,204
|
31
|
-
deltacat/aws/s3u.py,sha256=f-Q-L4mSxKgBGiT2X5JuF423GHkYGhLI61pymR8u6hg,18817
|
32
|
-
deltacat/aws/redshift/__init__.py,sha256=pJ0pHF2extLnNKyt7bhCVBsf61Ri-M-eoejqsAhlJlU,247
|
33
|
-
deltacat/aws/redshift/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
34
|
-
deltacat/aws/redshift/model/manifest.py,sha256=mMsUgB2YxtmvRuJg2YJozKbcPlAKHcWT6GB_7zyI9oo,9575
|
35
|
-
deltacat/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
36
|
-
deltacat/catalog/delegate.py,sha256=zde_eXYcQaUheXQcNz58BwcaXieYfeCXYoAuBis6Pqs,9272
|
37
|
-
deltacat/catalog/interface.py,sha256=TTFHmbSWd9lUQx5T0RhOAddCU2HcQrXlN0dD-ewYs4M,6783
|
38
|
-
deltacat/catalog/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
39
|
-
deltacat/catalog/model/catalog.py,sha256=0nw-xk5qKl4xJC51U6IBPRB76pfwMkr3vmNRn25VRdE,2270
|
40
|
-
deltacat/catalog/model/table_definition.py,sha256=pUc8sHSjhD3ifGrUeWevruTNsNUD0xtaPcc_sddTjjo,697
|
41
|
-
deltacat/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
42
|
-
deltacat/compute/compactor/__init__.py,sha256=MLh6Y9yP0piKMW5Xh9JdS-qJ7PVaJXpzk13mLLzZxig,1103
|
43
|
-
deltacat/compute/compactor/compaction_session.py,sha256=40hsNMr_VfMnbjNvUZuU2AUdBepMIDSh7if7Xp9m3GI,26159
|
44
|
-
deltacat/compute/compactor/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
45
|
-
deltacat/compute/compactor/model/delta_annotated.py,sha256=EXxV-bdwhCqzjzxSy4GAH7xFzC9g49VNZXkrIzjxdb4,7599
|
46
|
-
deltacat/compute/compactor/model/delta_file_envelope.py,sha256=5VeD4krZpI3mWGK1odwvcRz9CK5_IbutN__SLrAv2gQ,1855
|
47
|
-
deltacat/compute/compactor/model/delta_file_locator.py,sha256=_WMa37EVFmpR-BERBEKY8-Xh3chtTUs5hKfgNsx65jc,1736
|
48
|
-
deltacat/compute/compactor/model/materialize_result.py,sha256=96OnkdwZ6jhLc4QWd1PLNUbxnhaCXt0qyOW7xl_RU8M,1267
|
49
|
-
deltacat/compute/compactor/model/primary_key_index.py,sha256=uVFCef5vIYTdvtSN4T8iP80iQNEahH8qBPZmNC6ybc8,10524
|
50
|
-
deltacat/compute/compactor/model/pyarrow_write_result.py,sha256=YjHklyhJm_uZIjW45Xj0prl1pLRCAPjlPG4ypPSa8s8,1343
|
51
|
-
deltacat/compute/compactor/model/round_completion_info.py,sha256=rElYyqdavQhpzBDUat1v63XMKbbbmn8RfN7ezoVl_u0,2755
|
52
|
-
deltacat/compute/compactor/model/sort_key.py,sha256=tU9XJa455wuvssb3oeuYwPagPMU55TT5iH6TIZD8TK0,4062
|
53
|
-
deltacat/compute/compactor/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
54
|
-
deltacat/compute/compactor/steps/dedupe.py,sha256=TXnfPoPlTMs6duh-uDovCWqTmyr6o8a2K0YTbyyZ4m4,16277
|
55
|
-
deltacat/compute/compactor/steps/hash_bucket.py,sha256=hu4cG9p73All4HkSFF3FlTPJigQhv5x3vtWUJLCUuoo,6636
|
56
|
-
deltacat/compute/compactor/steps/materialize.py,sha256=UJL_V9rZX5KvD6hRR3SNQ5rKXQzp00lsUu5lUpllq28,7272
|
57
|
-
deltacat/compute/compactor/steps/rehash/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
58
|
-
deltacat/compute/compactor/steps/rehash/rehash_bucket.py,sha256=h5w-lNTiaufIO_gbMVnCn3vJ4s9RdT9_NzkeyX9aIQg,1794
|
59
|
-
deltacat/compute/compactor/steps/rehash/rewrite_index.py,sha256=n743Xo1j_eUl22_AFEhTfrCzYSydwpXFaFCfruJr4Kk,1869
|
60
|
-
deltacat/compute/compactor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
61
|
-
deltacat/compute/compactor/utils/io.py,sha256=1E0MG_mxk_RGzl4I5xYUSuyWXAaOuWkBSxOX4khx0LY,9678
|
62
|
-
deltacat/compute/compactor/utils/primary_key_index.py,sha256=H2odhlzplrgL1LcwelaRoVNFHNy3p_1NgFW0dfaSesw,10864
|
63
|
-
deltacat/compute/compactor/utils/round_completion_file.py,sha256=uvf-pyIQCVmfpM0Ew1qjZ7OVK4-jJifuu-u_dverEPw,2116
|
64
|
-
deltacat/compute/compactor/utils/system_columns.py,sha256=1M4JHwqDIRnfQaqvPz8_zBw05de45Db06To-3wiUVUk,7003
|
65
|
-
deltacat/compute/metastats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
66
|
-
deltacat/compute/metastats/meta_stats.py,sha256=kCRWfQ2zDVrjfIvhE1S6B0ShRPn5bRQA-z8r8VE2Oco,20588
|
67
|
-
deltacat/compute/metastats/stats.py,sha256=LI9ndlwfAs7Ad-MsKaBeGL92JieRh8a54RxwXfRqDCU,7375
|
68
|
-
deltacat/compute/metastats/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
69
|
-
deltacat/compute/metastats/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
70
|
-
deltacat/compute/metastats/model/partition_stats_dict.py,sha256=o7BlH-Eyv9RzesLTw7OoTKWqkHFI9bE0ZuefjTr3viA,1405
|
71
|
-
deltacat/compute/metastats/model/stats_cluster_size_estimator.py,sha256=3965WMAHQAzUcQSnK3UaW-ULGIC7qibaz0frl7h0DdI,2766
|
72
|
-
deltacat/compute/metastats/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
73
|
-
deltacat/compute/metastats/utils/constants.py,sha256=bFUPKmR3FkjEnwpHuToQYZ9QcHqYpd4OMMSwVwnJcaA,869
|
74
|
-
deltacat/compute/metastats/utils/io.py,sha256=NBEu4wkS8wNUYXULHXWGq1fEscmEneNNlJgHUgtNxnk,8531
|
75
|
-
deltacat/compute/metastats/utils/pyarrow_memory_estimation_function.py,sha256=BYdnNZywoEoZIcbikuWYU97Ox3BRdPJExNUL10QnPkY,1135
|
76
|
-
deltacat/compute/metastats/utils/ray_utils.py,sha256=-SDouyx5O0Qt4ZiGFmU0lGHvWZoXO3w5BS_Vx39gJa4,4591
|
77
|
-
deltacat/compute/stats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
78
|
-
deltacat/compute/stats/basic.py,sha256=fL5UgyHKeUdQqzr1lXstbHmWSIvawTTXk5pRmhyH-HQ,9415
|
79
|
-
deltacat/compute/stats/types.py,sha256=cp0lT8nITTKbnkc03OysRjXfcfXzQml9a4wqCnR6kqs,215
|
80
|
-
deltacat/compute/stats/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
81
|
-
deltacat/compute/stats/models/delta_column_stats.py,sha256=_m-fGbx2njONr9N2ihkIajUzVvmy8weDG5BLJdMaoQU,3296
|
82
|
-
deltacat/compute/stats/models/delta_stats.py,sha256=yzxum2boP5c4cTSNaLOSgdioQ4ZPtSoMX_Ju3k-4MY4,8711
|
83
|
-
deltacat/compute/stats/models/delta_stats_cache_result.py,sha256=ga_by81TNDpptjYYUzpQHVRCR95_NnZ1a59apmvVBUo,1609
|
84
|
-
deltacat/compute/stats/models/manifest_entry_stats.py,sha256=vGTOq_a1XFFf3N_rcB4QXTHuEx-T3P5KwJllrKi7zuw,2363
|
85
|
-
deltacat/compute/stats/models/stats_result.py,sha256=j3bjU8LdLbTkpM4ZZojQ70WfqHsXFNftJG2PwKReSRg,3753
|
86
|
-
deltacat/compute/stats/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
87
|
-
deltacat/compute/stats/utils/intervals.py,sha256=PmbE1856k_y0p6lNrQ3rNC_RBT8PROA2wQRFVM2s8P8,3049
|
88
|
-
deltacat/compute/stats/utils/io.py,sha256=H-xIrWOqN4JSDuo3imrKDEyU_xDrfKU2tFwdOg7Jne4,8587
|
89
|
-
deltacat/compute/stats/utils/manifest_stats_file.py,sha256=rc3BdQ-QjgRSz6vBGw_x3shShnkNVeVVMEzs8n9Ir6A,3811
|
90
|
-
deltacat/io/__init__.py,sha256=4fQUUNSj8pnpFw4oF6CuDbhizj5JU0Jml2qQd-lVUvA,153
|
91
|
-
deltacat/io/dataset.py,sha256=DG1TfhAX1dWYVcUAryOgLyqgDQ71syVRvkFxPS0BJjQ,3821
|
92
|
-
deltacat/io/read_api.py,sha256=kkcclEshS5MEEI6gzJ0HkcIug2ph_DE8aUdBddUaRPU,7109
|
93
|
-
deltacat/io/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
94
|
-
deltacat/io/aws/redshift/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
95
|
-
deltacat/io/aws/redshift/redshift_datasource.py,sha256=BWlQoAVnGi8Urj5rF5ZOePWy0O-xxOgJIwLt-f-s6BM,22765
|
96
|
-
deltacat/storage/__init__.py,sha256=doAx2Ojk8BJik80crGrlX0rph8HKk2Cm5JJ6HqfP5qc,1337
|
97
|
-
deltacat/storage/interface.py,sha256=d2b3Y1caGuqS-QaSpaCw6gnAkSCZgQfGEWKuVIocHIE,22080
|
98
|
-
deltacat/storage/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
99
|
-
deltacat/storage/model/delta.py,sha256=1fXuIOq_S4u164fVnFe58jVKR5NTARP2bWqAZOkEpOY,13912
|
100
|
-
deltacat/storage/model/list_result.py,sha256=zGa32B3c98qtxRJq376h3eHQmQLulhhOna0G8psPsJY,2445
|
101
|
-
deltacat/storage/model/locator.py,sha256=1S7szmDSx-R4Z3arFNILOvS4t7dF7_rJNV9fHyRc3G4,1296
|
102
|
-
deltacat/storage/model/namespace.py,sha256=pvz4NNU5W4l8LbLGJPbmPb9uwL6rlTr64yAtwJ7GHVs,2108
|
103
|
-
deltacat/storage/model/partition.py,sha256=XcYxzfW3adTJYB8ZkeirCdZ41UhoBH8CojZ84KUJo40,11165
|
104
|
-
deltacat/storage/model/stream.py,sha256=gLzpscjeU4EHGowXDyJMqstN64v57FzGlyDdZJTwuwQ,7818
|
105
|
-
deltacat/storage/model/table.py,sha256=zTlM1B2R7fpN5F0ktA_KpE2DlmQBSn8iNOpLI9PP65w,4252
|
106
|
-
deltacat/storage/model/table_version.py,sha256=qDCafR53oMBDUMcg6sxkGOFZdWyf7oL1sew5VKobIss,7086
|
107
|
-
deltacat/storage/model/types.py,sha256=u2yCTqxzoQz8pcrzr8uD0xXP35kPisQKcbDgzac1YQ8,1579
|
108
|
-
deltacat/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
109
|
-
deltacat/tests/stats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
110
|
-
deltacat/tests/stats/test_intervals.py,sha256=NhKBqTgTRcVPqrExlJNrVHyToeaA9dX01UwJsKzQZMM,2137
|
111
|
-
deltacat/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
112
|
-
deltacat/types/media.py,sha256=S0cYjbcRSN8I9EewZvmezYTWbIy01Sa2pLl67hMThwE,2186
|
113
|
-
deltacat/types/tables.py,sha256=KCXmpv-hVRbsySGDaFM1gTsygociRx0TkrxLY1vNNNc,3814
|
114
|
-
deltacat/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
115
|
-
deltacat/utils/common.py,sha256=0yenkHroJiQjeenXZnNSDSD06-KJuBIsw2GAMqqQSKI,1449
|
116
|
-
deltacat/utils/numpy.py,sha256=D1dCa7xTOlZXzxDhaETzl1c8zJNxGpBr_-_Cb5yzXAI,2094
|
117
|
-
deltacat/utils/pandas.py,sha256=SBEnH2eQgjrQkdW6Dvk3Ky3reTI0xYRGFE2JIOsZqaA,10067
|
118
|
-
deltacat/utils/performance.py,sha256=GsHuRG8Zi6iVmIC9F4fXUDbEFk03fErVJy8JjYCWsUE,670
|
119
|
-
deltacat/utils/placement.py,sha256=knanJlEMSGMA3t-UISsLTUP-fQ9CzzR1LZO29L2HfB4,8823
|
120
|
-
deltacat/utils/pyarrow.py,sha256=mc93BdFip-0aLdLuQtNXlf7LoFX2k5txdOhWYehq1-g,10611
|
121
|
-
deltacat/utils/ray_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
122
|
-
deltacat/utils/ray_utils/collections.py,sha256=2B90lyLulxS7tBMkyzLAVC1fIeqYspxuaVBeXGveYlE,1950
|
123
|
-
deltacat/utils/ray_utils/concurrency.py,sha256=2L6VVQrjTN1DTNBL8B4xy5Ty6lmO6G_M8vN9TDJuLrA,4906
|
124
|
-
deltacat/utils/ray_utils/dataset.py,sha256=JPonxxxmDuAUiYqmBBTM2hyXHPggNGgp3QvsUxSjvNI,3319
|
125
|
-
deltacat/utils/ray_utils/performance.py,sha256=v0JHrjlTGw4-ypzy_ZBlY4mOQN9o9euQiwc6IWk7-bY,515
|
126
|
-
deltacat/utils/ray_utils/runtime.py,sha256=8Z7WlIYylij93vhhwUf3bijrpGKCEpEKlXVI5KCC1ps,4810
|
127
|
-
deltacat-0.1.8.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
128
|
-
deltacat-0.1.8.dist-info/METADATA,sha256=s9S-3IUuKC_fjg5w9M3ODpRaQvy_YdVRePQo-RyJAvM,1414
|
129
|
-
deltacat-0.1.8.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
|
130
|
-
deltacat-0.1.8.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
|
131
|
-
deltacat-0.1.8.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|