deltacat 0.1.8__py3-none-any.whl → 0.1.11__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. deltacat/__init__.py +41 -15
  2. deltacat/aws/clients.py +12 -31
  3. deltacat/aws/constants.py +1 -1
  4. deltacat/aws/redshift/__init__.py +7 -2
  5. deltacat/aws/redshift/model/manifest.py +54 -50
  6. deltacat/aws/s3u.py +188 -218
  7. deltacat/catalog/delegate.py +151 -185
  8. deltacat/catalog/interface.py +78 -97
  9. deltacat/catalog/model/catalog.py +21 -21
  10. deltacat/catalog/model/table_definition.py +11 -9
  11. deltacat/compute/compactor/__init__.py +12 -16
  12. deltacat/compute/compactor/compaction_session.py +259 -316
  13. deltacat/compute/compactor/model/delta_annotated.py +60 -44
  14. deltacat/compute/compactor/model/delta_file_envelope.py +5 -6
  15. deltacat/compute/compactor/model/delta_file_locator.py +10 -8
  16. deltacat/compute/compactor/model/materialize_result.py +6 -7
  17. deltacat/compute/compactor/model/primary_key_index.py +38 -34
  18. deltacat/compute/compactor/model/pyarrow_write_result.py +3 -4
  19. deltacat/compute/compactor/model/round_completion_info.py +25 -19
  20. deltacat/compute/compactor/model/sort_key.py +18 -15
  21. deltacat/compute/compactor/steps/dedupe.py +152 -259
  22. deltacat/compute/compactor/steps/hash_bucket.py +57 -73
  23. deltacat/compute/compactor/steps/materialize.py +138 -99
  24. deltacat/compute/compactor/steps/rehash/rehash_bucket.py +13 -13
  25. deltacat/compute/compactor/steps/rehash/rewrite_index.py +11 -13
  26. deltacat/compute/compactor/utils/io.py +59 -47
  27. deltacat/compute/compactor/utils/primary_key_index.py +131 -90
  28. deltacat/compute/compactor/utils/round_completion_file.py +22 -23
  29. deltacat/compute/compactor/utils/system_columns.py +33 -42
  30. deltacat/compute/metastats/meta_stats.py +235 -157
  31. deltacat/compute/metastats/model/partition_stats_dict.py +7 -10
  32. deltacat/compute/metastats/model/stats_cluster_size_estimator.py +13 -5
  33. deltacat/compute/metastats/stats.py +95 -64
  34. deltacat/compute/metastats/utils/io.py +100 -53
  35. deltacat/compute/metastats/utils/pyarrow_memory_estimation_function.py +5 -2
  36. deltacat/compute/metastats/utils/ray_utils.py +38 -33
  37. deltacat/compute/stats/basic.py +107 -69
  38. deltacat/compute/stats/models/delta_column_stats.py +11 -8
  39. deltacat/compute/stats/models/delta_stats.py +59 -32
  40. deltacat/compute/stats/models/delta_stats_cache_result.py +4 -1
  41. deltacat/compute/stats/models/manifest_entry_stats.py +12 -6
  42. deltacat/compute/stats/models/stats_result.py +24 -14
  43. deltacat/compute/stats/utils/intervals.py +16 -9
  44. deltacat/compute/stats/utils/io.py +86 -51
  45. deltacat/compute/stats/utils/manifest_stats_file.py +24 -33
  46. deltacat/constants.py +8 -10
  47. deltacat/io/__init__.py +2 -2
  48. deltacat/io/aws/redshift/redshift_datasource.py +157 -143
  49. deltacat/io/dataset.py +14 -17
  50. deltacat/io/read_api.py +36 -33
  51. deltacat/logs.py +94 -42
  52. deltacat/storage/__init__.py +18 -8
  53. deltacat/storage/interface.py +196 -213
  54. deltacat/storage/model/delta.py +45 -51
  55. deltacat/storage/model/list_result.py +12 -8
  56. deltacat/storage/model/namespace.py +4 -5
  57. deltacat/storage/model/partition.py +42 -42
  58. deltacat/storage/model/stream.py +29 -30
  59. deltacat/storage/model/table.py +14 -14
  60. deltacat/storage/model/table_version.py +32 -31
  61. deltacat/storage/model/types.py +1 -0
  62. deltacat/tests/stats/test_intervals.py +11 -24
  63. deltacat/tests/utils/test_record_batch_tables.py +284 -0
  64. deltacat/types/media.py +3 -4
  65. deltacat/types/tables.py +31 -21
  66. deltacat/utils/common.py +5 -11
  67. deltacat/utils/numpy.py +20 -22
  68. deltacat/utils/pandas.py +73 -100
  69. deltacat/utils/performance.py +3 -9
  70. deltacat/utils/placement.py +276 -231
  71. deltacat/utils/pyarrow.py +302 -89
  72. deltacat/utils/ray_utils/collections.py +2 -1
  73. deltacat/utils/ray_utils/concurrency.py +38 -32
  74. deltacat/utils/ray_utils/dataset.py +28 -28
  75. deltacat/utils/ray_utils/performance.py +5 -9
  76. deltacat/utils/ray_utils/runtime.py +9 -10
  77. {deltacat-0.1.8.dist-info → deltacat-0.1.11.dist-info}/METADATA +22 -12
  78. deltacat-0.1.11.dist-info/RECORD +110 -0
  79. {deltacat-0.1.8.dist-info → deltacat-0.1.11.dist-info}/WHEEL +1 -1
  80. deltacat/autoscaler/events/__init__.py +0 -0
  81. deltacat/autoscaler/events/compaction/__init__.py +0 -0
  82. deltacat/autoscaler/events/compaction/cluster.py +0 -82
  83. deltacat/autoscaler/events/compaction/collections/__init__.py +0 -0
  84. deltacat/autoscaler/events/compaction/collections/partition_key_value.py +0 -36
  85. deltacat/autoscaler/events/compaction/dispatcher.py +0 -28
  86. deltacat/autoscaler/events/compaction/input.py +0 -27
  87. deltacat/autoscaler/events/compaction/process.py +0 -25
  88. deltacat/autoscaler/events/compaction/session_manager.py +0 -13
  89. deltacat/autoscaler/events/compaction/utils.py +0 -216
  90. deltacat/autoscaler/events/compaction/workflow.py +0 -303
  91. deltacat/autoscaler/events/dispatcher.py +0 -95
  92. deltacat/autoscaler/events/dynamodb/__init__.py +0 -0
  93. deltacat/autoscaler/events/dynamodb/event_store.py +0 -164
  94. deltacat/autoscaler/events/event_store.py +0 -55
  95. deltacat/autoscaler/events/exceptions.py +0 -6
  96. deltacat/autoscaler/events/processor.py +0 -177
  97. deltacat/autoscaler/events/session_manager.py +0 -25
  98. deltacat/autoscaler/events/states.py +0 -88
  99. deltacat/autoscaler/events/workflow.py +0 -54
  100. deltacat/autoscaler/node_group.py +0 -230
  101. deltacat/autoscaler/utils.py +0 -69
  102. deltacat-0.1.8.dist-info/RECORD +0 -131
  103. /deltacat/{autoscaler → tests/utils}/__init__.py +0 -0
  104. {deltacat-0.1.8.dist-info → deltacat-0.1.11.dist-info}/LICENSE +0 -0
  105. {deltacat-0.1.8.dist-info → deltacat-0.1.11.dist-info}/top_level.txt +0 -0
@@ -1,230 +0,0 @@
1
- import ray
2
- import re
3
- import time
4
- import yaml
5
- import logging
6
- from typing import Optional, Union, List, Dict, Any, Callable, Tuple
7
- from ray.util.placement_group import (
8
- placement_group,
9
- placement_group_table,
10
- remove_placement_group,
11
- get_current_placement_group
12
- )
13
-
14
- from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
15
- from ray.experimental.state.api import list_placement_groups, get_node, get_placement_group
16
-
17
-
18
- from deltacat import logs
19
- logger = logs.configure_deltacat_logger(logging.getLogger(__name__))
20
-
21
- class NodeGroupManager():
22
-
23
- def __init__(self,path: str, gname: str):
24
- """Node Group Manager
25
- Args:
26
- path: cluster yaml file
27
- gname: node group prefix, e.g., 'partition'
28
- """
29
- #cluster init status:
30
- self.NODE_GROUP_PREFIX=gname
31
- self.cluster_config=self._read_yaml(path)
32
- self.init_groups = self._cluster_node_groups(self.cluster_config)
33
- self.init_group_res = self._parse_node_resources()
34
-
35
- def _cluster_node_groups(self, config: Dict[str, Any]) -> Dict[str, Any]:
36
- """Get Worker Groups
37
- Args:
38
- config: cluster yaml data
39
- Returns:
40
- worker groups: a dict of worker node group
41
-
42
- """
43
- avail_node_types = list(config['available_node_types'].items())
44
- #exclude head node type
45
- head_node_types = [nt for nt in avail_node_types if 'resources' in nt[1] and 'CPU' in nt[1]['resources'] and nt[1]['resources']['CPU']==0][0]
46
- worker_node_types = [x for x in avail_node_types if x !=head_node_types]
47
- #assuming homogenous cluster
48
- #in future, update with fleet resource
49
- if len(worker_node_types)>0:
50
- self.INSTANCE_TYPE = worker_node_types[0][1]['node_config']['InstanceType']
51
- return worker_node_types
52
-
53
-
54
- def _read_yaml(self, path: str) -> Dict[str, Any]:
55
- with open(path, "rt") as f:
56
- return yaml.safe_load(f)
57
-
58
- def _update_groups(self) -> List[Tuple[str, float]]:
59
- """
60
- Node groups can come and go during runtime, whenever a node group is needed, we need to check the current available groups
61
- Returns:
62
- current_groups: dict of custom resource groups
63
- """
64
- #Add 1.1 second latency to avoid inconsistency issue between raylet and head
65
- time.sleep(1.1)
66
- all_available_res = ray.available_resources()
67
- current_groups =[(k,all_available_res[k]) for k in all_available_res.keys() if self.NODE_GROUP_PREFIX in k]
68
- return current_groups
69
-
70
- def _parse_node_resources(self) -> Dict[str, Dict[str, float]]:
71
- """
72
- Parse resources per node to get detailed resource tighted to each node group
73
- Returns:
74
- group_res: a dict of resources, e.g., {'CPU':0,'memory':0,'object_store_memory':0}
75
- """
76
- all_available_resources= ray._private.state.state._available_resources_per_node()
77
- group_keys = [x[0] for x in self.init_groups]
78
- group_res={}
79
- for k in group_keys:
80
- group_res[k]={'CPU':0,'memory':0,'object_store_memory':0,'node_id':[]}
81
- for v in all_available_resources.values():
82
- keys =v.keys()
83
- r = re.compile(self.NODE_GROUP_PREFIX)
84
- partition=list(filter(r.match, list(keys)))
85
- r = re.compile("node:")
86
- node_id = list(filter(r.match, list(keys)))
87
- if len(partition)>0:
88
- partition = partition[0]
89
- if len(node_id)>0:
90
- node_id = node_id[0]
91
- if self.NODE_GROUP_PREFIX in partition:
92
- group_res[partition]['CPU']+=v['CPU']
93
- group_res[partition]['memory']+=v['memory']
94
- group_res[partition]['object_store_memory']+=v['object_store_memory']
95
- group_res[partition]['node_id'].append(node_id)
96
- return group_res
97
-
98
- def _update_group_res(self, gname: str) -> Dict[str, Union[str, float]]:
99
- """
100
- Get the realtime resource of a node group
101
- Args:
102
- gname: name of node group
103
- Returns:
104
- group_res: dict of updated resource(cpu, memory, object store memory) for a given group
105
- """
106
- all_available_resources= ray._private.state.state._available_resources_per_node()
107
- group_res={'CPU':0,'memory':0,'object_store_memory':0,'node_id':[]}
108
- for v in all_available_resources.values():
109
- keys =v.keys()
110
- r = re.compile("node:")
111
- node_id = list(filter(r.match, list(keys)))
112
- if len(node_id)>0:
113
- node_id = node_id[0]
114
- if gname in v.keys():
115
- group_res['CPU']+=v['CPU']
116
- group_res['memory']+=v['memory']
117
- group_res['object_store_memory']+=v['object_store_memory']
118
- group_res['node_id'].append(node_id)
119
- return group_res
120
-
121
- def get_one_group(self) -> Optional[Dict[str, Union[str, float]]]:
122
- """
123
- Pop up one node group
124
- Returns:
125
- group_res: dict of node group resource, {"group":"partition_1","CPU":2,...}
126
- """
127
- current_groups = self._update_groups()
128
- if len(current_groups)>0:
129
- gname = current_groups[-1][0]
130
- group_res=self._update_group_res(gname)
131
- group_res['group']=gname
132
- try:
133
- group_res['group_res']=ray.available_resources()[gname]
134
- except Exception as e:
135
- print("There is no available resources for %s"%gname)
136
- return None
137
- return group_res
138
- else:
139
- return None
140
-
141
- def get_group_by_name(self, gname: str) -> Optional[Dict[str, Union[str, float]]]:
142
- """
143
- Get the specific node group given its pre-filled name
144
- Args:
145
- gname: name of the node group
146
- Returns:
147
- group_res: dict of node group resource
148
-
149
- """
150
- group_res=self._update_group_res(gname)
151
- group_res['group']=gname
152
- try:
153
- group_res['group_res']=ray.available_resources()[gname]
154
- except Exception as e:
155
- print("There is no available resources for %s"%gname)
156
- return None
157
- return group_res
158
-
159
-
160
- class PlacementGroupManager():
161
- """Placement Group Manager
162
- Create a list of placement group with the desired number of cpus
163
- e.g., create a pg with 32 cpus, then this class will look for a node that has 32 cpus, and collect all
164
- resources, including cpu, memory, and object store;
165
- How to use:
166
- ```
167
- from deltacat.autoscaler.node_group import PlacementGroupManager as pgm
168
- pgm = pgm(10, 32)
169
- pg_configs = pgm.pgs
170
- opts = pg_configs[0][0]
171
- fun.options(**opts).remote()
172
- ```
173
- Args:
174
- num_pgs: number of placement groups to be created
175
- instance_cpus: number of cpus per instance
176
- """
177
- def __init__(self, num_pgs: int, instance_cpus: int, time_out: Optional[float] = None):
178
- #self._pg_configs =self._config(num_pgs, instance_cpus, time_out)
179
- self._pg_configs = ray.get([_config.remote(instance_cpus) for _ in range(num_pgs)])
180
- @property
181
- def pgs(self):
182
- return self._pg_configs
183
-
184
- @ray.remote
185
- def _config(instance_cpus: int, time_out: Optional[float] = None) -> Tuple[Dict[str,Any], Dict[str,Any]]:
186
- pg_config = None
187
- try:
188
- opts ={}
189
- cluster_resources={}
190
- bundle1={'CPU': instance_cpus} # hardcode for now
191
- pg = placement_group([bundle1], strategy="PACK")
192
- ray.get(pg.ready(), timeout=time_out)
193
- if not pg:
194
- return
195
- opts = {"scheduling_strategy":PlacementGroupSchedulingStrategy(
196
- placement_group=pg, placement_group_capture_child_tasks=True)
197
- }
198
- pg_id = placement_group_table(pg)['placement_group_id']
199
- #print("pg id{}".format(pg_id))
200
- pg_details = get_placement_group(pg_id)
201
- #print("pgs-{}".format(pgs))
202
- bundles = pg_details['bundles']
203
- node_ids =[]
204
- for bd in bundles:
205
- node_ids.append(bd['node_id'])
206
- #query available resources given list of node id
207
- all_nodes_available_res = ray._private.state.state._available_resources_per_node()
208
- pg_res = {'CPU':0,'memory':0,'object_store_memory':0,'node_id':[]}
209
- for node_id in node_ids:
210
- if node_id in all_nodes_available_res:
211
- v = all_nodes_available_res[node_id]
212
- node_detail = get_node(node_id)
213
- pg_res['CPU']+=node_detail['resources_total']['CPU']
214
- pg_res['memory']+=v['memory']
215
- pg_res['object_store_memory']+=v['object_store_memory']
216
- #pg_res['node_id'].append(node_id)
217
- cluster_resources['CPU'] = int(pg_res['CPU'])
218
- cluster_resources['memory'] = float(pg_res['memory'])
219
- cluster_resources['object_store_memory'] = float(pg_res['object_store_memory'])
220
- #cluster_resources['node_id'] = pg_res['node_id']
221
- #cluster_cpus = cluster_resources['CPU']
222
- pg_config=[opts,cluster_resources]
223
- print("pg has resources:{}".format(cluster_resources))
224
-
225
- except Exception as e:
226
- print("Error in creating pg:{}".format(e))
227
- logger.error(f"placement group error{e}")
228
- pass
229
- return pg_config
230
-
@@ -1,69 +0,0 @@
1
- import os
2
- import subprocess
3
- from typing import Any, Optional, List
4
-
5
- import ray
6
-
7
-
8
- def run_cmd(cmd: str,
9
- background_process: bool = False,
10
- stdout: Optional[Any] = subprocess.DEVNULL,
11
- stderr: Optional[Any] = subprocess.STDOUT) -> None:
12
- cmd_list = cmd.split(" ")
13
- if background_process:
14
- cmd_list = ["nohup"] + cmd_list
15
- subprocess.Popen(cmd_list,
16
- stdout=stdout,
17
- stderr=stderr)
18
- return
19
- else:
20
- exit_code = subprocess.call(cmd_list, stdout=stdout, stderr=stderr)
21
- assert exit_code == 0, f"`{cmd}` failed. Exit code: {exit_code}"
22
-
23
-
24
- def ray_submit(cluster_cfg: str,
25
- path_to_script: str,
26
- script_arguments: Optional[List[str]] = None,
27
- background_process: bool = False,
28
- start_cluster: bool = True,
29
- stop_cluster: bool = True,
30
- stdout: Optional[Any] = subprocess.DEVNULL,
31
- stderr: Optional[Any] = subprocess.STDOUT) -> None:
32
- cluster_args = []
33
- if start_cluster:
34
- cluster_args.append("--start")
35
- if stop_cluster:
36
- cluster_args.append("--stop")
37
-
38
- if script_arguments is None:
39
- script_arguments = []
40
-
41
- script_arguments_str = " ".join(script_arguments)
42
-
43
- cmd = f"ray submit --log-style record --no-config-cache {' '.join(cluster_args)} {cluster_cfg} {path_to_script}"
44
- if script_arguments:
45
- cmd = f"{cmd} -- {script_arguments_str}"
46
-
47
- run_cmd(cmd, background_process=background_process, stdout=stdout, stderr=stderr)
48
- print(f"Submitted script on Ray cluster '{cluster_cfg}' with command '{cmd}'")
49
-
50
-
51
- def ray_up(cluster_cfg: str) -> None:
52
- print(f"Starting Ray cluster '{cluster_cfg}'")
53
- run_cmd(f"ray up {cluster_cfg} -y --no-config-cache --no-restart")
54
- print(f"Started Ray cluster '{cluster_cfg}'")
55
-
56
-
57
- def get_head_node_ip(cluster_cfg: str) -> str:
58
- print(f"Getting Ray cluster head node IP for '{cluster_cfg}'")
59
- proc = subprocess.run(
60
- f"ray get-head-ip {cluster_cfg}",
61
- shell=True,
62
- capture_output=True,
63
- text=True,
64
- check=True)
65
- # the head node IP should be the last line printed to stdout
66
- head_node_ip = proc.stdout.splitlines()[-1]
67
- print(f"Ray cluster head node IP for '{cluster_cfg}': {head_node_ip}")
68
- return head_node_ip
69
-
@@ -1,131 +0,0 @@
1
- deltacat/__init__.py,sha256=PykA9TJ4ofPC4lwnnzyAgabk_POjYX8QZ-b1TOwe8IY,1675
2
- deltacat/constants.py,sha256=ZGSRdqQQEIuQGO_Qwkl3tQxdZGVl4oLH01tCdXvyURo,1092
3
- deltacat/exceptions.py,sha256=x7qem7FLujXf-DzPsNcQ-XYkW3cF3A0YGIbxkcpz0Mw,146
4
- deltacat/logs.py,sha256=ZvMw7mhQZTNZH1Czwd_rW1OAnnFiQpogN4441UjWVQo,3879
5
- deltacat/autoscaler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
6
- deltacat/autoscaler/node_group.py,sha256=On7UffaEsl_iSFTY5biAakECRYCou8r6P661WzgeUqU,8026
7
- deltacat/autoscaler/utils.py,sha256=76bcUt2tEu6fxUpKBQwyWr_Ea2Xnwvjff2OxvLhQ0sc,2368
8
- deltacat/autoscaler/events/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
9
- deltacat/autoscaler/events/dispatcher.py,sha256=T2oa9wmm3bcCmmoBmj8fjZojR9m0aTe2OVp8uKE5ZMQ,3792
10
- deltacat/autoscaler/events/event_store.py,sha256=26uggtK8DbGv4tflW80z32KKwGzvfNYoNeYBfPqGgcw,1764
11
- deltacat/autoscaler/events/exceptions.py,sha256=1vdhgrCaXd10-NNM5Etem-q-041ZJBzom0IXKSfJpQk,97
12
- deltacat/autoscaler/events/processor.py,sha256=vW9K0paNBXaNQyJbRxjTzc4r4GbMmM7tyVIWXoUNWWs,7655
13
- deltacat/autoscaler/events/session_manager.py,sha256=Hb1zpNrc3r2tPFTJMrPW8c5oDF0XtsxSpJM1OcK__Wk,598
14
- deltacat/autoscaler/events/states.py,sha256=AAMcIuH9lNSZ3uj7UUQgE9CgCs-rph1dXy0fQ0r-bNE,2080
15
- deltacat/autoscaler/events/workflow.py,sha256=4o6AWguNiZB2hO6H7vIqTvYWxoVIQKi8xwWIcU2n8NI,2151
16
- deltacat/autoscaler/events/compaction/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
17
- deltacat/autoscaler/events/compaction/cluster.py,sha256=CPVG4gOaQonUX4uXpM4Ax8F_Wz826yG3SesqPrYbfvM,3323
18
- deltacat/autoscaler/events/compaction/dispatcher.py,sha256=8PcjIkR930-hMqP1kpJXbeMIj41ezX4gQop8DkQEvTU,1176
19
- deltacat/autoscaler/events/compaction/input.py,sha256=4lYsUSmsxvB4DbkdEJ3SMFxO8avJW8qakhkEaN69Wkk,1176
20
- deltacat/autoscaler/events/compaction/process.py,sha256=0iyqngVv-Ib0VIQRQH8U-Q29Zl90rpzwb3TxWNTtWFQ,1161
21
- deltacat/autoscaler/events/compaction/session_manager.py,sha256=gO2nNPTAorQzCaQzpCYXcstlU8aZLOEPH95d_Et4ci8,598
22
- deltacat/autoscaler/events/compaction/utils.py,sha256=k_x4wBo5WvzBMV21EwFbNnqp1NyirR9LZc9Ez9hG6k8,10073
23
- deltacat/autoscaler/events/compaction/workflow.py,sha256=dYEJBpiJ66XIkinD-g28JM018UMR5wUN2DLXfW-VWHI,18182
24
- deltacat/autoscaler/events/compaction/collections/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
25
- deltacat/autoscaler/events/compaction/collections/partition_key_value.py,sha256=kc5regg9DIKmFAlEriGYf6QoIna63eIMPSBhcVCccFk,787
26
- deltacat/autoscaler/events/dynamodb/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
27
- deltacat/autoscaler/events/dynamodb/event_store.py,sha256=_2_nHcpLOqml7pDowOy5hwbmEzdwzG_OERpCApS6I10,5686
28
- deltacat/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
29
- deltacat/aws/clients.py,sha256=5i2F-o3fRI4JzaYw-2015lwf1fYs_Zir6JnSPQK5Mjo,1936
30
- deltacat/aws/constants.py,sha256=jbcUubyhkMOmDh_XpTljBX-0hnhYJduetJcLqRmpfl4,204
31
- deltacat/aws/s3u.py,sha256=f-Q-L4mSxKgBGiT2X5JuF423GHkYGhLI61pymR8u6hg,18817
32
- deltacat/aws/redshift/__init__.py,sha256=pJ0pHF2extLnNKyt7bhCVBsf61Ri-M-eoejqsAhlJlU,247
33
- deltacat/aws/redshift/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
34
- deltacat/aws/redshift/model/manifest.py,sha256=mMsUgB2YxtmvRuJg2YJozKbcPlAKHcWT6GB_7zyI9oo,9575
35
- deltacat/catalog/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
36
- deltacat/catalog/delegate.py,sha256=zde_eXYcQaUheXQcNz58BwcaXieYfeCXYoAuBis6Pqs,9272
37
- deltacat/catalog/interface.py,sha256=TTFHmbSWd9lUQx5T0RhOAddCU2HcQrXlN0dD-ewYs4M,6783
38
- deltacat/catalog/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
39
- deltacat/catalog/model/catalog.py,sha256=0nw-xk5qKl4xJC51U6IBPRB76pfwMkr3vmNRn25VRdE,2270
40
- deltacat/catalog/model/table_definition.py,sha256=pUc8sHSjhD3ifGrUeWevruTNsNUD0xtaPcc_sddTjjo,697
41
- deltacat/compute/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
42
- deltacat/compute/compactor/__init__.py,sha256=MLh6Y9yP0piKMW5Xh9JdS-qJ7PVaJXpzk13mLLzZxig,1103
43
- deltacat/compute/compactor/compaction_session.py,sha256=40hsNMr_VfMnbjNvUZuU2AUdBepMIDSh7if7Xp9m3GI,26159
44
- deltacat/compute/compactor/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
45
- deltacat/compute/compactor/model/delta_annotated.py,sha256=EXxV-bdwhCqzjzxSy4GAH7xFzC9g49VNZXkrIzjxdb4,7599
46
- deltacat/compute/compactor/model/delta_file_envelope.py,sha256=5VeD4krZpI3mWGK1odwvcRz9CK5_IbutN__SLrAv2gQ,1855
47
- deltacat/compute/compactor/model/delta_file_locator.py,sha256=_WMa37EVFmpR-BERBEKY8-Xh3chtTUs5hKfgNsx65jc,1736
48
- deltacat/compute/compactor/model/materialize_result.py,sha256=96OnkdwZ6jhLc4QWd1PLNUbxnhaCXt0qyOW7xl_RU8M,1267
49
- deltacat/compute/compactor/model/primary_key_index.py,sha256=uVFCef5vIYTdvtSN4T8iP80iQNEahH8qBPZmNC6ybc8,10524
50
- deltacat/compute/compactor/model/pyarrow_write_result.py,sha256=YjHklyhJm_uZIjW45Xj0prl1pLRCAPjlPG4ypPSa8s8,1343
51
- deltacat/compute/compactor/model/round_completion_info.py,sha256=rElYyqdavQhpzBDUat1v63XMKbbbmn8RfN7ezoVl_u0,2755
52
- deltacat/compute/compactor/model/sort_key.py,sha256=tU9XJa455wuvssb3oeuYwPagPMU55TT5iH6TIZD8TK0,4062
53
- deltacat/compute/compactor/steps/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
54
- deltacat/compute/compactor/steps/dedupe.py,sha256=TXnfPoPlTMs6duh-uDovCWqTmyr6o8a2K0YTbyyZ4m4,16277
55
- deltacat/compute/compactor/steps/hash_bucket.py,sha256=hu4cG9p73All4HkSFF3FlTPJigQhv5x3vtWUJLCUuoo,6636
56
- deltacat/compute/compactor/steps/materialize.py,sha256=UJL_V9rZX5KvD6hRR3SNQ5rKXQzp00lsUu5lUpllq28,7272
57
- deltacat/compute/compactor/steps/rehash/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
58
- deltacat/compute/compactor/steps/rehash/rehash_bucket.py,sha256=h5w-lNTiaufIO_gbMVnCn3vJ4s9RdT9_NzkeyX9aIQg,1794
59
- deltacat/compute/compactor/steps/rehash/rewrite_index.py,sha256=n743Xo1j_eUl22_AFEhTfrCzYSydwpXFaFCfruJr4Kk,1869
60
- deltacat/compute/compactor/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
61
- deltacat/compute/compactor/utils/io.py,sha256=1E0MG_mxk_RGzl4I5xYUSuyWXAaOuWkBSxOX4khx0LY,9678
62
- deltacat/compute/compactor/utils/primary_key_index.py,sha256=H2odhlzplrgL1LcwelaRoVNFHNy3p_1NgFW0dfaSesw,10864
63
- deltacat/compute/compactor/utils/round_completion_file.py,sha256=uvf-pyIQCVmfpM0Ew1qjZ7OVK4-jJifuu-u_dverEPw,2116
64
- deltacat/compute/compactor/utils/system_columns.py,sha256=1M4JHwqDIRnfQaqvPz8_zBw05de45Db06To-3wiUVUk,7003
65
- deltacat/compute/metastats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
66
- deltacat/compute/metastats/meta_stats.py,sha256=kCRWfQ2zDVrjfIvhE1S6B0ShRPn5bRQA-z8r8VE2Oco,20588
67
- deltacat/compute/metastats/stats.py,sha256=LI9ndlwfAs7Ad-MsKaBeGL92JieRh8a54RxwXfRqDCU,7375
68
- deltacat/compute/metastats/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
69
- deltacat/compute/metastats/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
70
- deltacat/compute/metastats/model/partition_stats_dict.py,sha256=o7BlH-Eyv9RzesLTw7OoTKWqkHFI9bE0ZuefjTr3viA,1405
71
- deltacat/compute/metastats/model/stats_cluster_size_estimator.py,sha256=3965WMAHQAzUcQSnK3UaW-ULGIC7qibaz0frl7h0DdI,2766
72
- deltacat/compute/metastats/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
73
- deltacat/compute/metastats/utils/constants.py,sha256=bFUPKmR3FkjEnwpHuToQYZ9QcHqYpd4OMMSwVwnJcaA,869
74
- deltacat/compute/metastats/utils/io.py,sha256=NBEu4wkS8wNUYXULHXWGq1fEscmEneNNlJgHUgtNxnk,8531
75
- deltacat/compute/metastats/utils/pyarrow_memory_estimation_function.py,sha256=BYdnNZywoEoZIcbikuWYU97Ox3BRdPJExNUL10QnPkY,1135
76
- deltacat/compute/metastats/utils/ray_utils.py,sha256=-SDouyx5O0Qt4ZiGFmU0lGHvWZoXO3w5BS_Vx39gJa4,4591
77
- deltacat/compute/stats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
78
- deltacat/compute/stats/basic.py,sha256=fL5UgyHKeUdQqzr1lXstbHmWSIvawTTXk5pRmhyH-HQ,9415
79
- deltacat/compute/stats/types.py,sha256=cp0lT8nITTKbnkc03OysRjXfcfXzQml9a4wqCnR6kqs,215
80
- deltacat/compute/stats/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
81
- deltacat/compute/stats/models/delta_column_stats.py,sha256=_m-fGbx2njONr9N2ihkIajUzVvmy8weDG5BLJdMaoQU,3296
82
- deltacat/compute/stats/models/delta_stats.py,sha256=yzxum2boP5c4cTSNaLOSgdioQ4ZPtSoMX_Ju3k-4MY4,8711
83
- deltacat/compute/stats/models/delta_stats_cache_result.py,sha256=ga_by81TNDpptjYYUzpQHVRCR95_NnZ1a59apmvVBUo,1609
84
- deltacat/compute/stats/models/manifest_entry_stats.py,sha256=vGTOq_a1XFFf3N_rcB4QXTHuEx-T3P5KwJllrKi7zuw,2363
85
- deltacat/compute/stats/models/stats_result.py,sha256=j3bjU8LdLbTkpM4ZZojQ70WfqHsXFNftJG2PwKReSRg,3753
86
- deltacat/compute/stats/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
87
- deltacat/compute/stats/utils/intervals.py,sha256=PmbE1856k_y0p6lNrQ3rNC_RBT8PROA2wQRFVM2s8P8,3049
88
- deltacat/compute/stats/utils/io.py,sha256=H-xIrWOqN4JSDuo3imrKDEyU_xDrfKU2tFwdOg7Jne4,8587
89
- deltacat/compute/stats/utils/manifest_stats_file.py,sha256=rc3BdQ-QjgRSz6vBGw_x3shShnkNVeVVMEzs8n9Ir6A,3811
90
- deltacat/io/__init__.py,sha256=4fQUUNSj8pnpFw4oF6CuDbhizj5JU0Jml2qQd-lVUvA,153
91
- deltacat/io/dataset.py,sha256=DG1TfhAX1dWYVcUAryOgLyqgDQ71syVRvkFxPS0BJjQ,3821
92
- deltacat/io/read_api.py,sha256=kkcclEshS5MEEI6gzJ0HkcIug2ph_DE8aUdBddUaRPU,7109
93
- deltacat/io/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
94
- deltacat/io/aws/redshift/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
95
- deltacat/io/aws/redshift/redshift_datasource.py,sha256=BWlQoAVnGi8Urj5rF5ZOePWy0O-xxOgJIwLt-f-s6BM,22765
96
- deltacat/storage/__init__.py,sha256=doAx2Ojk8BJik80crGrlX0rph8HKk2Cm5JJ6HqfP5qc,1337
97
- deltacat/storage/interface.py,sha256=d2b3Y1caGuqS-QaSpaCw6gnAkSCZgQfGEWKuVIocHIE,22080
98
- deltacat/storage/model/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
99
- deltacat/storage/model/delta.py,sha256=1fXuIOq_S4u164fVnFe58jVKR5NTARP2bWqAZOkEpOY,13912
100
- deltacat/storage/model/list_result.py,sha256=zGa32B3c98qtxRJq376h3eHQmQLulhhOna0G8psPsJY,2445
101
- deltacat/storage/model/locator.py,sha256=1S7szmDSx-R4Z3arFNILOvS4t7dF7_rJNV9fHyRc3G4,1296
102
- deltacat/storage/model/namespace.py,sha256=pvz4NNU5W4l8LbLGJPbmPb9uwL6rlTr64yAtwJ7GHVs,2108
103
- deltacat/storage/model/partition.py,sha256=XcYxzfW3adTJYB8ZkeirCdZ41UhoBH8CojZ84KUJo40,11165
104
- deltacat/storage/model/stream.py,sha256=gLzpscjeU4EHGowXDyJMqstN64v57FzGlyDdZJTwuwQ,7818
105
- deltacat/storage/model/table.py,sha256=zTlM1B2R7fpN5F0ktA_KpE2DlmQBSn8iNOpLI9PP65w,4252
106
- deltacat/storage/model/table_version.py,sha256=qDCafR53oMBDUMcg6sxkGOFZdWyf7oL1sew5VKobIss,7086
107
- deltacat/storage/model/types.py,sha256=u2yCTqxzoQz8pcrzr8uD0xXP35kPisQKcbDgzac1YQ8,1579
108
- deltacat/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
109
- deltacat/tests/stats/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
110
- deltacat/tests/stats/test_intervals.py,sha256=NhKBqTgTRcVPqrExlJNrVHyToeaA9dX01UwJsKzQZMM,2137
111
- deltacat/types/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
112
- deltacat/types/media.py,sha256=S0cYjbcRSN8I9EewZvmezYTWbIy01Sa2pLl67hMThwE,2186
113
- deltacat/types/tables.py,sha256=KCXmpv-hVRbsySGDaFM1gTsygociRx0TkrxLY1vNNNc,3814
114
- deltacat/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
115
- deltacat/utils/common.py,sha256=0yenkHroJiQjeenXZnNSDSD06-KJuBIsw2GAMqqQSKI,1449
116
- deltacat/utils/numpy.py,sha256=D1dCa7xTOlZXzxDhaETzl1c8zJNxGpBr_-_Cb5yzXAI,2094
117
- deltacat/utils/pandas.py,sha256=SBEnH2eQgjrQkdW6Dvk3Ky3reTI0xYRGFE2JIOsZqaA,10067
118
- deltacat/utils/performance.py,sha256=GsHuRG8Zi6iVmIC9F4fXUDbEFk03fErVJy8JjYCWsUE,670
119
- deltacat/utils/placement.py,sha256=knanJlEMSGMA3t-UISsLTUP-fQ9CzzR1LZO29L2HfB4,8823
120
- deltacat/utils/pyarrow.py,sha256=mc93BdFip-0aLdLuQtNXlf7LoFX2k5txdOhWYehq1-g,10611
121
- deltacat/utils/ray_utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
122
- deltacat/utils/ray_utils/collections.py,sha256=2B90lyLulxS7tBMkyzLAVC1fIeqYspxuaVBeXGveYlE,1950
123
- deltacat/utils/ray_utils/concurrency.py,sha256=2L6VVQrjTN1DTNBL8B4xy5Ty6lmO6G_M8vN9TDJuLrA,4906
124
- deltacat/utils/ray_utils/dataset.py,sha256=JPonxxxmDuAUiYqmBBTM2hyXHPggNGgp3QvsUxSjvNI,3319
125
- deltacat/utils/ray_utils/performance.py,sha256=v0JHrjlTGw4-ypzy_ZBlY4mOQN9o9euQiwc6IWk7-bY,515
126
- deltacat/utils/ray_utils/runtime.py,sha256=8Z7WlIYylij93vhhwUf3bijrpGKCEpEKlXVI5KCC1ps,4810
127
- deltacat-0.1.8.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
128
- deltacat-0.1.8.dist-info/METADATA,sha256=s9S-3IUuKC_fjg5w9M3ODpRaQvy_YdVRePQo-RyJAvM,1414
129
- deltacat-0.1.8.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
130
- deltacat-0.1.8.dist-info/top_level.txt,sha256=RWdIcid4Bv2i2ozLVh-70kJpyB61xEKXod9XXGpiono,9
131
- deltacat-0.1.8.dist-info/RECORD,,
File without changes