awx-zipline-ai 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- agent/__init__.py +1 -0
- agent/constants.py +15 -0
- agent/ttypes.py +1684 -0
- ai/__init__.py +0 -0
- ai/chronon/__init__.py +0 -0
- ai/chronon/airflow_helpers.py +251 -0
- ai/chronon/api/__init__.py +1 -0
- ai/chronon/api/common/__init__.py +1 -0
- ai/chronon/api/common/constants.py +15 -0
- ai/chronon/api/common/ttypes.py +1844 -0
- ai/chronon/api/constants.py +15 -0
- ai/chronon/api/ttypes.py +3624 -0
- ai/chronon/cli/compile/column_hashing.py +313 -0
- ai/chronon/cli/compile/compile_context.py +177 -0
- ai/chronon/cli/compile/compiler.py +160 -0
- ai/chronon/cli/compile/conf_validator.py +590 -0
- ai/chronon/cli/compile/display/class_tracker.py +112 -0
- ai/chronon/cli/compile/display/compile_status.py +95 -0
- ai/chronon/cli/compile/display/compiled_obj.py +12 -0
- ai/chronon/cli/compile/display/console.py +3 -0
- ai/chronon/cli/compile/display/diff_result.py +46 -0
- ai/chronon/cli/compile/fill_templates.py +40 -0
- ai/chronon/cli/compile/parse_configs.py +141 -0
- ai/chronon/cli/compile/parse_teams.py +238 -0
- ai/chronon/cli/compile/serializer.py +115 -0
- ai/chronon/cli/git_utils.py +156 -0
- ai/chronon/cli/logger.py +61 -0
- ai/chronon/constants.py +3 -0
- ai/chronon/eval/__init__.py +122 -0
- ai/chronon/eval/query_parsing.py +19 -0
- ai/chronon/eval/sample_tables.py +100 -0
- ai/chronon/eval/table_scan.py +186 -0
- ai/chronon/fetcher/__init__.py +1 -0
- ai/chronon/fetcher/constants.py +15 -0
- ai/chronon/fetcher/ttypes.py +127 -0
- ai/chronon/group_by.py +692 -0
- ai/chronon/hub/__init__.py +1 -0
- ai/chronon/hub/constants.py +15 -0
- ai/chronon/hub/ttypes.py +1228 -0
- ai/chronon/join.py +566 -0
- ai/chronon/logger.py +24 -0
- ai/chronon/model.py +35 -0
- ai/chronon/observability/__init__.py +1 -0
- ai/chronon/observability/constants.py +15 -0
- ai/chronon/observability/ttypes.py +2192 -0
- ai/chronon/orchestration/__init__.py +1 -0
- ai/chronon/orchestration/constants.py +15 -0
- ai/chronon/orchestration/ttypes.py +4406 -0
- ai/chronon/planner/__init__.py +1 -0
- ai/chronon/planner/constants.py +15 -0
- ai/chronon/planner/ttypes.py +1686 -0
- ai/chronon/query.py +126 -0
- ai/chronon/repo/__init__.py +40 -0
- ai/chronon/repo/aws.py +298 -0
- ai/chronon/repo/cluster.py +65 -0
- ai/chronon/repo/compile.py +56 -0
- ai/chronon/repo/constants.py +164 -0
- ai/chronon/repo/default_runner.py +291 -0
- ai/chronon/repo/explore.py +421 -0
- ai/chronon/repo/extract_objects.py +137 -0
- ai/chronon/repo/gcp.py +585 -0
- ai/chronon/repo/gitpython_utils.py +14 -0
- ai/chronon/repo/hub_runner.py +171 -0
- ai/chronon/repo/hub_uploader.py +108 -0
- ai/chronon/repo/init.py +53 -0
- ai/chronon/repo/join_backfill.py +105 -0
- ai/chronon/repo/run.py +293 -0
- ai/chronon/repo/serializer.py +141 -0
- ai/chronon/repo/team_json_utils.py +46 -0
- ai/chronon/repo/utils.py +472 -0
- ai/chronon/repo/zipline.py +51 -0
- ai/chronon/repo/zipline_hub.py +105 -0
- ai/chronon/resources/gcp/README.md +174 -0
- ai/chronon/resources/gcp/group_bys/test/__init__.py +0 -0
- ai/chronon/resources/gcp/group_bys/test/data.py +34 -0
- ai/chronon/resources/gcp/joins/test/__init__.py +0 -0
- ai/chronon/resources/gcp/joins/test/data.py +30 -0
- ai/chronon/resources/gcp/sources/test/__init__.py +0 -0
- ai/chronon/resources/gcp/sources/test/data.py +23 -0
- ai/chronon/resources/gcp/teams.py +70 -0
- ai/chronon/resources/gcp/zipline-cli-install.sh +54 -0
- ai/chronon/source.py +88 -0
- ai/chronon/staging_query.py +185 -0
- ai/chronon/types.py +57 -0
- ai/chronon/utils.py +557 -0
- ai/chronon/windows.py +50 -0
- awx_zipline_ai-0.2.0.dist-info/METADATA +173 -0
- awx_zipline_ai-0.2.0.dist-info/RECORD +93 -0
- awx_zipline_ai-0.2.0.dist-info/WHEEL +5 -0
- awx_zipline_ai-0.2.0.dist-info/entry_points.txt +2 -0
- awx_zipline_ai-0.2.0.dist-info/licenses/LICENSE +202 -0
- awx_zipline_ai-0.2.0.dist-info/top_level.txt +3 -0
- jars/__init__.py +0 -0
ai/chronon/api/ttypes.py
ADDED
|
@@ -0,0 +1,3624 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Autogenerated by Thrift Compiler (0.22.0)
|
|
3
|
+
#
|
|
4
|
+
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
|
|
5
|
+
#
|
|
6
|
+
# options string: py
|
|
7
|
+
#
|
|
8
|
+
|
|
9
|
+
from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException
|
|
10
|
+
from thrift.protocol.TProtocol import TProtocolException
|
|
11
|
+
from thrift.TRecursive import fix_spec
|
|
12
|
+
from uuid import UUID
|
|
13
|
+
|
|
14
|
+
import sys
|
|
15
|
+
import ai.chronon.api.common.ttypes
|
|
16
|
+
import ai.chronon.observability.ttypes
|
|
17
|
+
|
|
18
|
+
from thrift.transport import TTransport
|
|
19
|
+
all_structs = []
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class Operation(object):
|
|
23
|
+
MIN = 0
|
|
24
|
+
MAX = 1
|
|
25
|
+
FIRST = 2
|
|
26
|
+
LAST = 3
|
|
27
|
+
UNIQUE_COUNT = 4
|
|
28
|
+
APPROX_UNIQUE_COUNT = 5
|
|
29
|
+
COUNT = 6
|
|
30
|
+
SUM = 7
|
|
31
|
+
AVERAGE = 8
|
|
32
|
+
VARIANCE = 9
|
|
33
|
+
SKEW = 10
|
|
34
|
+
KURTOSIS = 11
|
|
35
|
+
APPROX_PERCENTILE = 12
|
|
36
|
+
LAST_K = 13
|
|
37
|
+
FIRST_K = 14
|
|
38
|
+
TOP_K = 15
|
|
39
|
+
BOTTOM_K = 16
|
|
40
|
+
HISTOGRAM = 17
|
|
41
|
+
APPROX_FREQUENT_K = 18
|
|
42
|
+
APPROX_HEAVY_HITTERS_K = 19
|
|
43
|
+
UNIQUE_TOP_K = 20
|
|
44
|
+
|
|
45
|
+
_VALUES_TO_NAMES = {
|
|
46
|
+
0: "MIN",
|
|
47
|
+
1: "MAX",
|
|
48
|
+
2: "FIRST",
|
|
49
|
+
3: "LAST",
|
|
50
|
+
4: "UNIQUE_COUNT",
|
|
51
|
+
5: "APPROX_UNIQUE_COUNT",
|
|
52
|
+
6: "COUNT",
|
|
53
|
+
7: "SUM",
|
|
54
|
+
8: "AVERAGE",
|
|
55
|
+
9: "VARIANCE",
|
|
56
|
+
10: "SKEW",
|
|
57
|
+
11: "KURTOSIS",
|
|
58
|
+
12: "APPROX_PERCENTILE",
|
|
59
|
+
13: "LAST_K",
|
|
60
|
+
14: "FIRST_K",
|
|
61
|
+
15: "TOP_K",
|
|
62
|
+
16: "BOTTOM_K",
|
|
63
|
+
17: "HISTOGRAM",
|
|
64
|
+
18: "APPROX_FREQUENT_K",
|
|
65
|
+
19: "APPROX_HEAVY_HITTERS_K",
|
|
66
|
+
20: "UNIQUE_TOP_K",
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
_NAMES_TO_VALUES = {
|
|
70
|
+
"MIN": 0,
|
|
71
|
+
"MAX": 1,
|
|
72
|
+
"FIRST": 2,
|
|
73
|
+
"LAST": 3,
|
|
74
|
+
"UNIQUE_COUNT": 4,
|
|
75
|
+
"APPROX_UNIQUE_COUNT": 5,
|
|
76
|
+
"COUNT": 6,
|
|
77
|
+
"SUM": 7,
|
|
78
|
+
"AVERAGE": 8,
|
|
79
|
+
"VARIANCE": 9,
|
|
80
|
+
"SKEW": 10,
|
|
81
|
+
"KURTOSIS": 11,
|
|
82
|
+
"APPROX_PERCENTILE": 12,
|
|
83
|
+
"LAST_K": 13,
|
|
84
|
+
"FIRST_K": 14,
|
|
85
|
+
"TOP_K": 15,
|
|
86
|
+
"BOTTOM_K": 16,
|
|
87
|
+
"HISTOGRAM": 17,
|
|
88
|
+
"APPROX_FREQUENT_K": 18,
|
|
89
|
+
"APPROX_HEAVY_HITTERS_K": 19,
|
|
90
|
+
"UNIQUE_TOP_K": 20,
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
class Accuracy(object):
|
|
95
|
+
TEMPORAL = 0
|
|
96
|
+
SNAPSHOT = 1
|
|
97
|
+
|
|
98
|
+
_VALUES_TO_NAMES = {
|
|
99
|
+
0: "TEMPORAL",
|
|
100
|
+
1: "SNAPSHOT",
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
_NAMES_TO_VALUES = {
|
|
104
|
+
"TEMPORAL": 0,
|
|
105
|
+
"SNAPSHOT": 1,
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
class EngineType(object):
|
|
110
|
+
SPARK = 0
|
|
111
|
+
BIGQUERY = 1
|
|
112
|
+
|
|
113
|
+
_VALUES_TO_NAMES = {
|
|
114
|
+
0: "SPARK",
|
|
115
|
+
1: "BIGQUERY",
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
_NAMES_TO_VALUES = {
|
|
119
|
+
"SPARK": 0,
|
|
120
|
+
"BIGQUERY": 1,
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class DataKind(object):
|
|
125
|
+
BOOLEAN = 0
|
|
126
|
+
BYTE = 1
|
|
127
|
+
SHORT = 2
|
|
128
|
+
INT = 3
|
|
129
|
+
LONG = 4
|
|
130
|
+
FLOAT = 5
|
|
131
|
+
DOUBLE = 6
|
|
132
|
+
STRING = 7
|
|
133
|
+
BINARY = 8
|
|
134
|
+
DATE = 9
|
|
135
|
+
TIMESTAMP = 10
|
|
136
|
+
MAP = 11
|
|
137
|
+
LIST = 12
|
|
138
|
+
STRUCT = 13
|
|
139
|
+
|
|
140
|
+
_VALUES_TO_NAMES = {
|
|
141
|
+
0: "BOOLEAN",
|
|
142
|
+
1: "BYTE",
|
|
143
|
+
2: "SHORT",
|
|
144
|
+
3: "INT",
|
|
145
|
+
4: "LONG",
|
|
146
|
+
5: "FLOAT",
|
|
147
|
+
6: "DOUBLE",
|
|
148
|
+
7: "STRING",
|
|
149
|
+
8: "BINARY",
|
|
150
|
+
9: "DATE",
|
|
151
|
+
10: "TIMESTAMP",
|
|
152
|
+
11: "MAP",
|
|
153
|
+
12: "LIST",
|
|
154
|
+
13: "STRUCT",
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
_NAMES_TO_VALUES = {
|
|
158
|
+
"BOOLEAN": 0,
|
|
159
|
+
"BYTE": 1,
|
|
160
|
+
"SHORT": 2,
|
|
161
|
+
"INT": 3,
|
|
162
|
+
"LONG": 4,
|
|
163
|
+
"FLOAT": 5,
|
|
164
|
+
"DOUBLE": 6,
|
|
165
|
+
"STRING": 7,
|
|
166
|
+
"BINARY": 8,
|
|
167
|
+
"DATE": 9,
|
|
168
|
+
"TIMESTAMP": 10,
|
|
169
|
+
"MAP": 11,
|
|
170
|
+
"LIST": 12,
|
|
171
|
+
"STRUCT": 13,
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
class ModelType(object):
|
|
176
|
+
XGBoost = 0
|
|
177
|
+
PyTorch = 1
|
|
178
|
+
TensorFlow = 2
|
|
179
|
+
ScikitLearn = 3
|
|
180
|
+
LightGBM = 4
|
|
181
|
+
Other = 100
|
|
182
|
+
|
|
183
|
+
_VALUES_TO_NAMES = {
|
|
184
|
+
0: "XGBoost",
|
|
185
|
+
1: "PyTorch",
|
|
186
|
+
2: "TensorFlow",
|
|
187
|
+
3: "ScikitLearn",
|
|
188
|
+
4: "LightGBM",
|
|
189
|
+
100: "Other",
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
_NAMES_TO_VALUES = {
|
|
193
|
+
"XGBoost": 0,
|
|
194
|
+
"PyTorch": 1,
|
|
195
|
+
"TensorFlow": 2,
|
|
196
|
+
"ScikitLearn": 3,
|
|
197
|
+
"LightGBM": 4,
|
|
198
|
+
"Other": 100,
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
class DataModel(object):
|
|
203
|
+
ENTITIES = 0
|
|
204
|
+
EVENTS = 1
|
|
205
|
+
|
|
206
|
+
_VALUES_TO_NAMES = {
|
|
207
|
+
0: "ENTITIES",
|
|
208
|
+
1: "EVENTS",
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
_NAMES_TO_VALUES = {
|
|
212
|
+
"ENTITIES": 0,
|
|
213
|
+
"EVENTS": 1,
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class Query(object):
|
|
218
|
+
"""
|
|
219
|
+
Attributes:
|
|
220
|
+
- selects
|
|
221
|
+
- wheres
|
|
222
|
+
- startPartition
|
|
223
|
+
- endPartition
|
|
224
|
+
- timeColumn
|
|
225
|
+
- setups
|
|
226
|
+
- mutationTimeColumn
|
|
227
|
+
- reversalColumn
|
|
228
|
+
- partitionColumn: Chronon expects all its batch input data to be date/time partitioned.
|
|
229
|
+
We in-turn produce partitioned outputs.
|
|
230
|
+
|
|
231
|
+
- partitionFormat: Partition format in the java DateFormatter spec:
|
|
232
|
+
see: https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html
|
|
233
|
+
|
|
234
|
+
- partitionInterval: Indicates the timespan of a given interval of the source
|
|
235
|
+
|
|
236
|
+
- partitionLag: Indicates how often this source is typically delayed by.
|
|
237
|
+
Should be a multiple of partitionInterval
|
|
238
|
+
|
|
239
|
+
- subPartitionsToWaitFor: Additional partitions to be used in sensing that the source data has landed.
|
|
240
|
+
Should be a full partition string, such as `hr=23:00'
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
"""
|
|
244
|
+
thrift_spec = None
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def __init__(self, selects = None, wheres = None, startPartition = None, endPartition = None, timeColumn = None, setups = None, mutationTimeColumn = None, reversalColumn = None, partitionColumn = None, partitionFormat = None, partitionInterval = None, partitionLag = None, subPartitionsToWaitFor = None,):
|
|
248
|
+
self.selects = selects
|
|
249
|
+
self.wheres = wheres
|
|
250
|
+
self.startPartition = startPartition
|
|
251
|
+
self.endPartition = endPartition
|
|
252
|
+
self.timeColumn = timeColumn
|
|
253
|
+
self.setups = setups
|
|
254
|
+
self.mutationTimeColumn = mutationTimeColumn
|
|
255
|
+
self.reversalColumn = reversalColumn
|
|
256
|
+
self.partitionColumn = partitionColumn
|
|
257
|
+
self.partitionFormat = partitionFormat
|
|
258
|
+
self.partitionInterval = partitionInterval
|
|
259
|
+
self.partitionLag = partitionLag
|
|
260
|
+
self.subPartitionsToWaitFor = subPartitionsToWaitFor
|
|
261
|
+
|
|
262
|
+
def read(self, iprot):
|
|
263
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
264
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
265
|
+
return
|
|
266
|
+
iprot.readStructBegin()
|
|
267
|
+
while True:
|
|
268
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
269
|
+
if ftype == TType.STOP:
|
|
270
|
+
break
|
|
271
|
+
if fid == 1:
|
|
272
|
+
if ftype == TType.MAP:
|
|
273
|
+
self.selects = {}
|
|
274
|
+
(_ktype1, _vtype2, _size0) = iprot.readMapBegin()
|
|
275
|
+
for _i4 in range(_size0):
|
|
276
|
+
_key5 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
277
|
+
_val6 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
278
|
+
self.selects[_key5] = _val6
|
|
279
|
+
iprot.readMapEnd()
|
|
280
|
+
else:
|
|
281
|
+
iprot.skip(ftype)
|
|
282
|
+
elif fid == 2:
|
|
283
|
+
if ftype == TType.LIST:
|
|
284
|
+
self.wheres = []
|
|
285
|
+
(_etype10, _size7) = iprot.readListBegin()
|
|
286
|
+
for _i11 in range(_size7):
|
|
287
|
+
_elem12 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
288
|
+
self.wheres.append(_elem12)
|
|
289
|
+
iprot.readListEnd()
|
|
290
|
+
else:
|
|
291
|
+
iprot.skip(ftype)
|
|
292
|
+
elif fid == 3:
|
|
293
|
+
if ftype == TType.STRING:
|
|
294
|
+
self.startPartition = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
295
|
+
else:
|
|
296
|
+
iprot.skip(ftype)
|
|
297
|
+
elif fid == 4:
|
|
298
|
+
if ftype == TType.STRING:
|
|
299
|
+
self.endPartition = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
300
|
+
else:
|
|
301
|
+
iprot.skip(ftype)
|
|
302
|
+
elif fid == 5:
|
|
303
|
+
if ftype == TType.STRING:
|
|
304
|
+
self.timeColumn = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
305
|
+
else:
|
|
306
|
+
iprot.skip(ftype)
|
|
307
|
+
elif fid == 6:
|
|
308
|
+
if ftype == TType.LIST:
|
|
309
|
+
self.setups = []
|
|
310
|
+
(_etype16, _size13) = iprot.readListBegin()
|
|
311
|
+
for _i17 in range(_size13):
|
|
312
|
+
_elem18 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
313
|
+
self.setups.append(_elem18)
|
|
314
|
+
iprot.readListEnd()
|
|
315
|
+
else:
|
|
316
|
+
iprot.skip(ftype)
|
|
317
|
+
elif fid == 7:
|
|
318
|
+
if ftype == TType.STRING:
|
|
319
|
+
self.mutationTimeColumn = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
320
|
+
else:
|
|
321
|
+
iprot.skip(ftype)
|
|
322
|
+
elif fid == 8:
|
|
323
|
+
if ftype == TType.STRING:
|
|
324
|
+
self.reversalColumn = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
325
|
+
else:
|
|
326
|
+
iprot.skip(ftype)
|
|
327
|
+
elif fid == 20:
|
|
328
|
+
if ftype == TType.STRING:
|
|
329
|
+
self.partitionColumn = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
330
|
+
else:
|
|
331
|
+
iprot.skip(ftype)
|
|
332
|
+
elif fid == 21:
|
|
333
|
+
if ftype == TType.STRING:
|
|
334
|
+
self.partitionFormat = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
335
|
+
else:
|
|
336
|
+
iprot.skip(ftype)
|
|
337
|
+
elif fid == 22:
|
|
338
|
+
if ftype == TType.STRUCT:
|
|
339
|
+
self.partitionInterval = ai.chronon.api.common.ttypes.Window()
|
|
340
|
+
self.partitionInterval.read(iprot)
|
|
341
|
+
else:
|
|
342
|
+
iprot.skip(ftype)
|
|
343
|
+
elif fid == 23:
|
|
344
|
+
if ftype == TType.STRUCT:
|
|
345
|
+
self.partitionLag = ai.chronon.api.common.ttypes.Window()
|
|
346
|
+
self.partitionLag.read(iprot)
|
|
347
|
+
else:
|
|
348
|
+
iprot.skip(ftype)
|
|
349
|
+
elif fid == 24:
|
|
350
|
+
if ftype == TType.LIST:
|
|
351
|
+
self.subPartitionsToWaitFor = []
|
|
352
|
+
(_etype22, _size19) = iprot.readListBegin()
|
|
353
|
+
for _i23 in range(_size19):
|
|
354
|
+
_elem24 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
355
|
+
self.subPartitionsToWaitFor.append(_elem24)
|
|
356
|
+
iprot.readListEnd()
|
|
357
|
+
else:
|
|
358
|
+
iprot.skip(ftype)
|
|
359
|
+
else:
|
|
360
|
+
iprot.skip(ftype)
|
|
361
|
+
iprot.readFieldEnd()
|
|
362
|
+
iprot.readStructEnd()
|
|
363
|
+
|
|
364
|
+
def write(self, oprot):
|
|
365
|
+
self.validate()
|
|
366
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
367
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
368
|
+
return
|
|
369
|
+
oprot.writeStructBegin('Query')
|
|
370
|
+
if self.selects is not None:
|
|
371
|
+
oprot.writeFieldBegin('selects', TType.MAP, 1)
|
|
372
|
+
oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.selects))
|
|
373
|
+
for kiter25, viter26 in self.selects.items():
|
|
374
|
+
oprot.writeString(kiter25.encode('utf-8') if sys.version_info[0] == 2 else kiter25)
|
|
375
|
+
oprot.writeString(viter26.encode('utf-8') if sys.version_info[0] == 2 else viter26)
|
|
376
|
+
oprot.writeMapEnd()
|
|
377
|
+
oprot.writeFieldEnd()
|
|
378
|
+
if self.wheres is not None:
|
|
379
|
+
oprot.writeFieldBegin('wheres', TType.LIST, 2)
|
|
380
|
+
oprot.writeListBegin(TType.STRING, len(self.wheres))
|
|
381
|
+
for iter27 in self.wheres:
|
|
382
|
+
oprot.writeString(iter27.encode('utf-8') if sys.version_info[0] == 2 else iter27)
|
|
383
|
+
oprot.writeListEnd()
|
|
384
|
+
oprot.writeFieldEnd()
|
|
385
|
+
if self.startPartition is not None:
|
|
386
|
+
oprot.writeFieldBegin('startPartition', TType.STRING, 3)
|
|
387
|
+
oprot.writeString(self.startPartition.encode('utf-8') if sys.version_info[0] == 2 else self.startPartition)
|
|
388
|
+
oprot.writeFieldEnd()
|
|
389
|
+
if self.endPartition is not None:
|
|
390
|
+
oprot.writeFieldBegin('endPartition', TType.STRING, 4)
|
|
391
|
+
oprot.writeString(self.endPartition.encode('utf-8') if sys.version_info[0] == 2 else self.endPartition)
|
|
392
|
+
oprot.writeFieldEnd()
|
|
393
|
+
if self.timeColumn is not None:
|
|
394
|
+
oprot.writeFieldBegin('timeColumn', TType.STRING, 5)
|
|
395
|
+
oprot.writeString(self.timeColumn.encode('utf-8') if sys.version_info[0] == 2 else self.timeColumn)
|
|
396
|
+
oprot.writeFieldEnd()
|
|
397
|
+
if self.setups is not None:
|
|
398
|
+
oprot.writeFieldBegin('setups', TType.LIST, 6)
|
|
399
|
+
oprot.writeListBegin(TType.STRING, len(self.setups))
|
|
400
|
+
for iter28 in self.setups:
|
|
401
|
+
oprot.writeString(iter28.encode('utf-8') if sys.version_info[0] == 2 else iter28)
|
|
402
|
+
oprot.writeListEnd()
|
|
403
|
+
oprot.writeFieldEnd()
|
|
404
|
+
if self.mutationTimeColumn is not None:
|
|
405
|
+
oprot.writeFieldBegin('mutationTimeColumn', TType.STRING, 7)
|
|
406
|
+
oprot.writeString(self.mutationTimeColumn.encode('utf-8') if sys.version_info[0] == 2 else self.mutationTimeColumn)
|
|
407
|
+
oprot.writeFieldEnd()
|
|
408
|
+
if self.reversalColumn is not None:
|
|
409
|
+
oprot.writeFieldBegin('reversalColumn', TType.STRING, 8)
|
|
410
|
+
oprot.writeString(self.reversalColumn.encode('utf-8') if sys.version_info[0] == 2 else self.reversalColumn)
|
|
411
|
+
oprot.writeFieldEnd()
|
|
412
|
+
if self.partitionColumn is not None:
|
|
413
|
+
oprot.writeFieldBegin('partitionColumn', TType.STRING, 20)
|
|
414
|
+
oprot.writeString(self.partitionColumn.encode('utf-8') if sys.version_info[0] == 2 else self.partitionColumn)
|
|
415
|
+
oprot.writeFieldEnd()
|
|
416
|
+
if self.partitionFormat is not None:
|
|
417
|
+
oprot.writeFieldBegin('partitionFormat', TType.STRING, 21)
|
|
418
|
+
oprot.writeString(self.partitionFormat.encode('utf-8') if sys.version_info[0] == 2 else self.partitionFormat)
|
|
419
|
+
oprot.writeFieldEnd()
|
|
420
|
+
if self.partitionInterval is not None:
|
|
421
|
+
oprot.writeFieldBegin('partitionInterval', TType.STRUCT, 22)
|
|
422
|
+
self.partitionInterval.write(oprot)
|
|
423
|
+
oprot.writeFieldEnd()
|
|
424
|
+
if self.partitionLag is not None:
|
|
425
|
+
oprot.writeFieldBegin('partitionLag', TType.STRUCT, 23)
|
|
426
|
+
self.partitionLag.write(oprot)
|
|
427
|
+
oprot.writeFieldEnd()
|
|
428
|
+
if self.subPartitionsToWaitFor is not None:
|
|
429
|
+
oprot.writeFieldBegin('subPartitionsToWaitFor', TType.LIST, 24)
|
|
430
|
+
oprot.writeListBegin(TType.STRING, len(self.subPartitionsToWaitFor))
|
|
431
|
+
for iter29 in self.subPartitionsToWaitFor:
|
|
432
|
+
oprot.writeString(iter29.encode('utf-8') if sys.version_info[0] == 2 else iter29)
|
|
433
|
+
oprot.writeListEnd()
|
|
434
|
+
oprot.writeFieldEnd()
|
|
435
|
+
oprot.writeFieldStop()
|
|
436
|
+
oprot.writeStructEnd()
|
|
437
|
+
|
|
438
|
+
def validate(self):
|
|
439
|
+
return
|
|
440
|
+
|
|
441
|
+
def __repr__(self):
|
|
442
|
+
L = ['%s=%r' % (key, value)
|
|
443
|
+
for key, value in self.__dict__.items()]
|
|
444
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
445
|
+
|
|
446
|
+
def __eq__(self, other):
|
|
447
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
448
|
+
|
|
449
|
+
def __ne__(self, other):
|
|
450
|
+
return not (self == other)
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
class StagingQuery(object):
|
|
454
|
+
"""
|
|
455
|
+
Staging Query encapsulates arbitrary spark computation. One key feature is that the computation follows a
|
|
456
|
+
"fill-what's-missing" pattern. Basically instead of explicitly specifying dates you specify two macros.
|
|
457
|
+
`{{ start_date }}` and `{{end_date}}`. Chronon will pass in earliest-missing-partition for `start_date` and
|
|
458
|
+
execution-date / today for `end_date`. So the query will compute multiple partitions at once.
|
|
459
|
+
|
|
460
|
+
Attributes:
|
|
461
|
+
- metaData: Contains name, team, output_namespace, execution parameters etc. Things that don't change the semantics of the computation itself.
|
|
462
|
+
|
|
463
|
+
- query: Arbitrary spark query that should be written with `{{ start_date }}`, `{{ end_date }}` and `{{ latest_date }}` templates
|
|
464
|
+
- `{{ start_date }}` will be set to this user provided start date, future incremental runs will set it to the latest existing partition + 1 day.
|
|
465
|
+
- `{{ start_date(offset=-10, lower_bound='2023-01-01', upper_bound='2024-01-01') }}` will shift the date back one day and bound it with lower and upper bounds.
|
|
466
|
+
- `{{ end_date }}` is the end partition of the computing range. offsetting and bounding the end_date also works as described above.
|
|
467
|
+
- `{{ latest_date }}` is the end partition independent of the computing range (meant for cumulative sources). offsetting and bounding the end_date also works as described above.
|
|
468
|
+
- `{{ max_date(table=namespace.my_table) }}` is the max partition available for a given table.
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
- startPartition: on the first run, `{{ start_date }}` will be set to this user provided start date, future incremental runs will set it to the latest existing partition + 1 day.
|
|
472
|
+
|
|
473
|
+
- setups: Spark SQL setup statements. Used typically to register UDFs.
|
|
474
|
+
|
|
475
|
+
- partitionColumn: Only needed for `max_date` template
|
|
476
|
+
|
|
477
|
+
- engineType: By default, spark is the compute engine. You can specify an override (eg. bigquery, etc.)
|
|
478
|
+
|
|
479
|
+
- tableDependencies
|
|
480
|
+
- recomputeDays
|
|
481
|
+
|
|
482
|
+
"""
|
|
483
|
+
thrift_spec = None
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def __init__(self, metaData = None, query = None, startPartition = None, setups = None, partitionColumn = None, engineType = None, tableDependencies = None, recomputeDays = None,):
|
|
487
|
+
self.metaData = metaData
|
|
488
|
+
self.query = query
|
|
489
|
+
self.startPartition = startPartition
|
|
490
|
+
self.setups = setups
|
|
491
|
+
self.partitionColumn = partitionColumn
|
|
492
|
+
self.engineType = engineType
|
|
493
|
+
self.tableDependencies = tableDependencies
|
|
494
|
+
self.recomputeDays = recomputeDays
|
|
495
|
+
|
|
496
|
+
def read(self, iprot):
|
|
497
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
498
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
499
|
+
return
|
|
500
|
+
iprot.readStructBegin()
|
|
501
|
+
while True:
|
|
502
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
503
|
+
if ftype == TType.STOP:
|
|
504
|
+
break
|
|
505
|
+
if fid == 1:
|
|
506
|
+
if ftype == TType.STRUCT:
|
|
507
|
+
self.metaData = MetaData()
|
|
508
|
+
self.metaData.read(iprot)
|
|
509
|
+
else:
|
|
510
|
+
iprot.skip(ftype)
|
|
511
|
+
elif fid == 2:
|
|
512
|
+
if ftype == TType.STRING:
|
|
513
|
+
self.query = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
514
|
+
else:
|
|
515
|
+
iprot.skip(ftype)
|
|
516
|
+
elif fid == 3:
|
|
517
|
+
if ftype == TType.STRING:
|
|
518
|
+
self.startPartition = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
519
|
+
else:
|
|
520
|
+
iprot.skip(ftype)
|
|
521
|
+
elif fid == 4:
|
|
522
|
+
if ftype == TType.LIST:
|
|
523
|
+
self.setups = []
|
|
524
|
+
(_etype33, _size30) = iprot.readListBegin()
|
|
525
|
+
for _i34 in range(_size30):
|
|
526
|
+
_elem35 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
527
|
+
self.setups.append(_elem35)
|
|
528
|
+
iprot.readListEnd()
|
|
529
|
+
else:
|
|
530
|
+
iprot.skip(ftype)
|
|
531
|
+
elif fid == 5:
|
|
532
|
+
if ftype == TType.STRING:
|
|
533
|
+
self.partitionColumn = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
534
|
+
else:
|
|
535
|
+
iprot.skip(ftype)
|
|
536
|
+
elif fid == 6:
|
|
537
|
+
if ftype == TType.I32:
|
|
538
|
+
self.engineType = iprot.readI32()
|
|
539
|
+
else:
|
|
540
|
+
iprot.skip(ftype)
|
|
541
|
+
elif fid == 7:
|
|
542
|
+
if ftype == TType.LIST:
|
|
543
|
+
self.tableDependencies = []
|
|
544
|
+
(_etype39, _size36) = iprot.readListBegin()
|
|
545
|
+
for _i40 in range(_size36):
|
|
546
|
+
_elem41 = ai.chronon.api.common.ttypes.TableDependency()
|
|
547
|
+
_elem41.read(iprot)
|
|
548
|
+
self.tableDependencies.append(_elem41)
|
|
549
|
+
iprot.readListEnd()
|
|
550
|
+
else:
|
|
551
|
+
iprot.skip(ftype)
|
|
552
|
+
elif fid == 20:
|
|
553
|
+
if ftype == TType.I32:
|
|
554
|
+
self.recomputeDays = iprot.readI32()
|
|
555
|
+
else:
|
|
556
|
+
iprot.skip(ftype)
|
|
557
|
+
else:
|
|
558
|
+
iprot.skip(ftype)
|
|
559
|
+
iprot.readFieldEnd()
|
|
560
|
+
iprot.readStructEnd()
|
|
561
|
+
|
|
562
|
+
def write(self, oprot):
|
|
563
|
+
self.validate()
|
|
564
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
565
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
566
|
+
return
|
|
567
|
+
oprot.writeStructBegin('StagingQuery')
|
|
568
|
+
if self.metaData is not None:
|
|
569
|
+
oprot.writeFieldBegin('metaData', TType.STRUCT, 1)
|
|
570
|
+
self.metaData.write(oprot)
|
|
571
|
+
oprot.writeFieldEnd()
|
|
572
|
+
if self.query is not None:
|
|
573
|
+
oprot.writeFieldBegin('query', TType.STRING, 2)
|
|
574
|
+
oprot.writeString(self.query.encode('utf-8') if sys.version_info[0] == 2 else self.query)
|
|
575
|
+
oprot.writeFieldEnd()
|
|
576
|
+
if self.startPartition is not None:
|
|
577
|
+
oprot.writeFieldBegin('startPartition', TType.STRING, 3)
|
|
578
|
+
oprot.writeString(self.startPartition.encode('utf-8') if sys.version_info[0] == 2 else self.startPartition)
|
|
579
|
+
oprot.writeFieldEnd()
|
|
580
|
+
if self.setups is not None:
|
|
581
|
+
oprot.writeFieldBegin('setups', TType.LIST, 4)
|
|
582
|
+
oprot.writeListBegin(TType.STRING, len(self.setups))
|
|
583
|
+
for iter42 in self.setups:
|
|
584
|
+
oprot.writeString(iter42.encode('utf-8') if sys.version_info[0] == 2 else iter42)
|
|
585
|
+
oprot.writeListEnd()
|
|
586
|
+
oprot.writeFieldEnd()
|
|
587
|
+
if self.partitionColumn is not None:
|
|
588
|
+
oprot.writeFieldBegin('partitionColumn', TType.STRING, 5)
|
|
589
|
+
oprot.writeString(self.partitionColumn.encode('utf-8') if sys.version_info[0] == 2 else self.partitionColumn)
|
|
590
|
+
oprot.writeFieldEnd()
|
|
591
|
+
if self.engineType is not None:
|
|
592
|
+
oprot.writeFieldBegin('engineType', TType.I32, 6)
|
|
593
|
+
oprot.writeI32(self.engineType)
|
|
594
|
+
oprot.writeFieldEnd()
|
|
595
|
+
if self.tableDependencies is not None:
|
|
596
|
+
oprot.writeFieldBegin('tableDependencies', TType.LIST, 7)
|
|
597
|
+
oprot.writeListBegin(TType.STRUCT, len(self.tableDependencies))
|
|
598
|
+
for iter43 in self.tableDependencies:
|
|
599
|
+
iter43.write(oprot)
|
|
600
|
+
oprot.writeListEnd()
|
|
601
|
+
oprot.writeFieldEnd()
|
|
602
|
+
if self.recomputeDays is not None:
|
|
603
|
+
oprot.writeFieldBegin('recomputeDays', TType.I32, 20)
|
|
604
|
+
oprot.writeI32(self.recomputeDays)
|
|
605
|
+
oprot.writeFieldEnd()
|
|
606
|
+
oprot.writeFieldStop()
|
|
607
|
+
oprot.writeStructEnd()
|
|
608
|
+
|
|
609
|
+
def validate(self):
|
|
610
|
+
return
|
|
611
|
+
|
|
612
|
+
def __repr__(self):
|
|
613
|
+
L = ['%s=%r' % (key, value)
|
|
614
|
+
for key, value in self.__dict__.items()]
|
|
615
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
616
|
+
|
|
617
|
+
def __eq__(self, other):
|
|
618
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
619
|
+
|
|
620
|
+
def __ne__(self, other):
|
|
621
|
+
return not (self == other)
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
class EventSource(object):
|
|
625
|
+
"""
|
|
626
|
+
Attributes:
|
|
627
|
+
- table: Table currently needs to be a 'ds' (date string - yyyy-MM-dd) partitioned hive table. Table names can contain subpartition specs, example db.table/system=mobile/currency=USD
|
|
628
|
+
|
|
629
|
+
- topic: Topic is a kafka table. The table contains all the events historically came through this topic.
|
|
630
|
+
|
|
631
|
+
- query: The logic used to scan both the table and the topic. Contains row level transformations and filtering expressed as Spark SQL statements.
|
|
632
|
+
|
|
633
|
+
- isCumulative: If each new hive partition contains not just the current day's events but the entire set of events since the begininng. The key property is that the events are not mutated across partitions.
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
"""
|
|
637
|
+
thrift_spec = None
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
def __init__(self, table = None, topic = None, query = None, isCumulative = None,):
|
|
641
|
+
self.table = table
|
|
642
|
+
self.topic = topic
|
|
643
|
+
self.query = query
|
|
644
|
+
self.isCumulative = isCumulative
|
|
645
|
+
|
|
646
|
+
def read(self, iprot):
|
|
647
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
648
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
649
|
+
return
|
|
650
|
+
iprot.readStructBegin()
|
|
651
|
+
while True:
|
|
652
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
653
|
+
if ftype == TType.STOP:
|
|
654
|
+
break
|
|
655
|
+
if fid == 1:
|
|
656
|
+
if ftype == TType.STRING:
|
|
657
|
+
self.table = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
658
|
+
else:
|
|
659
|
+
iprot.skip(ftype)
|
|
660
|
+
elif fid == 2:
|
|
661
|
+
if ftype == TType.STRING:
|
|
662
|
+
self.topic = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
663
|
+
else:
|
|
664
|
+
iprot.skip(ftype)
|
|
665
|
+
elif fid == 3:
|
|
666
|
+
if ftype == TType.STRUCT:
|
|
667
|
+
self.query = Query()
|
|
668
|
+
self.query.read(iprot)
|
|
669
|
+
else:
|
|
670
|
+
iprot.skip(ftype)
|
|
671
|
+
elif fid == 4:
|
|
672
|
+
if ftype == TType.BOOL:
|
|
673
|
+
self.isCumulative = iprot.readBool()
|
|
674
|
+
else:
|
|
675
|
+
iprot.skip(ftype)
|
|
676
|
+
else:
|
|
677
|
+
iprot.skip(ftype)
|
|
678
|
+
iprot.readFieldEnd()
|
|
679
|
+
iprot.readStructEnd()
|
|
680
|
+
|
|
681
|
+
def write(self, oprot):
|
|
682
|
+
self.validate()
|
|
683
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
684
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
685
|
+
return
|
|
686
|
+
oprot.writeStructBegin('EventSource')
|
|
687
|
+
if self.table is not None:
|
|
688
|
+
oprot.writeFieldBegin('table', TType.STRING, 1)
|
|
689
|
+
oprot.writeString(self.table.encode('utf-8') if sys.version_info[0] == 2 else self.table)
|
|
690
|
+
oprot.writeFieldEnd()
|
|
691
|
+
if self.topic is not None:
|
|
692
|
+
oprot.writeFieldBegin('topic', TType.STRING, 2)
|
|
693
|
+
oprot.writeString(self.topic.encode('utf-8') if sys.version_info[0] == 2 else self.topic)
|
|
694
|
+
oprot.writeFieldEnd()
|
|
695
|
+
if self.query is not None:
|
|
696
|
+
oprot.writeFieldBegin('query', TType.STRUCT, 3)
|
|
697
|
+
self.query.write(oprot)
|
|
698
|
+
oprot.writeFieldEnd()
|
|
699
|
+
if self.isCumulative is not None:
|
|
700
|
+
oprot.writeFieldBegin('isCumulative', TType.BOOL, 4)
|
|
701
|
+
oprot.writeBool(self.isCumulative)
|
|
702
|
+
oprot.writeFieldEnd()
|
|
703
|
+
oprot.writeFieldStop()
|
|
704
|
+
oprot.writeStructEnd()
|
|
705
|
+
|
|
706
|
+
def validate(self):
|
|
707
|
+
return
|
|
708
|
+
|
|
709
|
+
def __repr__(self):
|
|
710
|
+
L = ['%s=%r' % (key, value)
|
|
711
|
+
for key, value in self.__dict__.items()]
|
|
712
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
713
|
+
|
|
714
|
+
def __eq__(self, other):
|
|
715
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
716
|
+
|
|
717
|
+
def __ne__(self, other):
|
|
718
|
+
return not (self == other)
|
|
719
|
+
|
|
720
|
+
|
|
721
|
+
class EntitySource(object):
|
|
722
|
+
"""
|
|
723
|
+
Entity Sources represent data that gets mutated over-time - at row-level. This is a group of three data elements.
|
|
724
|
+
snapshotTable, mutationTable and mutationTopic. mutationTable and mutationTopic are only necessary if we are trying
|
|
725
|
+
to create realtime or point-in-time aggregations over these sources. Entity sources usually map 1:1 with a database
|
|
726
|
+
tables in your OLTP store that typically serves live application traffic. When mutation data is absent they map 1:1
|
|
727
|
+
to `dim` tables in star schema.
|
|
728
|
+
|
|
729
|
+
Attributes:
|
|
730
|
+
- snapshotTable: Snapshot table currently needs to be a 'ds' (date string - yyyy-MM-dd) partitioned hive table.
|
|
731
|
+
- mutationTable: Topic is a kafka table. The table contains all the events that historically came through this topic.
|
|
732
|
+
- mutationTopic: The logic used to scan both the table and the topic. Contains row level transformations and filtering expressed as Spark SQL statements.
|
|
733
|
+
- query: If each new hive partition contains not just the current day's events but the entire set of events since the begininng. The key property is that the events are not mutated across partitions.
|
|
734
|
+
|
|
735
|
+
"""
|
|
736
|
+
thrift_spec = None
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
def __init__(self, snapshotTable = None, mutationTable = None, mutationTopic = None, query = None,):
|
|
740
|
+
self.snapshotTable = snapshotTable
|
|
741
|
+
self.mutationTable = mutationTable
|
|
742
|
+
self.mutationTopic = mutationTopic
|
|
743
|
+
self.query = query
|
|
744
|
+
|
|
745
|
+
def read(self, iprot):
|
|
746
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
747
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
748
|
+
return
|
|
749
|
+
iprot.readStructBegin()
|
|
750
|
+
while True:
|
|
751
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
752
|
+
if ftype == TType.STOP:
|
|
753
|
+
break
|
|
754
|
+
if fid == 1:
|
|
755
|
+
if ftype == TType.STRING:
|
|
756
|
+
self.snapshotTable = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
757
|
+
else:
|
|
758
|
+
iprot.skip(ftype)
|
|
759
|
+
elif fid == 2:
|
|
760
|
+
if ftype == TType.STRING:
|
|
761
|
+
self.mutationTable = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
762
|
+
else:
|
|
763
|
+
iprot.skip(ftype)
|
|
764
|
+
elif fid == 3:
|
|
765
|
+
if ftype == TType.STRING:
|
|
766
|
+
self.mutationTopic = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
767
|
+
else:
|
|
768
|
+
iprot.skip(ftype)
|
|
769
|
+
elif fid == 4:
|
|
770
|
+
if ftype == TType.STRUCT:
|
|
771
|
+
self.query = Query()
|
|
772
|
+
self.query.read(iprot)
|
|
773
|
+
else:
|
|
774
|
+
iprot.skip(ftype)
|
|
775
|
+
else:
|
|
776
|
+
iprot.skip(ftype)
|
|
777
|
+
iprot.readFieldEnd()
|
|
778
|
+
iprot.readStructEnd()
|
|
779
|
+
|
|
780
|
+
def write(self, oprot):
|
|
781
|
+
self.validate()
|
|
782
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
783
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
784
|
+
return
|
|
785
|
+
oprot.writeStructBegin('EntitySource')
|
|
786
|
+
if self.snapshotTable is not None:
|
|
787
|
+
oprot.writeFieldBegin('snapshotTable', TType.STRING, 1)
|
|
788
|
+
oprot.writeString(self.snapshotTable.encode('utf-8') if sys.version_info[0] == 2 else self.snapshotTable)
|
|
789
|
+
oprot.writeFieldEnd()
|
|
790
|
+
if self.mutationTable is not None:
|
|
791
|
+
oprot.writeFieldBegin('mutationTable', TType.STRING, 2)
|
|
792
|
+
oprot.writeString(self.mutationTable.encode('utf-8') if sys.version_info[0] == 2 else self.mutationTable)
|
|
793
|
+
oprot.writeFieldEnd()
|
|
794
|
+
if self.mutationTopic is not None:
|
|
795
|
+
oprot.writeFieldBegin('mutationTopic', TType.STRING, 3)
|
|
796
|
+
oprot.writeString(self.mutationTopic.encode('utf-8') if sys.version_info[0] == 2 else self.mutationTopic)
|
|
797
|
+
oprot.writeFieldEnd()
|
|
798
|
+
if self.query is not None:
|
|
799
|
+
oprot.writeFieldBegin('query', TType.STRUCT, 4)
|
|
800
|
+
self.query.write(oprot)
|
|
801
|
+
oprot.writeFieldEnd()
|
|
802
|
+
oprot.writeFieldStop()
|
|
803
|
+
oprot.writeStructEnd()
|
|
804
|
+
|
|
805
|
+
def validate(self):
|
|
806
|
+
return
|
|
807
|
+
|
|
808
|
+
def __repr__(self):
|
|
809
|
+
L = ['%s=%r' % (key, value)
|
|
810
|
+
for key, value in self.__dict__.items()]
|
|
811
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
812
|
+
|
|
813
|
+
def __eq__(self, other):
|
|
814
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
815
|
+
|
|
816
|
+
def __ne__(self, other):
|
|
817
|
+
return not (self == other)
|
|
818
|
+
|
|
819
|
+
|
|
820
|
+
class ExternalSource(object):
|
|
821
|
+
"""
|
|
822
|
+
Attributes:
|
|
823
|
+
- metadata
|
|
824
|
+
- keySchema
|
|
825
|
+
- valueSchema
|
|
826
|
+
|
|
827
|
+
"""
|
|
828
|
+
thrift_spec = None
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def __init__(self, metadata = None, keySchema = None, valueSchema = None,):
|
|
832
|
+
self.metadata = metadata
|
|
833
|
+
self.keySchema = keySchema
|
|
834
|
+
self.valueSchema = valueSchema
|
|
835
|
+
|
|
836
|
+
def read(self, iprot):
|
|
837
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
838
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
839
|
+
return
|
|
840
|
+
iprot.readStructBegin()
|
|
841
|
+
while True:
|
|
842
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
843
|
+
if ftype == TType.STOP:
|
|
844
|
+
break
|
|
845
|
+
if fid == 1:
|
|
846
|
+
if ftype == TType.STRUCT:
|
|
847
|
+
self.metadata = MetaData()
|
|
848
|
+
self.metadata.read(iprot)
|
|
849
|
+
else:
|
|
850
|
+
iprot.skip(ftype)
|
|
851
|
+
elif fid == 2:
|
|
852
|
+
if ftype == TType.STRUCT:
|
|
853
|
+
self.keySchema = TDataType()
|
|
854
|
+
self.keySchema.read(iprot)
|
|
855
|
+
else:
|
|
856
|
+
iprot.skip(ftype)
|
|
857
|
+
elif fid == 3:
|
|
858
|
+
if ftype == TType.STRUCT:
|
|
859
|
+
self.valueSchema = TDataType()
|
|
860
|
+
self.valueSchema.read(iprot)
|
|
861
|
+
else:
|
|
862
|
+
iprot.skip(ftype)
|
|
863
|
+
else:
|
|
864
|
+
iprot.skip(ftype)
|
|
865
|
+
iprot.readFieldEnd()
|
|
866
|
+
iprot.readStructEnd()
|
|
867
|
+
|
|
868
|
+
def write(self, oprot):
|
|
869
|
+
self.validate()
|
|
870
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
871
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
872
|
+
return
|
|
873
|
+
oprot.writeStructBegin('ExternalSource')
|
|
874
|
+
if self.metadata is not None:
|
|
875
|
+
oprot.writeFieldBegin('metadata', TType.STRUCT, 1)
|
|
876
|
+
self.metadata.write(oprot)
|
|
877
|
+
oprot.writeFieldEnd()
|
|
878
|
+
if self.keySchema is not None:
|
|
879
|
+
oprot.writeFieldBegin('keySchema', TType.STRUCT, 2)
|
|
880
|
+
self.keySchema.write(oprot)
|
|
881
|
+
oprot.writeFieldEnd()
|
|
882
|
+
if self.valueSchema is not None:
|
|
883
|
+
oprot.writeFieldBegin('valueSchema', TType.STRUCT, 3)
|
|
884
|
+
self.valueSchema.write(oprot)
|
|
885
|
+
oprot.writeFieldEnd()
|
|
886
|
+
oprot.writeFieldStop()
|
|
887
|
+
oprot.writeStructEnd()
|
|
888
|
+
|
|
889
|
+
def validate(self):
|
|
890
|
+
return
|
|
891
|
+
|
|
892
|
+
def __repr__(self):
|
|
893
|
+
L = ['%s=%r' % (key, value)
|
|
894
|
+
for key, value in self.__dict__.items()]
|
|
895
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
896
|
+
|
|
897
|
+
def __eq__(self, other):
|
|
898
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
899
|
+
|
|
900
|
+
def __ne__(self, other):
|
|
901
|
+
return not (self == other)
|
|
902
|
+
|
|
903
|
+
|
|
904
|
+
class JoinSource(object):
|
|
905
|
+
"""
|
|
906
|
+
Output of a Join can be used as input to downstream computations like GroupBy or a Join.
|
|
907
|
+
Below is a short description of each of the cases we handle.
|
|
908
|
+
Case #1: a join's source is another join [TODO]
|
|
909
|
+
- while serving, we expect the keys for the upstream join to be passed in the request.
|
|
910
|
+
we will query upstream first, and use the result to query downstream
|
|
911
|
+
- while backfill, we will backfill the upstream first, and use the table as the left of the subsequent join
|
|
912
|
+
- this is currently a "to do" because users can achieve this by themselves unlike case 2:
|
|
913
|
+
Case #2: a join is the source of another GroupBy
|
|
914
|
+
- We will support arbitrarily long transformation chains with this.
|
|
915
|
+
- for batch (Accuracy.SNAPSHOT), we simply backfill the join first and compute groupBy as usual
|
|
916
|
+
- will substitute the joinSource with the resulting table and continue computation
|
|
917
|
+
- we will add a "resolve source" step prior to backfills that will compute the parent join and update the source
|
|
918
|
+
- for realtime (Accuracy.TEMPORAL), we need to do "stream enrichment"
|
|
919
|
+
- we will simply issue "fetchJoin" and create an enriched source. Note the join left should be of type "events".
|
|
920
|
+
|
|
921
|
+
|
|
922
|
+
Attributes:
|
|
923
|
+
- join
|
|
924
|
+
- query
|
|
925
|
+
|
|
926
|
+
"""
|
|
927
|
+
thrift_spec = None
|
|
928
|
+
|
|
929
|
+
|
|
930
|
+
def __init__(self, join = None, query = None,):
|
|
931
|
+
self.join = join
|
|
932
|
+
self.query = query
|
|
933
|
+
|
|
934
|
+
def read(self, iprot):
|
|
935
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
936
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
937
|
+
return
|
|
938
|
+
iprot.readStructBegin()
|
|
939
|
+
while True:
|
|
940
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
941
|
+
if ftype == TType.STOP:
|
|
942
|
+
break
|
|
943
|
+
if fid == 1:
|
|
944
|
+
if ftype == TType.STRUCT:
|
|
945
|
+
self.join = Join()
|
|
946
|
+
self.join.read(iprot)
|
|
947
|
+
else:
|
|
948
|
+
iprot.skip(ftype)
|
|
949
|
+
elif fid == 2:
|
|
950
|
+
if ftype == TType.STRUCT:
|
|
951
|
+
self.query = Query()
|
|
952
|
+
self.query.read(iprot)
|
|
953
|
+
else:
|
|
954
|
+
iprot.skip(ftype)
|
|
955
|
+
else:
|
|
956
|
+
iprot.skip(ftype)
|
|
957
|
+
iprot.readFieldEnd()
|
|
958
|
+
iprot.readStructEnd()
|
|
959
|
+
|
|
960
|
+
def write(self, oprot):
|
|
961
|
+
self.validate()
|
|
962
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
963
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
964
|
+
return
|
|
965
|
+
oprot.writeStructBegin('JoinSource')
|
|
966
|
+
if self.join is not None:
|
|
967
|
+
oprot.writeFieldBegin('join', TType.STRUCT, 1)
|
|
968
|
+
self.join.write(oprot)
|
|
969
|
+
oprot.writeFieldEnd()
|
|
970
|
+
if self.query is not None:
|
|
971
|
+
oprot.writeFieldBegin('query', TType.STRUCT, 2)
|
|
972
|
+
self.query.write(oprot)
|
|
973
|
+
oprot.writeFieldEnd()
|
|
974
|
+
oprot.writeFieldStop()
|
|
975
|
+
oprot.writeStructEnd()
|
|
976
|
+
|
|
977
|
+
def validate(self):
|
|
978
|
+
return
|
|
979
|
+
|
|
980
|
+
def __repr__(self):
|
|
981
|
+
L = ['%s=%r' % (key, value)
|
|
982
|
+
for key, value in self.__dict__.items()]
|
|
983
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
984
|
+
|
|
985
|
+
def __eq__(self, other):
|
|
986
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
987
|
+
|
|
988
|
+
def __ne__(self, other):
|
|
989
|
+
return not (self == other)
|
|
990
|
+
|
|
991
|
+
|
|
992
|
+
class Source(object):
|
|
993
|
+
"""
|
|
994
|
+
Attributes:
|
|
995
|
+
- events
|
|
996
|
+
- entities
|
|
997
|
+
- joinSource
|
|
998
|
+
|
|
999
|
+
"""
|
|
1000
|
+
thrift_spec = None
|
|
1001
|
+
|
|
1002
|
+
|
|
1003
|
+
def __init__(self, events = None, entities = None, joinSource = None,):
|
|
1004
|
+
self.events = events
|
|
1005
|
+
self.entities = entities
|
|
1006
|
+
self.joinSource = joinSource
|
|
1007
|
+
|
|
1008
|
+
def read(self, iprot):
|
|
1009
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
1010
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
1011
|
+
return
|
|
1012
|
+
iprot.readStructBegin()
|
|
1013
|
+
while True:
|
|
1014
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
1015
|
+
if ftype == TType.STOP:
|
|
1016
|
+
break
|
|
1017
|
+
if fid == 1:
|
|
1018
|
+
if ftype == TType.STRUCT:
|
|
1019
|
+
self.events = EventSource()
|
|
1020
|
+
self.events.read(iprot)
|
|
1021
|
+
else:
|
|
1022
|
+
iprot.skip(ftype)
|
|
1023
|
+
elif fid == 2:
|
|
1024
|
+
if ftype == TType.STRUCT:
|
|
1025
|
+
self.entities = EntitySource()
|
|
1026
|
+
self.entities.read(iprot)
|
|
1027
|
+
else:
|
|
1028
|
+
iprot.skip(ftype)
|
|
1029
|
+
elif fid == 3:
|
|
1030
|
+
if ftype == TType.STRUCT:
|
|
1031
|
+
self.joinSource = JoinSource()
|
|
1032
|
+
self.joinSource.read(iprot)
|
|
1033
|
+
else:
|
|
1034
|
+
iprot.skip(ftype)
|
|
1035
|
+
else:
|
|
1036
|
+
iprot.skip(ftype)
|
|
1037
|
+
iprot.readFieldEnd()
|
|
1038
|
+
iprot.readStructEnd()
|
|
1039
|
+
|
|
1040
|
+
def write(self, oprot):
|
|
1041
|
+
self.validate()
|
|
1042
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
1043
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
1044
|
+
return
|
|
1045
|
+
oprot.writeStructBegin('Source')
|
|
1046
|
+
if self.events is not None:
|
|
1047
|
+
oprot.writeFieldBegin('events', TType.STRUCT, 1)
|
|
1048
|
+
self.events.write(oprot)
|
|
1049
|
+
oprot.writeFieldEnd()
|
|
1050
|
+
if self.entities is not None:
|
|
1051
|
+
oprot.writeFieldBegin('entities', TType.STRUCT, 2)
|
|
1052
|
+
self.entities.write(oprot)
|
|
1053
|
+
oprot.writeFieldEnd()
|
|
1054
|
+
if self.joinSource is not None:
|
|
1055
|
+
oprot.writeFieldBegin('joinSource', TType.STRUCT, 3)
|
|
1056
|
+
self.joinSource.write(oprot)
|
|
1057
|
+
oprot.writeFieldEnd()
|
|
1058
|
+
oprot.writeFieldStop()
|
|
1059
|
+
oprot.writeStructEnd()
|
|
1060
|
+
|
|
1061
|
+
def validate(self):
|
|
1062
|
+
return
|
|
1063
|
+
|
|
1064
|
+
def __repr__(self):
|
|
1065
|
+
L = ['%s=%r' % (key, value)
|
|
1066
|
+
for key, value in self.__dict__.items()]
|
|
1067
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
1068
|
+
|
|
1069
|
+
def __eq__(self, other):
|
|
1070
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
1071
|
+
|
|
1072
|
+
def __ne__(self, other):
|
|
1073
|
+
return not (self == other)
|
|
1074
|
+
|
|
1075
|
+
|
|
1076
|
+
class Aggregation(object):
|
|
1077
|
+
"""
|
|
1078
|
+
Chronon provides a powerful aggregations primitive - that takes the familiar aggregation operation, via groupBy in
|
|
1079
|
+
SQL and extends it with three things - windowing, bucketing and auto-explode.
|
|
1080
|
+
|
|
1081
|
+
Attributes:
|
|
1082
|
+
- inputColumn: The column as specified in source.query.selects - on which we need to aggregate with.
|
|
1083
|
+
|
|
1084
|
+
- operation: The type of aggregation that needs to be performed on the inputColumn.
|
|
1085
|
+
|
|
1086
|
+
- argMap: Extra arguments that needs to be passed to some of the operations like LAST_K, APPROX_PERCENTILE.
|
|
1087
|
+
|
|
1088
|
+
- windows: For TEMPORAL case windows are sawtooth. Meaning head slides ahead continuously in time, whereas, the tail only hops ahead, at discrete points in time. Hop is determined by the window size automatically. The maximum hop size is 1/12 of window size. You can specify multiple such windows at once.
|
|
1089
|
+
- Window > 12 days -> Hop Size = 1 day
|
|
1090
|
+
- Window > 12 hours -> Hop Size = 1 hr
|
|
1091
|
+
- Window > 1hr -> Hop Size = 5 minutes
|
|
1092
|
+
- buckets: This is an additional layer of aggregation. You can key a group_by by user, and bucket a “item_view” count by “item_category”. This will produce one row per user, with column containing map of “item_category” to “view_count”. You can specify multiple such buckets at once
|
|
1093
|
+
|
|
1094
|
+
"""
|
|
1095
|
+
thrift_spec = None
|
|
1096
|
+
|
|
1097
|
+
|
|
1098
|
+
def __init__(self, inputColumn = None, operation = None, argMap = None, windows = None, buckets = None,):
|
|
1099
|
+
self.inputColumn = inputColumn
|
|
1100
|
+
self.operation = operation
|
|
1101
|
+
self.argMap = argMap
|
|
1102
|
+
self.windows = windows
|
|
1103
|
+
self.buckets = buckets
|
|
1104
|
+
|
|
1105
|
+
def read(self, iprot):
|
|
1106
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
1107
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
1108
|
+
return
|
|
1109
|
+
iprot.readStructBegin()
|
|
1110
|
+
while True:
|
|
1111
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
1112
|
+
if ftype == TType.STOP:
|
|
1113
|
+
break
|
|
1114
|
+
if fid == 1:
|
|
1115
|
+
if ftype == TType.STRING:
|
|
1116
|
+
self.inputColumn = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1117
|
+
else:
|
|
1118
|
+
iprot.skip(ftype)
|
|
1119
|
+
elif fid == 2:
|
|
1120
|
+
if ftype == TType.I32:
|
|
1121
|
+
self.operation = iprot.readI32()
|
|
1122
|
+
else:
|
|
1123
|
+
iprot.skip(ftype)
|
|
1124
|
+
elif fid == 3:
|
|
1125
|
+
if ftype == TType.MAP:
|
|
1126
|
+
self.argMap = {}
|
|
1127
|
+
(_ktype45, _vtype46, _size44) = iprot.readMapBegin()
|
|
1128
|
+
for _i48 in range(_size44):
|
|
1129
|
+
_key49 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1130
|
+
_val50 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1131
|
+
self.argMap[_key49] = _val50
|
|
1132
|
+
iprot.readMapEnd()
|
|
1133
|
+
else:
|
|
1134
|
+
iprot.skip(ftype)
|
|
1135
|
+
elif fid == 4:
|
|
1136
|
+
if ftype == TType.LIST:
|
|
1137
|
+
self.windows = []
|
|
1138
|
+
(_etype54, _size51) = iprot.readListBegin()
|
|
1139
|
+
for _i55 in range(_size51):
|
|
1140
|
+
_elem56 = ai.chronon.api.common.ttypes.Window()
|
|
1141
|
+
_elem56.read(iprot)
|
|
1142
|
+
self.windows.append(_elem56)
|
|
1143
|
+
iprot.readListEnd()
|
|
1144
|
+
else:
|
|
1145
|
+
iprot.skip(ftype)
|
|
1146
|
+
elif fid == 5:
|
|
1147
|
+
if ftype == TType.LIST:
|
|
1148
|
+
self.buckets = []
|
|
1149
|
+
(_etype60, _size57) = iprot.readListBegin()
|
|
1150
|
+
for _i61 in range(_size57):
|
|
1151
|
+
_elem62 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1152
|
+
self.buckets.append(_elem62)
|
|
1153
|
+
iprot.readListEnd()
|
|
1154
|
+
else:
|
|
1155
|
+
iprot.skip(ftype)
|
|
1156
|
+
else:
|
|
1157
|
+
iprot.skip(ftype)
|
|
1158
|
+
iprot.readFieldEnd()
|
|
1159
|
+
iprot.readStructEnd()
|
|
1160
|
+
|
|
1161
|
+
def write(self, oprot):
|
|
1162
|
+
self.validate()
|
|
1163
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
1164
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
1165
|
+
return
|
|
1166
|
+
oprot.writeStructBegin('Aggregation')
|
|
1167
|
+
if self.inputColumn is not None:
|
|
1168
|
+
oprot.writeFieldBegin('inputColumn', TType.STRING, 1)
|
|
1169
|
+
oprot.writeString(self.inputColumn.encode('utf-8') if sys.version_info[0] == 2 else self.inputColumn)
|
|
1170
|
+
oprot.writeFieldEnd()
|
|
1171
|
+
if self.operation is not None:
|
|
1172
|
+
oprot.writeFieldBegin('operation', TType.I32, 2)
|
|
1173
|
+
oprot.writeI32(self.operation)
|
|
1174
|
+
oprot.writeFieldEnd()
|
|
1175
|
+
if self.argMap is not None:
|
|
1176
|
+
oprot.writeFieldBegin('argMap', TType.MAP, 3)
|
|
1177
|
+
oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.argMap))
|
|
1178
|
+
for kiter63, viter64 in self.argMap.items():
|
|
1179
|
+
oprot.writeString(kiter63.encode('utf-8') if sys.version_info[0] == 2 else kiter63)
|
|
1180
|
+
oprot.writeString(viter64.encode('utf-8') if sys.version_info[0] == 2 else viter64)
|
|
1181
|
+
oprot.writeMapEnd()
|
|
1182
|
+
oprot.writeFieldEnd()
|
|
1183
|
+
if self.windows is not None:
|
|
1184
|
+
oprot.writeFieldBegin('windows', TType.LIST, 4)
|
|
1185
|
+
oprot.writeListBegin(TType.STRUCT, len(self.windows))
|
|
1186
|
+
for iter65 in self.windows:
|
|
1187
|
+
iter65.write(oprot)
|
|
1188
|
+
oprot.writeListEnd()
|
|
1189
|
+
oprot.writeFieldEnd()
|
|
1190
|
+
if self.buckets is not None:
|
|
1191
|
+
oprot.writeFieldBegin('buckets', TType.LIST, 5)
|
|
1192
|
+
oprot.writeListBegin(TType.STRING, len(self.buckets))
|
|
1193
|
+
for iter66 in self.buckets:
|
|
1194
|
+
oprot.writeString(iter66.encode('utf-8') if sys.version_info[0] == 2 else iter66)
|
|
1195
|
+
oprot.writeListEnd()
|
|
1196
|
+
oprot.writeFieldEnd()
|
|
1197
|
+
oprot.writeFieldStop()
|
|
1198
|
+
oprot.writeStructEnd()
|
|
1199
|
+
|
|
1200
|
+
def validate(self):
|
|
1201
|
+
return
|
|
1202
|
+
|
|
1203
|
+
def __repr__(self):
|
|
1204
|
+
L = ['%s=%r' % (key, value)
|
|
1205
|
+
for key, value in self.__dict__.items()]
|
|
1206
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
1207
|
+
|
|
1208
|
+
def __eq__(self, other):
|
|
1209
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
1210
|
+
|
|
1211
|
+
def __ne__(self, other):
|
|
1212
|
+
return not (self == other)
|
|
1213
|
+
|
|
1214
|
+
|
|
1215
|
+
class AggregationPart(object):
|
|
1216
|
+
"""
|
|
1217
|
+
Attributes:
|
|
1218
|
+
- inputColumn
|
|
1219
|
+
- operation
|
|
1220
|
+
- argMap
|
|
1221
|
+
- window
|
|
1222
|
+
- bucket
|
|
1223
|
+
|
|
1224
|
+
"""
|
|
1225
|
+
thrift_spec = None
|
|
1226
|
+
|
|
1227
|
+
|
|
1228
|
+
def __init__(self, inputColumn = None, operation = None, argMap = None, window = None, bucket = None,):
|
|
1229
|
+
self.inputColumn = inputColumn
|
|
1230
|
+
self.operation = operation
|
|
1231
|
+
self.argMap = argMap
|
|
1232
|
+
self.window = window
|
|
1233
|
+
self.bucket = bucket
|
|
1234
|
+
|
|
1235
|
+
def read(self, iprot):
|
|
1236
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
1237
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
1238
|
+
return
|
|
1239
|
+
iprot.readStructBegin()
|
|
1240
|
+
while True:
|
|
1241
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
1242
|
+
if ftype == TType.STOP:
|
|
1243
|
+
break
|
|
1244
|
+
if fid == 1:
|
|
1245
|
+
if ftype == TType.STRING:
|
|
1246
|
+
self.inputColumn = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1247
|
+
else:
|
|
1248
|
+
iprot.skip(ftype)
|
|
1249
|
+
elif fid == 2:
|
|
1250
|
+
if ftype == TType.I32:
|
|
1251
|
+
self.operation = iprot.readI32()
|
|
1252
|
+
else:
|
|
1253
|
+
iprot.skip(ftype)
|
|
1254
|
+
elif fid == 3:
|
|
1255
|
+
if ftype == TType.MAP:
|
|
1256
|
+
self.argMap = {}
|
|
1257
|
+
(_ktype68, _vtype69, _size67) = iprot.readMapBegin()
|
|
1258
|
+
for _i71 in range(_size67):
|
|
1259
|
+
_key72 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1260
|
+
_val73 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1261
|
+
self.argMap[_key72] = _val73
|
|
1262
|
+
iprot.readMapEnd()
|
|
1263
|
+
else:
|
|
1264
|
+
iprot.skip(ftype)
|
|
1265
|
+
elif fid == 4:
|
|
1266
|
+
if ftype == TType.STRUCT:
|
|
1267
|
+
self.window = ai.chronon.api.common.ttypes.Window()
|
|
1268
|
+
self.window.read(iprot)
|
|
1269
|
+
else:
|
|
1270
|
+
iprot.skip(ftype)
|
|
1271
|
+
elif fid == 5:
|
|
1272
|
+
if ftype == TType.STRING:
|
|
1273
|
+
self.bucket = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1274
|
+
else:
|
|
1275
|
+
iprot.skip(ftype)
|
|
1276
|
+
else:
|
|
1277
|
+
iprot.skip(ftype)
|
|
1278
|
+
iprot.readFieldEnd()
|
|
1279
|
+
iprot.readStructEnd()
|
|
1280
|
+
|
|
1281
|
+
def write(self, oprot):
|
|
1282
|
+
self.validate()
|
|
1283
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
1284
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
1285
|
+
return
|
|
1286
|
+
oprot.writeStructBegin('AggregationPart')
|
|
1287
|
+
if self.inputColumn is not None:
|
|
1288
|
+
oprot.writeFieldBegin('inputColumn', TType.STRING, 1)
|
|
1289
|
+
oprot.writeString(self.inputColumn.encode('utf-8') if sys.version_info[0] == 2 else self.inputColumn)
|
|
1290
|
+
oprot.writeFieldEnd()
|
|
1291
|
+
if self.operation is not None:
|
|
1292
|
+
oprot.writeFieldBegin('operation', TType.I32, 2)
|
|
1293
|
+
oprot.writeI32(self.operation)
|
|
1294
|
+
oprot.writeFieldEnd()
|
|
1295
|
+
if self.argMap is not None:
|
|
1296
|
+
oprot.writeFieldBegin('argMap', TType.MAP, 3)
|
|
1297
|
+
oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.argMap))
|
|
1298
|
+
for kiter74, viter75 in self.argMap.items():
|
|
1299
|
+
oprot.writeString(kiter74.encode('utf-8') if sys.version_info[0] == 2 else kiter74)
|
|
1300
|
+
oprot.writeString(viter75.encode('utf-8') if sys.version_info[0] == 2 else viter75)
|
|
1301
|
+
oprot.writeMapEnd()
|
|
1302
|
+
oprot.writeFieldEnd()
|
|
1303
|
+
if self.window is not None:
|
|
1304
|
+
oprot.writeFieldBegin('window', TType.STRUCT, 4)
|
|
1305
|
+
self.window.write(oprot)
|
|
1306
|
+
oprot.writeFieldEnd()
|
|
1307
|
+
if self.bucket is not None:
|
|
1308
|
+
oprot.writeFieldBegin('bucket', TType.STRING, 5)
|
|
1309
|
+
oprot.writeString(self.bucket.encode('utf-8') if sys.version_info[0] == 2 else self.bucket)
|
|
1310
|
+
oprot.writeFieldEnd()
|
|
1311
|
+
oprot.writeFieldStop()
|
|
1312
|
+
oprot.writeStructEnd()
|
|
1313
|
+
|
|
1314
|
+
def validate(self):
|
|
1315
|
+
return
|
|
1316
|
+
|
|
1317
|
+
def __repr__(self):
|
|
1318
|
+
L = ['%s=%r' % (key, value)
|
|
1319
|
+
for key, value in self.__dict__.items()]
|
|
1320
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
1321
|
+
|
|
1322
|
+
def __eq__(self, other):
|
|
1323
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
1324
|
+
|
|
1325
|
+
def __ne__(self, other):
|
|
1326
|
+
return not (self == other)
|
|
1327
|
+
|
|
1328
|
+
|
|
1329
|
+
class MetaData(object):
|
|
1330
|
+
"""
|
|
1331
|
+
contains configs params that don't change the contents of the output.
|
|
1332
|
+
|
|
1333
|
+
|
|
1334
|
+
Attributes:
|
|
1335
|
+
- name
|
|
1336
|
+
- team
|
|
1337
|
+
- version
|
|
1338
|
+
- outputNamespace
|
|
1339
|
+
- additionalOutputPartitionColumns: By default we will just partition the output by the date column - set via "spark.chronon.partition.column"
|
|
1340
|
+
With this we will partition the output with the specified additional columns
|
|
1341
|
+
|
|
1342
|
+
- tableProperties
|
|
1343
|
+
- tags
|
|
1344
|
+
- columnTags
|
|
1345
|
+
- online
|
|
1346
|
+
- production
|
|
1347
|
+
- sourceFile
|
|
1348
|
+
- customJson
|
|
1349
|
+
- consistencyCheck
|
|
1350
|
+
- samplePercent
|
|
1351
|
+
- consistencySamplePercent
|
|
1352
|
+
- driftSpec
|
|
1353
|
+
- executionInfo
|
|
1354
|
+
|
|
1355
|
+
"""
|
|
1356
|
+
thrift_spec = None
|
|
1357
|
+
|
|
1358
|
+
|
|
1359
|
+
def __init__(self, name = None, team = None, version = None, outputNamespace = None, additionalOutputPartitionColumns = None, tableProperties = None, tags = None, columnTags = None, online = None, production = None, sourceFile = None, customJson = None, consistencyCheck = None, samplePercent = None, consistencySamplePercent = None, driftSpec = None, executionInfo = None,):
|
|
1360
|
+
self.name = name
|
|
1361
|
+
self.team = team
|
|
1362
|
+
self.version = version
|
|
1363
|
+
self.outputNamespace = outputNamespace
|
|
1364
|
+
self.additionalOutputPartitionColumns = additionalOutputPartitionColumns
|
|
1365
|
+
self.tableProperties = tableProperties
|
|
1366
|
+
self.tags = tags
|
|
1367
|
+
self.columnTags = columnTags
|
|
1368
|
+
self.online = online
|
|
1369
|
+
self.production = production
|
|
1370
|
+
self.sourceFile = sourceFile
|
|
1371
|
+
self.customJson = customJson
|
|
1372
|
+
self.consistencyCheck = consistencyCheck
|
|
1373
|
+
self.samplePercent = samplePercent
|
|
1374
|
+
self.consistencySamplePercent = consistencySamplePercent
|
|
1375
|
+
self.driftSpec = driftSpec
|
|
1376
|
+
self.executionInfo = executionInfo
|
|
1377
|
+
|
|
1378
|
+
def read(self, iprot):
|
|
1379
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
1380
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
1381
|
+
return
|
|
1382
|
+
iprot.readStructBegin()
|
|
1383
|
+
while True:
|
|
1384
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
1385
|
+
if ftype == TType.STOP:
|
|
1386
|
+
break
|
|
1387
|
+
if fid == 1:
|
|
1388
|
+
if ftype == TType.STRING:
|
|
1389
|
+
self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1390
|
+
else:
|
|
1391
|
+
iprot.skip(ftype)
|
|
1392
|
+
elif fid == 2:
|
|
1393
|
+
if ftype == TType.STRING:
|
|
1394
|
+
self.team = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1395
|
+
else:
|
|
1396
|
+
iprot.skip(ftype)
|
|
1397
|
+
elif fid == 3:
|
|
1398
|
+
if ftype == TType.STRING:
|
|
1399
|
+
self.version = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1400
|
+
else:
|
|
1401
|
+
iprot.skip(ftype)
|
|
1402
|
+
elif fid == 4:
|
|
1403
|
+
if ftype == TType.STRING:
|
|
1404
|
+
self.outputNamespace = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1405
|
+
else:
|
|
1406
|
+
iprot.skip(ftype)
|
|
1407
|
+
elif fid == 5:
|
|
1408
|
+
if ftype == TType.LIST:
|
|
1409
|
+
self.additionalOutputPartitionColumns = []
|
|
1410
|
+
(_etype79, _size76) = iprot.readListBegin()
|
|
1411
|
+
for _i80 in range(_size76):
|
|
1412
|
+
_elem81 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1413
|
+
self.additionalOutputPartitionColumns.append(_elem81)
|
|
1414
|
+
iprot.readListEnd()
|
|
1415
|
+
else:
|
|
1416
|
+
iprot.skip(ftype)
|
|
1417
|
+
elif fid == 6:
|
|
1418
|
+
if ftype == TType.MAP:
|
|
1419
|
+
self.tableProperties = {}
|
|
1420
|
+
(_ktype83, _vtype84, _size82) = iprot.readMapBegin()
|
|
1421
|
+
for _i86 in range(_size82):
|
|
1422
|
+
_key87 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1423
|
+
_val88 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1424
|
+
self.tableProperties[_key87] = _val88
|
|
1425
|
+
iprot.readMapEnd()
|
|
1426
|
+
else:
|
|
1427
|
+
iprot.skip(ftype)
|
|
1428
|
+
elif fid == 20:
|
|
1429
|
+
if ftype == TType.MAP:
|
|
1430
|
+
self.tags = {}
|
|
1431
|
+
(_ktype90, _vtype91, _size89) = iprot.readMapBegin()
|
|
1432
|
+
for _i93 in range(_size89):
|
|
1433
|
+
_key94 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1434
|
+
_val95 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1435
|
+
self.tags[_key94] = _val95
|
|
1436
|
+
iprot.readMapEnd()
|
|
1437
|
+
else:
|
|
1438
|
+
iprot.skip(ftype)
|
|
1439
|
+
elif fid == 21:
|
|
1440
|
+
if ftype == TType.MAP:
|
|
1441
|
+
self.columnTags = {}
|
|
1442
|
+
(_ktype97, _vtype98, _size96) = iprot.readMapBegin()
|
|
1443
|
+
for _i100 in range(_size96):
|
|
1444
|
+
_key101 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1445
|
+
_val102 = {}
|
|
1446
|
+
(_ktype104, _vtype105, _size103) = iprot.readMapBegin()
|
|
1447
|
+
for _i107 in range(_size103):
|
|
1448
|
+
_key108 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1449
|
+
_val109 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1450
|
+
_val102[_key108] = _val109
|
|
1451
|
+
iprot.readMapEnd()
|
|
1452
|
+
self.columnTags[_key101] = _val102
|
|
1453
|
+
iprot.readMapEnd()
|
|
1454
|
+
else:
|
|
1455
|
+
iprot.skip(ftype)
|
|
1456
|
+
elif fid == 100:
|
|
1457
|
+
if ftype == TType.BOOL:
|
|
1458
|
+
self.online = iprot.readBool()
|
|
1459
|
+
else:
|
|
1460
|
+
iprot.skip(ftype)
|
|
1461
|
+
elif fid == 101:
|
|
1462
|
+
if ftype == TType.BOOL:
|
|
1463
|
+
self.production = iprot.readBool()
|
|
1464
|
+
else:
|
|
1465
|
+
iprot.skip(ftype)
|
|
1466
|
+
elif fid == 102:
|
|
1467
|
+
if ftype == TType.STRING:
|
|
1468
|
+
self.sourceFile = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1469
|
+
else:
|
|
1470
|
+
iprot.skip(ftype)
|
|
1471
|
+
elif fid == 103:
|
|
1472
|
+
if ftype == TType.STRING:
|
|
1473
|
+
self.customJson = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1474
|
+
else:
|
|
1475
|
+
iprot.skip(ftype)
|
|
1476
|
+
elif fid == 200:
|
|
1477
|
+
if ftype == TType.BOOL:
|
|
1478
|
+
self.consistencyCheck = iprot.readBool()
|
|
1479
|
+
else:
|
|
1480
|
+
iprot.skip(ftype)
|
|
1481
|
+
elif fid == 201:
|
|
1482
|
+
if ftype == TType.DOUBLE:
|
|
1483
|
+
self.samplePercent = iprot.readDouble()
|
|
1484
|
+
else:
|
|
1485
|
+
iprot.skip(ftype)
|
|
1486
|
+
elif fid == 202:
|
|
1487
|
+
if ftype == TType.DOUBLE:
|
|
1488
|
+
self.consistencySamplePercent = iprot.readDouble()
|
|
1489
|
+
else:
|
|
1490
|
+
iprot.skip(ftype)
|
|
1491
|
+
elif fid == 203:
|
|
1492
|
+
if ftype == TType.STRUCT:
|
|
1493
|
+
self.driftSpec = ai.chronon.observability.ttypes.DriftSpec()
|
|
1494
|
+
self.driftSpec.read(iprot)
|
|
1495
|
+
else:
|
|
1496
|
+
iprot.skip(ftype)
|
|
1497
|
+
elif fid == 204:
|
|
1498
|
+
if ftype == TType.STRUCT:
|
|
1499
|
+
self.executionInfo = ai.chronon.api.common.ttypes.ExecutionInfo()
|
|
1500
|
+
self.executionInfo.read(iprot)
|
|
1501
|
+
else:
|
|
1502
|
+
iprot.skip(ftype)
|
|
1503
|
+
else:
|
|
1504
|
+
iprot.skip(ftype)
|
|
1505
|
+
iprot.readFieldEnd()
|
|
1506
|
+
iprot.readStructEnd()
|
|
1507
|
+
|
|
1508
|
+
def write(self, oprot):
|
|
1509
|
+
self.validate()
|
|
1510
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
1511
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
1512
|
+
return
|
|
1513
|
+
oprot.writeStructBegin('MetaData')
|
|
1514
|
+
if self.name is not None:
|
|
1515
|
+
oprot.writeFieldBegin('name', TType.STRING, 1)
|
|
1516
|
+
oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name)
|
|
1517
|
+
oprot.writeFieldEnd()
|
|
1518
|
+
if self.team is not None:
|
|
1519
|
+
oprot.writeFieldBegin('team', TType.STRING, 2)
|
|
1520
|
+
oprot.writeString(self.team.encode('utf-8') if sys.version_info[0] == 2 else self.team)
|
|
1521
|
+
oprot.writeFieldEnd()
|
|
1522
|
+
if self.version is not None:
|
|
1523
|
+
oprot.writeFieldBegin('version', TType.STRING, 3)
|
|
1524
|
+
oprot.writeString(self.version.encode('utf-8') if sys.version_info[0] == 2 else self.version)
|
|
1525
|
+
oprot.writeFieldEnd()
|
|
1526
|
+
if self.outputNamespace is not None:
|
|
1527
|
+
oprot.writeFieldBegin('outputNamespace', TType.STRING, 4)
|
|
1528
|
+
oprot.writeString(self.outputNamespace.encode('utf-8') if sys.version_info[0] == 2 else self.outputNamespace)
|
|
1529
|
+
oprot.writeFieldEnd()
|
|
1530
|
+
if self.additionalOutputPartitionColumns is not None:
|
|
1531
|
+
oprot.writeFieldBegin('additionalOutputPartitionColumns', TType.LIST, 5)
|
|
1532
|
+
oprot.writeListBegin(TType.STRING, len(self.additionalOutputPartitionColumns))
|
|
1533
|
+
for iter110 in self.additionalOutputPartitionColumns:
|
|
1534
|
+
oprot.writeString(iter110.encode('utf-8') if sys.version_info[0] == 2 else iter110)
|
|
1535
|
+
oprot.writeListEnd()
|
|
1536
|
+
oprot.writeFieldEnd()
|
|
1537
|
+
if self.tableProperties is not None:
|
|
1538
|
+
oprot.writeFieldBegin('tableProperties', TType.MAP, 6)
|
|
1539
|
+
oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.tableProperties))
|
|
1540
|
+
for kiter111, viter112 in self.tableProperties.items():
|
|
1541
|
+
oprot.writeString(kiter111.encode('utf-8') if sys.version_info[0] == 2 else kiter111)
|
|
1542
|
+
oprot.writeString(viter112.encode('utf-8') if sys.version_info[0] == 2 else viter112)
|
|
1543
|
+
oprot.writeMapEnd()
|
|
1544
|
+
oprot.writeFieldEnd()
|
|
1545
|
+
if self.tags is not None:
|
|
1546
|
+
oprot.writeFieldBegin('tags', TType.MAP, 20)
|
|
1547
|
+
oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.tags))
|
|
1548
|
+
for kiter113, viter114 in self.tags.items():
|
|
1549
|
+
oprot.writeString(kiter113.encode('utf-8') if sys.version_info[0] == 2 else kiter113)
|
|
1550
|
+
oprot.writeString(viter114.encode('utf-8') if sys.version_info[0] == 2 else viter114)
|
|
1551
|
+
oprot.writeMapEnd()
|
|
1552
|
+
oprot.writeFieldEnd()
|
|
1553
|
+
if self.columnTags is not None:
|
|
1554
|
+
oprot.writeFieldBegin('columnTags', TType.MAP, 21)
|
|
1555
|
+
oprot.writeMapBegin(TType.STRING, TType.MAP, len(self.columnTags))
|
|
1556
|
+
for kiter115, viter116 in self.columnTags.items():
|
|
1557
|
+
oprot.writeString(kiter115.encode('utf-8') if sys.version_info[0] == 2 else kiter115)
|
|
1558
|
+
oprot.writeMapBegin(TType.STRING, TType.STRING, len(viter116))
|
|
1559
|
+
for kiter117, viter118 in viter116.items():
|
|
1560
|
+
oprot.writeString(kiter117.encode('utf-8') if sys.version_info[0] == 2 else kiter117)
|
|
1561
|
+
oprot.writeString(viter118.encode('utf-8') if sys.version_info[0] == 2 else viter118)
|
|
1562
|
+
oprot.writeMapEnd()
|
|
1563
|
+
oprot.writeMapEnd()
|
|
1564
|
+
oprot.writeFieldEnd()
|
|
1565
|
+
if self.online is not None:
|
|
1566
|
+
oprot.writeFieldBegin('online', TType.BOOL, 100)
|
|
1567
|
+
oprot.writeBool(self.online)
|
|
1568
|
+
oprot.writeFieldEnd()
|
|
1569
|
+
if self.production is not None:
|
|
1570
|
+
oprot.writeFieldBegin('production', TType.BOOL, 101)
|
|
1571
|
+
oprot.writeBool(self.production)
|
|
1572
|
+
oprot.writeFieldEnd()
|
|
1573
|
+
if self.sourceFile is not None:
|
|
1574
|
+
oprot.writeFieldBegin('sourceFile', TType.STRING, 102)
|
|
1575
|
+
oprot.writeString(self.sourceFile.encode('utf-8') if sys.version_info[0] == 2 else self.sourceFile)
|
|
1576
|
+
oprot.writeFieldEnd()
|
|
1577
|
+
if self.customJson is not None:
|
|
1578
|
+
oprot.writeFieldBegin('customJson', TType.STRING, 103)
|
|
1579
|
+
oprot.writeString(self.customJson.encode('utf-8') if sys.version_info[0] == 2 else self.customJson)
|
|
1580
|
+
oprot.writeFieldEnd()
|
|
1581
|
+
if self.consistencyCheck is not None:
|
|
1582
|
+
oprot.writeFieldBegin('consistencyCheck', TType.BOOL, 200)
|
|
1583
|
+
oprot.writeBool(self.consistencyCheck)
|
|
1584
|
+
oprot.writeFieldEnd()
|
|
1585
|
+
if self.samplePercent is not None:
|
|
1586
|
+
oprot.writeFieldBegin('samplePercent', TType.DOUBLE, 201)
|
|
1587
|
+
oprot.writeDouble(self.samplePercent)
|
|
1588
|
+
oprot.writeFieldEnd()
|
|
1589
|
+
if self.consistencySamplePercent is not None:
|
|
1590
|
+
oprot.writeFieldBegin('consistencySamplePercent', TType.DOUBLE, 202)
|
|
1591
|
+
oprot.writeDouble(self.consistencySamplePercent)
|
|
1592
|
+
oprot.writeFieldEnd()
|
|
1593
|
+
if self.driftSpec is not None:
|
|
1594
|
+
oprot.writeFieldBegin('driftSpec', TType.STRUCT, 203)
|
|
1595
|
+
self.driftSpec.write(oprot)
|
|
1596
|
+
oprot.writeFieldEnd()
|
|
1597
|
+
if self.executionInfo is not None:
|
|
1598
|
+
oprot.writeFieldBegin('executionInfo', TType.STRUCT, 204)
|
|
1599
|
+
self.executionInfo.write(oprot)
|
|
1600
|
+
oprot.writeFieldEnd()
|
|
1601
|
+
oprot.writeFieldStop()
|
|
1602
|
+
oprot.writeStructEnd()
|
|
1603
|
+
|
|
1604
|
+
def validate(self):
|
|
1605
|
+
return
|
|
1606
|
+
|
|
1607
|
+
def __repr__(self):
|
|
1608
|
+
L = ['%s=%r' % (key, value)
|
|
1609
|
+
for key, value in self.__dict__.items()]
|
|
1610
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
1611
|
+
|
|
1612
|
+
def __eq__(self, other):
|
|
1613
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
1614
|
+
|
|
1615
|
+
def __ne__(self, other):
|
|
1616
|
+
return not (self == other)
|
|
1617
|
+
|
|
1618
|
+
|
|
1619
|
+
class GroupBy(object):
|
|
1620
|
+
"""
|
|
1621
|
+
Attributes:
|
|
1622
|
+
- metaData
|
|
1623
|
+
- sources
|
|
1624
|
+
- keyColumns
|
|
1625
|
+
- aggregations
|
|
1626
|
+
- accuracy
|
|
1627
|
+
- backfillStartDate
|
|
1628
|
+
- derivations
|
|
1629
|
+
|
|
1630
|
+
"""
|
|
1631
|
+
thrift_spec = None
|
|
1632
|
+
|
|
1633
|
+
|
|
1634
|
+
def __init__(self, metaData = None, sources = None, keyColumns = None, aggregations = None, accuracy = None, backfillStartDate = None, derivations = None,):
|
|
1635
|
+
self.metaData = metaData
|
|
1636
|
+
self.sources = sources
|
|
1637
|
+
self.keyColumns = keyColumns
|
|
1638
|
+
self.aggregations = aggregations
|
|
1639
|
+
self.accuracy = accuracy
|
|
1640
|
+
self.backfillStartDate = backfillStartDate
|
|
1641
|
+
self.derivations = derivations
|
|
1642
|
+
|
|
1643
|
+
def read(self, iprot):
|
|
1644
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
1645
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
1646
|
+
return
|
|
1647
|
+
iprot.readStructBegin()
|
|
1648
|
+
while True:
|
|
1649
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
1650
|
+
if ftype == TType.STOP:
|
|
1651
|
+
break
|
|
1652
|
+
if fid == 1:
|
|
1653
|
+
if ftype == TType.STRUCT:
|
|
1654
|
+
self.metaData = MetaData()
|
|
1655
|
+
self.metaData.read(iprot)
|
|
1656
|
+
else:
|
|
1657
|
+
iprot.skip(ftype)
|
|
1658
|
+
elif fid == 2:
|
|
1659
|
+
if ftype == TType.LIST:
|
|
1660
|
+
self.sources = []
|
|
1661
|
+
(_etype122, _size119) = iprot.readListBegin()
|
|
1662
|
+
for _i123 in range(_size119):
|
|
1663
|
+
_elem124 = Source()
|
|
1664
|
+
_elem124.read(iprot)
|
|
1665
|
+
self.sources.append(_elem124)
|
|
1666
|
+
iprot.readListEnd()
|
|
1667
|
+
else:
|
|
1668
|
+
iprot.skip(ftype)
|
|
1669
|
+
elif fid == 3:
|
|
1670
|
+
if ftype == TType.LIST:
|
|
1671
|
+
self.keyColumns = []
|
|
1672
|
+
(_etype128, _size125) = iprot.readListBegin()
|
|
1673
|
+
for _i129 in range(_size125):
|
|
1674
|
+
_elem130 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1675
|
+
self.keyColumns.append(_elem130)
|
|
1676
|
+
iprot.readListEnd()
|
|
1677
|
+
else:
|
|
1678
|
+
iprot.skip(ftype)
|
|
1679
|
+
elif fid == 4:
|
|
1680
|
+
if ftype == TType.LIST:
|
|
1681
|
+
self.aggregations = []
|
|
1682
|
+
(_etype134, _size131) = iprot.readListBegin()
|
|
1683
|
+
for _i135 in range(_size131):
|
|
1684
|
+
_elem136 = Aggregation()
|
|
1685
|
+
_elem136.read(iprot)
|
|
1686
|
+
self.aggregations.append(_elem136)
|
|
1687
|
+
iprot.readListEnd()
|
|
1688
|
+
else:
|
|
1689
|
+
iprot.skip(ftype)
|
|
1690
|
+
elif fid == 5:
|
|
1691
|
+
if ftype == TType.I32:
|
|
1692
|
+
self.accuracy = iprot.readI32()
|
|
1693
|
+
else:
|
|
1694
|
+
iprot.skip(ftype)
|
|
1695
|
+
elif fid == 6:
|
|
1696
|
+
if ftype == TType.STRING:
|
|
1697
|
+
self.backfillStartDate = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1698
|
+
else:
|
|
1699
|
+
iprot.skip(ftype)
|
|
1700
|
+
elif fid == 7:
|
|
1701
|
+
if ftype == TType.LIST:
|
|
1702
|
+
self.derivations = []
|
|
1703
|
+
(_etype140, _size137) = iprot.readListBegin()
|
|
1704
|
+
for _i141 in range(_size137):
|
|
1705
|
+
_elem142 = Derivation()
|
|
1706
|
+
_elem142.read(iprot)
|
|
1707
|
+
self.derivations.append(_elem142)
|
|
1708
|
+
iprot.readListEnd()
|
|
1709
|
+
else:
|
|
1710
|
+
iprot.skip(ftype)
|
|
1711
|
+
else:
|
|
1712
|
+
iprot.skip(ftype)
|
|
1713
|
+
iprot.readFieldEnd()
|
|
1714
|
+
iprot.readStructEnd()
|
|
1715
|
+
|
|
1716
|
+
def write(self, oprot):
|
|
1717
|
+
self.validate()
|
|
1718
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
1719
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
1720
|
+
return
|
|
1721
|
+
oprot.writeStructBegin('GroupBy')
|
|
1722
|
+
if self.metaData is not None:
|
|
1723
|
+
oprot.writeFieldBegin('metaData', TType.STRUCT, 1)
|
|
1724
|
+
self.metaData.write(oprot)
|
|
1725
|
+
oprot.writeFieldEnd()
|
|
1726
|
+
if self.sources is not None:
|
|
1727
|
+
oprot.writeFieldBegin('sources', TType.LIST, 2)
|
|
1728
|
+
oprot.writeListBegin(TType.STRUCT, len(self.sources))
|
|
1729
|
+
for iter143 in self.sources:
|
|
1730
|
+
iter143.write(oprot)
|
|
1731
|
+
oprot.writeListEnd()
|
|
1732
|
+
oprot.writeFieldEnd()
|
|
1733
|
+
if self.keyColumns is not None:
|
|
1734
|
+
oprot.writeFieldBegin('keyColumns', TType.LIST, 3)
|
|
1735
|
+
oprot.writeListBegin(TType.STRING, len(self.keyColumns))
|
|
1736
|
+
for iter144 in self.keyColumns:
|
|
1737
|
+
oprot.writeString(iter144.encode('utf-8') if sys.version_info[0] == 2 else iter144)
|
|
1738
|
+
oprot.writeListEnd()
|
|
1739
|
+
oprot.writeFieldEnd()
|
|
1740
|
+
if self.aggregations is not None:
|
|
1741
|
+
oprot.writeFieldBegin('aggregations', TType.LIST, 4)
|
|
1742
|
+
oprot.writeListBegin(TType.STRUCT, len(self.aggregations))
|
|
1743
|
+
for iter145 in self.aggregations:
|
|
1744
|
+
iter145.write(oprot)
|
|
1745
|
+
oprot.writeListEnd()
|
|
1746
|
+
oprot.writeFieldEnd()
|
|
1747
|
+
if self.accuracy is not None:
|
|
1748
|
+
oprot.writeFieldBegin('accuracy', TType.I32, 5)
|
|
1749
|
+
oprot.writeI32(self.accuracy)
|
|
1750
|
+
oprot.writeFieldEnd()
|
|
1751
|
+
if self.backfillStartDate is not None:
|
|
1752
|
+
oprot.writeFieldBegin('backfillStartDate', TType.STRING, 6)
|
|
1753
|
+
oprot.writeString(self.backfillStartDate.encode('utf-8') if sys.version_info[0] == 2 else self.backfillStartDate)
|
|
1754
|
+
oprot.writeFieldEnd()
|
|
1755
|
+
if self.derivations is not None:
|
|
1756
|
+
oprot.writeFieldBegin('derivations', TType.LIST, 7)
|
|
1757
|
+
oprot.writeListBegin(TType.STRUCT, len(self.derivations))
|
|
1758
|
+
for iter146 in self.derivations:
|
|
1759
|
+
iter146.write(oprot)
|
|
1760
|
+
oprot.writeListEnd()
|
|
1761
|
+
oprot.writeFieldEnd()
|
|
1762
|
+
oprot.writeFieldStop()
|
|
1763
|
+
oprot.writeStructEnd()
|
|
1764
|
+
|
|
1765
|
+
def validate(self):
|
|
1766
|
+
return
|
|
1767
|
+
|
|
1768
|
+
def __repr__(self):
|
|
1769
|
+
L = ['%s=%r' % (key, value)
|
|
1770
|
+
for key, value in self.__dict__.items()]
|
|
1771
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
1772
|
+
|
|
1773
|
+
def __eq__(self, other):
|
|
1774
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
1775
|
+
|
|
1776
|
+
def __ne__(self, other):
|
|
1777
|
+
return not (self == other)
|
|
1778
|
+
|
|
1779
|
+
|
|
1780
|
+
class JoinPart(object):
|
|
1781
|
+
"""
|
|
1782
|
+
Attributes:
|
|
1783
|
+
- groupBy
|
|
1784
|
+
- keyMapping
|
|
1785
|
+
- prefix
|
|
1786
|
+
- useLongNames
|
|
1787
|
+
|
|
1788
|
+
"""
|
|
1789
|
+
thrift_spec = None
|
|
1790
|
+
|
|
1791
|
+
|
|
1792
|
+
def __init__(self, groupBy = None, keyMapping = None, prefix = None, useLongNames = None,):
|
|
1793
|
+
self.groupBy = groupBy
|
|
1794
|
+
self.keyMapping = keyMapping
|
|
1795
|
+
self.prefix = prefix
|
|
1796
|
+
self.useLongNames = useLongNames
|
|
1797
|
+
|
|
1798
|
+
def read(self, iprot):
|
|
1799
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
1800
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
1801
|
+
return
|
|
1802
|
+
iprot.readStructBegin()
|
|
1803
|
+
while True:
|
|
1804
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
1805
|
+
if ftype == TType.STOP:
|
|
1806
|
+
break
|
|
1807
|
+
if fid == 1:
|
|
1808
|
+
if ftype == TType.STRUCT:
|
|
1809
|
+
self.groupBy = GroupBy()
|
|
1810
|
+
self.groupBy.read(iprot)
|
|
1811
|
+
else:
|
|
1812
|
+
iprot.skip(ftype)
|
|
1813
|
+
elif fid == 2:
|
|
1814
|
+
if ftype == TType.MAP:
|
|
1815
|
+
self.keyMapping = {}
|
|
1816
|
+
(_ktype148, _vtype149, _size147) = iprot.readMapBegin()
|
|
1817
|
+
for _i151 in range(_size147):
|
|
1818
|
+
_key152 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1819
|
+
_val153 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1820
|
+
self.keyMapping[_key152] = _val153
|
|
1821
|
+
iprot.readMapEnd()
|
|
1822
|
+
else:
|
|
1823
|
+
iprot.skip(ftype)
|
|
1824
|
+
elif fid == 3:
|
|
1825
|
+
if ftype == TType.STRING:
|
|
1826
|
+
self.prefix = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1827
|
+
else:
|
|
1828
|
+
iprot.skip(ftype)
|
|
1829
|
+
elif fid == 10:
|
|
1830
|
+
if ftype == TType.BOOL:
|
|
1831
|
+
self.useLongNames = iprot.readBool()
|
|
1832
|
+
else:
|
|
1833
|
+
iprot.skip(ftype)
|
|
1834
|
+
else:
|
|
1835
|
+
iprot.skip(ftype)
|
|
1836
|
+
iprot.readFieldEnd()
|
|
1837
|
+
iprot.readStructEnd()
|
|
1838
|
+
|
|
1839
|
+
def write(self, oprot):
|
|
1840
|
+
self.validate()
|
|
1841
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
1842
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
1843
|
+
return
|
|
1844
|
+
oprot.writeStructBegin('JoinPart')
|
|
1845
|
+
if self.groupBy is not None:
|
|
1846
|
+
oprot.writeFieldBegin('groupBy', TType.STRUCT, 1)
|
|
1847
|
+
self.groupBy.write(oprot)
|
|
1848
|
+
oprot.writeFieldEnd()
|
|
1849
|
+
if self.keyMapping is not None:
|
|
1850
|
+
oprot.writeFieldBegin('keyMapping', TType.MAP, 2)
|
|
1851
|
+
oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.keyMapping))
|
|
1852
|
+
for kiter154, viter155 in self.keyMapping.items():
|
|
1853
|
+
oprot.writeString(kiter154.encode('utf-8') if sys.version_info[0] == 2 else kiter154)
|
|
1854
|
+
oprot.writeString(viter155.encode('utf-8') if sys.version_info[0] == 2 else viter155)
|
|
1855
|
+
oprot.writeMapEnd()
|
|
1856
|
+
oprot.writeFieldEnd()
|
|
1857
|
+
if self.prefix is not None:
|
|
1858
|
+
oprot.writeFieldBegin('prefix', TType.STRING, 3)
|
|
1859
|
+
oprot.writeString(self.prefix.encode('utf-8') if sys.version_info[0] == 2 else self.prefix)
|
|
1860
|
+
oprot.writeFieldEnd()
|
|
1861
|
+
if self.useLongNames is not None:
|
|
1862
|
+
oprot.writeFieldBegin('useLongNames', TType.BOOL, 10)
|
|
1863
|
+
oprot.writeBool(self.useLongNames)
|
|
1864
|
+
oprot.writeFieldEnd()
|
|
1865
|
+
oprot.writeFieldStop()
|
|
1866
|
+
oprot.writeStructEnd()
|
|
1867
|
+
|
|
1868
|
+
def validate(self):
|
|
1869
|
+
return
|
|
1870
|
+
|
|
1871
|
+
def __repr__(self):
|
|
1872
|
+
L = ['%s=%r' % (key, value)
|
|
1873
|
+
for key, value in self.__dict__.items()]
|
|
1874
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
1875
|
+
|
|
1876
|
+
def __eq__(self, other):
|
|
1877
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
1878
|
+
|
|
1879
|
+
def __ne__(self, other):
|
|
1880
|
+
return not (self == other)
|
|
1881
|
+
|
|
1882
|
+
|
|
1883
|
+
class ExternalPart(object):
|
|
1884
|
+
"""
|
|
1885
|
+
Attributes:
|
|
1886
|
+
- source
|
|
1887
|
+
- keyMapping
|
|
1888
|
+
- prefix
|
|
1889
|
+
|
|
1890
|
+
"""
|
|
1891
|
+
thrift_spec = None
|
|
1892
|
+
|
|
1893
|
+
|
|
1894
|
+
def __init__(self, source = None, keyMapping = None, prefix = None,):
|
|
1895
|
+
self.source = source
|
|
1896
|
+
self.keyMapping = keyMapping
|
|
1897
|
+
self.prefix = prefix
|
|
1898
|
+
|
|
1899
|
+
def read(self, iprot):
|
|
1900
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
1901
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
1902
|
+
return
|
|
1903
|
+
iprot.readStructBegin()
|
|
1904
|
+
while True:
|
|
1905
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
1906
|
+
if ftype == TType.STOP:
|
|
1907
|
+
break
|
|
1908
|
+
if fid == 1:
|
|
1909
|
+
if ftype == TType.STRUCT:
|
|
1910
|
+
self.source = ExternalSource()
|
|
1911
|
+
self.source.read(iprot)
|
|
1912
|
+
else:
|
|
1913
|
+
iprot.skip(ftype)
|
|
1914
|
+
elif fid == 2:
|
|
1915
|
+
if ftype == TType.MAP:
|
|
1916
|
+
self.keyMapping = {}
|
|
1917
|
+
(_ktype157, _vtype158, _size156) = iprot.readMapBegin()
|
|
1918
|
+
for _i160 in range(_size156):
|
|
1919
|
+
_key161 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1920
|
+
_val162 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1921
|
+
self.keyMapping[_key161] = _val162
|
|
1922
|
+
iprot.readMapEnd()
|
|
1923
|
+
else:
|
|
1924
|
+
iprot.skip(ftype)
|
|
1925
|
+
elif fid == 3:
|
|
1926
|
+
if ftype == TType.STRING:
|
|
1927
|
+
self.prefix = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1928
|
+
else:
|
|
1929
|
+
iprot.skip(ftype)
|
|
1930
|
+
else:
|
|
1931
|
+
iprot.skip(ftype)
|
|
1932
|
+
iprot.readFieldEnd()
|
|
1933
|
+
iprot.readStructEnd()
|
|
1934
|
+
|
|
1935
|
+
def write(self, oprot):
|
|
1936
|
+
self.validate()
|
|
1937
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
1938
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
1939
|
+
return
|
|
1940
|
+
oprot.writeStructBegin('ExternalPart')
|
|
1941
|
+
if self.source is not None:
|
|
1942
|
+
oprot.writeFieldBegin('source', TType.STRUCT, 1)
|
|
1943
|
+
self.source.write(oprot)
|
|
1944
|
+
oprot.writeFieldEnd()
|
|
1945
|
+
if self.keyMapping is not None:
|
|
1946
|
+
oprot.writeFieldBegin('keyMapping', TType.MAP, 2)
|
|
1947
|
+
oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.keyMapping))
|
|
1948
|
+
for kiter163, viter164 in self.keyMapping.items():
|
|
1949
|
+
oprot.writeString(kiter163.encode('utf-8') if sys.version_info[0] == 2 else kiter163)
|
|
1950
|
+
oprot.writeString(viter164.encode('utf-8') if sys.version_info[0] == 2 else viter164)
|
|
1951
|
+
oprot.writeMapEnd()
|
|
1952
|
+
oprot.writeFieldEnd()
|
|
1953
|
+
if self.prefix is not None:
|
|
1954
|
+
oprot.writeFieldBegin('prefix', TType.STRING, 3)
|
|
1955
|
+
oprot.writeString(self.prefix.encode('utf-8') if sys.version_info[0] == 2 else self.prefix)
|
|
1956
|
+
oprot.writeFieldEnd()
|
|
1957
|
+
oprot.writeFieldStop()
|
|
1958
|
+
oprot.writeStructEnd()
|
|
1959
|
+
|
|
1960
|
+
def validate(self):
|
|
1961
|
+
return
|
|
1962
|
+
|
|
1963
|
+
def __repr__(self):
|
|
1964
|
+
L = ['%s=%r' % (key, value)
|
|
1965
|
+
for key, value in self.__dict__.items()]
|
|
1966
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
1967
|
+
|
|
1968
|
+
def __eq__(self, other):
|
|
1969
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
1970
|
+
|
|
1971
|
+
def __ne__(self, other):
|
|
1972
|
+
return not (self == other)
|
|
1973
|
+
|
|
1974
|
+
|
|
1975
|
+
class Derivation(object):
|
|
1976
|
+
"""
|
|
1977
|
+
Attributes:
|
|
1978
|
+
- name
|
|
1979
|
+
- expression
|
|
1980
|
+
|
|
1981
|
+
"""
|
|
1982
|
+
thrift_spec = None
|
|
1983
|
+
|
|
1984
|
+
|
|
1985
|
+
def __init__(self, name = None, expression = None,):
|
|
1986
|
+
self.name = name
|
|
1987
|
+
self.expression = expression
|
|
1988
|
+
|
|
1989
|
+
def read(self, iprot):
|
|
1990
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
1991
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
1992
|
+
return
|
|
1993
|
+
iprot.readStructBegin()
|
|
1994
|
+
while True:
|
|
1995
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
1996
|
+
if ftype == TType.STOP:
|
|
1997
|
+
break
|
|
1998
|
+
if fid == 1:
|
|
1999
|
+
if ftype == TType.STRING:
|
|
2000
|
+
self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2001
|
+
else:
|
|
2002
|
+
iprot.skip(ftype)
|
|
2003
|
+
elif fid == 2:
|
|
2004
|
+
if ftype == TType.STRING:
|
|
2005
|
+
self.expression = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2006
|
+
else:
|
|
2007
|
+
iprot.skip(ftype)
|
|
2008
|
+
else:
|
|
2009
|
+
iprot.skip(ftype)
|
|
2010
|
+
iprot.readFieldEnd()
|
|
2011
|
+
iprot.readStructEnd()
|
|
2012
|
+
|
|
2013
|
+
def write(self, oprot):
|
|
2014
|
+
self.validate()
|
|
2015
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
2016
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
2017
|
+
return
|
|
2018
|
+
oprot.writeStructBegin('Derivation')
|
|
2019
|
+
if self.name is not None:
|
|
2020
|
+
oprot.writeFieldBegin('name', TType.STRING, 1)
|
|
2021
|
+
oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name)
|
|
2022
|
+
oprot.writeFieldEnd()
|
|
2023
|
+
if self.expression is not None:
|
|
2024
|
+
oprot.writeFieldBegin('expression', TType.STRING, 2)
|
|
2025
|
+
oprot.writeString(self.expression.encode('utf-8') if sys.version_info[0] == 2 else self.expression)
|
|
2026
|
+
oprot.writeFieldEnd()
|
|
2027
|
+
oprot.writeFieldStop()
|
|
2028
|
+
oprot.writeStructEnd()
|
|
2029
|
+
|
|
2030
|
+
def validate(self):
|
|
2031
|
+
return
|
|
2032
|
+
|
|
2033
|
+
def __repr__(self):
|
|
2034
|
+
L = ['%s=%r' % (key, value)
|
|
2035
|
+
for key, value in self.__dict__.items()]
|
|
2036
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
2037
|
+
|
|
2038
|
+
def __eq__(self, other):
|
|
2039
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
2040
|
+
|
|
2041
|
+
def __ne__(self, other):
|
|
2042
|
+
return not (self == other)
|
|
2043
|
+
|
|
2044
|
+
|
|
2045
|
+
class Join(object):
|
|
2046
|
+
"""
|
|
2047
|
+
Attributes:
|
|
2048
|
+
- metaData
|
|
2049
|
+
- left
|
|
2050
|
+
- joinParts
|
|
2051
|
+
- skewKeys
|
|
2052
|
+
- onlineExternalParts
|
|
2053
|
+
- labelParts
|
|
2054
|
+
- bootstrapParts
|
|
2055
|
+
- rowIds
|
|
2056
|
+
- derivations: List of a derived column names to the expression based on joinPart / externalPart columns
|
|
2057
|
+
The expression can be any valid Spark SQL select clause without aggregation functions.
|
|
2058
|
+
|
|
2059
|
+
joinPart column names are automatically constructed according to the below convention
|
|
2060
|
+
`{join_part_prefix}_{group_by_name}_{input_column_name}_{aggregation_operation}_{window}_{by_bucket}`
|
|
2061
|
+
prefix, window and bucket are optional. You can find the type information of columns using the analyzer tool.
|
|
2062
|
+
|
|
2063
|
+
externalPart column names are automatically constructed according to the below convention
|
|
2064
|
+
`ext_{external_source_name}_{value_column}`
|
|
2065
|
+
Types are defined along with the schema by users for external sources.
|
|
2066
|
+
|
|
2067
|
+
Including a column with key "*" and value "*", means that every raw column will be included along with the derived
|
|
2068
|
+
columns.
|
|
2069
|
+
|
|
2070
|
+
- useLongNames
|
|
2071
|
+
|
|
2072
|
+
"""
|
|
2073
|
+
thrift_spec = None
|
|
2074
|
+
|
|
2075
|
+
|
|
2076
|
+
def __init__(self, metaData = None, left = None, joinParts = None, skewKeys = None, onlineExternalParts = None, labelParts = None, bootstrapParts = None, rowIds = None, derivations = None, useLongNames = None,):
|
|
2077
|
+
self.metaData = metaData
|
|
2078
|
+
self.left = left
|
|
2079
|
+
self.joinParts = joinParts
|
|
2080
|
+
self.skewKeys = skewKeys
|
|
2081
|
+
self.onlineExternalParts = onlineExternalParts
|
|
2082
|
+
self.labelParts = labelParts
|
|
2083
|
+
self.bootstrapParts = bootstrapParts
|
|
2084
|
+
self.rowIds = rowIds
|
|
2085
|
+
self.derivations = derivations
|
|
2086
|
+
self.useLongNames = useLongNames
|
|
2087
|
+
|
|
2088
|
+
def read(self, iprot):
|
|
2089
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
2090
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
2091
|
+
return
|
|
2092
|
+
iprot.readStructBegin()
|
|
2093
|
+
while True:
|
|
2094
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
2095
|
+
if ftype == TType.STOP:
|
|
2096
|
+
break
|
|
2097
|
+
if fid == 1:
|
|
2098
|
+
if ftype == TType.STRUCT:
|
|
2099
|
+
self.metaData = MetaData()
|
|
2100
|
+
self.metaData.read(iprot)
|
|
2101
|
+
else:
|
|
2102
|
+
iprot.skip(ftype)
|
|
2103
|
+
elif fid == 2:
|
|
2104
|
+
if ftype == TType.STRUCT:
|
|
2105
|
+
self.left = Source()
|
|
2106
|
+
self.left.read(iprot)
|
|
2107
|
+
else:
|
|
2108
|
+
iprot.skip(ftype)
|
|
2109
|
+
elif fid == 3:
|
|
2110
|
+
if ftype == TType.LIST:
|
|
2111
|
+
self.joinParts = []
|
|
2112
|
+
(_etype168, _size165) = iprot.readListBegin()
|
|
2113
|
+
for _i169 in range(_size165):
|
|
2114
|
+
_elem170 = JoinPart()
|
|
2115
|
+
_elem170.read(iprot)
|
|
2116
|
+
self.joinParts.append(_elem170)
|
|
2117
|
+
iprot.readListEnd()
|
|
2118
|
+
else:
|
|
2119
|
+
iprot.skip(ftype)
|
|
2120
|
+
elif fid == 4:
|
|
2121
|
+
if ftype == TType.MAP:
|
|
2122
|
+
self.skewKeys = {}
|
|
2123
|
+
(_ktype172, _vtype173, _size171) = iprot.readMapBegin()
|
|
2124
|
+
for _i175 in range(_size171):
|
|
2125
|
+
_key176 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2126
|
+
_val177 = []
|
|
2127
|
+
(_etype181, _size178) = iprot.readListBegin()
|
|
2128
|
+
for _i182 in range(_size178):
|
|
2129
|
+
_elem183 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2130
|
+
_val177.append(_elem183)
|
|
2131
|
+
iprot.readListEnd()
|
|
2132
|
+
self.skewKeys[_key176] = _val177
|
|
2133
|
+
iprot.readMapEnd()
|
|
2134
|
+
else:
|
|
2135
|
+
iprot.skip(ftype)
|
|
2136
|
+
elif fid == 5:
|
|
2137
|
+
if ftype == TType.LIST:
|
|
2138
|
+
self.onlineExternalParts = []
|
|
2139
|
+
(_etype187, _size184) = iprot.readListBegin()
|
|
2140
|
+
for _i188 in range(_size184):
|
|
2141
|
+
_elem189 = ExternalPart()
|
|
2142
|
+
_elem189.read(iprot)
|
|
2143
|
+
self.onlineExternalParts.append(_elem189)
|
|
2144
|
+
iprot.readListEnd()
|
|
2145
|
+
else:
|
|
2146
|
+
iprot.skip(ftype)
|
|
2147
|
+
elif fid == 6:
|
|
2148
|
+
if ftype == TType.STRUCT:
|
|
2149
|
+
self.labelParts = LabelParts()
|
|
2150
|
+
self.labelParts.read(iprot)
|
|
2151
|
+
else:
|
|
2152
|
+
iprot.skip(ftype)
|
|
2153
|
+
elif fid == 7:
|
|
2154
|
+
if ftype == TType.LIST:
|
|
2155
|
+
self.bootstrapParts = []
|
|
2156
|
+
(_etype193, _size190) = iprot.readListBegin()
|
|
2157
|
+
for _i194 in range(_size190):
|
|
2158
|
+
_elem195 = BootstrapPart()
|
|
2159
|
+
_elem195.read(iprot)
|
|
2160
|
+
self.bootstrapParts.append(_elem195)
|
|
2161
|
+
iprot.readListEnd()
|
|
2162
|
+
else:
|
|
2163
|
+
iprot.skip(ftype)
|
|
2164
|
+
elif fid == 8:
|
|
2165
|
+
if ftype == TType.LIST:
|
|
2166
|
+
self.rowIds = []
|
|
2167
|
+
(_etype199, _size196) = iprot.readListBegin()
|
|
2168
|
+
for _i200 in range(_size196):
|
|
2169
|
+
_elem201 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2170
|
+
self.rowIds.append(_elem201)
|
|
2171
|
+
iprot.readListEnd()
|
|
2172
|
+
else:
|
|
2173
|
+
iprot.skip(ftype)
|
|
2174
|
+
elif fid == 9:
|
|
2175
|
+
if ftype == TType.LIST:
|
|
2176
|
+
self.derivations = []
|
|
2177
|
+
(_etype205, _size202) = iprot.readListBegin()
|
|
2178
|
+
for _i206 in range(_size202):
|
|
2179
|
+
_elem207 = Derivation()
|
|
2180
|
+
_elem207.read(iprot)
|
|
2181
|
+
self.derivations.append(_elem207)
|
|
2182
|
+
iprot.readListEnd()
|
|
2183
|
+
else:
|
|
2184
|
+
iprot.skip(ftype)
|
|
2185
|
+
elif fid == 50:
|
|
2186
|
+
if ftype == TType.BOOL:
|
|
2187
|
+
self.useLongNames = iprot.readBool()
|
|
2188
|
+
else:
|
|
2189
|
+
iprot.skip(ftype)
|
|
2190
|
+
else:
|
|
2191
|
+
iprot.skip(ftype)
|
|
2192
|
+
iprot.readFieldEnd()
|
|
2193
|
+
iprot.readStructEnd()
|
|
2194
|
+
|
|
2195
|
+
def write(self, oprot):
|
|
2196
|
+
self.validate()
|
|
2197
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
2198
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
2199
|
+
return
|
|
2200
|
+
oprot.writeStructBegin('Join')
|
|
2201
|
+
if self.metaData is not None:
|
|
2202
|
+
oprot.writeFieldBegin('metaData', TType.STRUCT, 1)
|
|
2203
|
+
self.metaData.write(oprot)
|
|
2204
|
+
oprot.writeFieldEnd()
|
|
2205
|
+
if self.left is not None:
|
|
2206
|
+
oprot.writeFieldBegin('left', TType.STRUCT, 2)
|
|
2207
|
+
self.left.write(oprot)
|
|
2208
|
+
oprot.writeFieldEnd()
|
|
2209
|
+
if self.joinParts is not None:
|
|
2210
|
+
oprot.writeFieldBegin('joinParts', TType.LIST, 3)
|
|
2211
|
+
oprot.writeListBegin(TType.STRUCT, len(self.joinParts))
|
|
2212
|
+
for iter208 in self.joinParts:
|
|
2213
|
+
iter208.write(oprot)
|
|
2214
|
+
oprot.writeListEnd()
|
|
2215
|
+
oprot.writeFieldEnd()
|
|
2216
|
+
if self.skewKeys is not None:
|
|
2217
|
+
oprot.writeFieldBegin('skewKeys', TType.MAP, 4)
|
|
2218
|
+
oprot.writeMapBegin(TType.STRING, TType.LIST, len(self.skewKeys))
|
|
2219
|
+
for kiter209, viter210 in self.skewKeys.items():
|
|
2220
|
+
oprot.writeString(kiter209.encode('utf-8') if sys.version_info[0] == 2 else kiter209)
|
|
2221
|
+
oprot.writeListBegin(TType.STRING, len(viter210))
|
|
2222
|
+
for iter211 in viter210:
|
|
2223
|
+
oprot.writeString(iter211.encode('utf-8') if sys.version_info[0] == 2 else iter211)
|
|
2224
|
+
oprot.writeListEnd()
|
|
2225
|
+
oprot.writeMapEnd()
|
|
2226
|
+
oprot.writeFieldEnd()
|
|
2227
|
+
if self.onlineExternalParts is not None:
|
|
2228
|
+
oprot.writeFieldBegin('onlineExternalParts', TType.LIST, 5)
|
|
2229
|
+
oprot.writeListBegin(TType.STRUCT, len(self.onlineExternalParts))
|
|
2230
|
+
for iter212 in self.onlineExternalParts:
|
|
2231
|
+
iter212.write(oprot)
|
|
2232
|
+
oprot.writeListEnd()
|
|
2233
|
+
oprot.writeFieldEnd()
|
|
2234
|
+
if self.labelParts is not None:
|
|
2235
|
+
oprot.writeFieldBegin('labelParts', TType.STRUCT, 6)
|
|
2236
|
+
self.labelParts.write(oprot)
|
|
2237
|
+
oprot.writeFieldEnd()
|
|
2238
|
+
if self.bootstrapParts is not None:
|
|
2239
|
+
oprot.writeFieldBegin('bootstrapParts', TType.LIST, 7)
|
|
2240
|
+
oprot.writeListBegin(TType.STRUCT, len(self.bootstrapParts))
|
|
2241
|
+
for iter213 in self.bootstrapParts:
|
|
2242
|
+
iter213.write(oprot)
|
|
2243
|
+
oprot.writeListEnd()
|
|
2244
|
+
oprot.writeFieldEnd()
|
|
2245
|
+
if self.rowIds is not None:
|
|
2246
|
+
oprot.writeFieldBegin('rowIds', TType.LIST, 8)
|
|
2247
|
+
oprot.writeListBegin(TType.STRING, len(self.rowIds))
|
|
2248
|
+
for iter214 in self.rowIds:
|
|
2249
|
+
oprot.writeString(iter214.encode('utf-8') if sys.version_info[0] == 2 else iter214)
|
|
2250
|
+
oprot.writeListEnd()
|
|
2251
|
+
oprot.writeFieldEnd()
|
|
2252
|
+
if self.derivations is not None:
|
|
2253
|
+
oprot.writeFieldBegin('derivations', TType.LIST, 9)
|
|
2254
|
+
oprot.writeListBegin(TType.STRUCT, len(self.derivations))
|
|
2255
|
+
for iter215 in self.derivations:
|
|
2256
|
+
iter215.write(oprot)
|
|
2257
|
+
oprot.writeListEnd()
|
|
2258
|
+
oprot.writeFieldEnd()
|
|
2259
|
+
if self.useLongNames is not None:
|
|
2260
|
+
oprot.writeFieldBegin('useLongNames', TType.BOOL, 50)
|
|
2261
|
+
oprot.writeBool(self.useLongNames)
|
|
2262
|
+
oprot.writeFieldEnd()
|
|
2263
|
+
oprot.writeFieldStop()
|
|
2264
|
+
oprot.writeStructEnd()
|
|
2265
|
+
|
|
2266
|
+
def validate(self):
|
|
2267
|
+
return
|
|
2268
|
+
|
|
2269
|
+
def __repr__(self):
|
|
2270
|
+
L = ['%s=%r' % (key, value)
|
|
2271
|
+
for key, value in self.__dict__.items()]
|
|
2272
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
2273
|
+
|
|
2274
|
+
def __eq__(self, other):
|
|
2275
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
2276
|
+
|
|
2277
|
+
def __ne__(self, other):
|
|
2278
|
+
return not (self == other)
|
|
2279
|
+
|
|
2280
|
+
|
|
2281
|
+
class BootstrapPart(object):
|
|
2282
|
+
"""
|
|
2283
|
+
Attributes:
|
|
2284
|
+
- metaData
|
|
2285
|
+
- table
|
|
2286
|
+
- query
|
|
2287
|
+
- keyColumns
|
|
2288
|
+
|
|
2289
|
+
"""
|
|
2290
|
+
thrift_spec = None
|
|
2291
|
+
|
|
2292
|
+
|
|
2293
|
+
def __init__(self, metaData = None, table = None, query = None, keyColumns = None,):
|
|
2294
|
+
self.metaData = metaData
|
|
2295
|
+
self.table = table
|
|
2296
|
+
self.query = query
|
|
2297
|
+
self.keyColumns = keyColumns
|
|
2298
|
+
|
|
2299
|
+
def read(self, iprot):
|
|
2300
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
2301
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
2302
|
+
return
|
|
2303
|
+
iprot.readStructBegin()
|
|
2304
|
+
while True:
|
|
2305
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
2306
|
+
if ftype == TType.STOP:
|
|
2307
|
+
break
|
|
2308
|
+
if fid == 1:
|
|
2309
|
+
if ftype == TType.STRUCT:
|
|
2310
|
+
self.metaData = MetaData()
|
|
2311
|
+
self.metaData.read(iprot)
|
|
2312
|
+
else:
|
|
2313
|
+
iprot.skip(ftype)
|
|
2314
|
+
elif fid == 2:
|
|
2315
|
+
if ftype == TType.STRING:
|
|
2316
|
+
self.table = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2317
|
+
else:
|
|
2318
|
+
iprot.skip(ftype)
|
|
2319
|
+
elif fid == 3:
|
|
2320
|
+
if ftype == TType.STRUCT:
|
|
2321
|
+
self.query = Query()
|
|
2322
|
+
self.query.read(iprot)
|
|
2323
|
+
else:
|
|
2324
|
+
iprot.skip(ftype)
|
|
2325
|
+
elif fid == 4:
|
|
2326
|
+
if ftype == TType.LIST:
|
|
2327
|
+
self.keyColumns = []
|
|
2328
|
+
(_etype219, _size216) = iprot.readListBegin()
|
|
2329
|
+
for _i220 in range(_size216):
|
|
2330
|
+
_elem221 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2331
|
+
self.keyColumns.append(_elem221)
|
|
2332
|
+
iprot.readListEnd()
|
|
2333
|
+
else:
|
|
2334
|
+
iprot.skip(ftype)
|
|
2335
|
+
else:
|
|
2336
|
+
iprot.skip(ftype)
|
|
2337
|
+
iprot.readFieldEnd()
|
|
2338
|
+
iprot.readStructEnd()
|
|
2339
|
+
|
|
2340
|
+
def write(self, oprot):
|
|
2341
|
+
self.validate()
|
|
2342
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
2343
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
2344
|
+
return
|
|
2345
|
+
oprot.writeStructBegin('BootstrapPart')
|
|
2346
|
+
if self.metaData is not None:
|
|
2347
|
+
oprot.writeFieldBegin('metaData', TType.STRUCT, 1)
|
|
2348
|
+
self.metaData.write(oprot)
|
|
2349
|
+
oprot.writeFieldEnd()
|
|
2350
|
+
if self.table is not None:
|
|
2351
|
+
oprot.writeFieldBegin('table', TType.STRING, 2)
|
|
2352
|
+
oprot.writeString(self.table.encode('utf-8') if sys.version_info[0] == 2 else self.table)
|
|
2353
|
+
oprot.writeFieldEnd()
|
|
2354
|
+
if self.query is not None:
|
|
2355
|
+
oprot.writeFieldBegin('query', TType.STRUCT, 3)
|
|
2356
|
+
self.query.write(oprot)
|
|
2357
|
+
oprot.writeFieldEnd()
|
|
2358
|
+
if self.keyColumns is not None:
|
|
2359
|
+
oprot.writeFieldBegin('keyColumns', TType.LIST, 4)
|
|
2360
|
+
oprot.writeListBegin(TType.STRING, len(self.keyColumns))
|
|
2361
|
+
for iter222 in self.keyColumns:
|
|
2362
|
+
oprot.writeString(iter222.encode('utf-8') if sys.version_info[0] == 2 else iter222)
|
|
2363
|
+
oprot.writeListEnd()
|
|
2364
|
+
oprot.writeFieldEnd()
|
|
2365
|
+
oprot.writeFieldStop()
|
|
2366
|
+
oprot.writeStructEnd()
|
|
2367
|
+
|
|
2368
|
+
def validate(self):
|
|
2369
|
+
return
|
|
2370
|
+
|
|
2371
|
+
def __repr__(self):
|
|
2372
|
+
L = ['%s=%r' % (key, value)
|
|
2373
|
+
for key, value in self.__dict__.items()]
|
|
2374
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
2375
|
+
|
|
2376
|
+
def __eq__(self, other):
|
|
2377
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
2378
|
+
|
|
2379
|
+
def __ne__(self, other):
|
|
2380
|
+
return not (self == other)
|
|
2381
|
+
|
|
2382
|
+
|
|
2383
|
+
class LabelParts(object):
|
|
2384
|
+
"""
|
|
2385
|
+
Attributes:
|
|
2386
|
+
- labels
|
|
2387
|
+
- leftStartOffset
|
|
2388
|
+
- leftEndOffset
|
|
2389
|
+
- metaData
|
|
2390
|
+
|
|
2391
|
+
"""
|
|
2392
|
+
thrift_spec = None
|
|
2393
|
+
|
|
2394
|
+
|
|
2395
|
+
def __init__(self, labels = None, leftStartOffset = None, leftEndOffset = None, metaData = None,):
|
|
2396
|
+
self.labels = labels
|
|
2397
|
+
self.leftStartOffset = leftStartOffset
|
|
2398
|
+
self.leftEndOffset = leftEndOffset
|
|
2399
|
+
self.metaData = metaData
|
|
2400
|
+
|
|
2401
|
+
def read(self, iprot):
|
|
2402
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
2403
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
2404
|
+
return
|
|
2405
|
+
iprot.readStructBegin()
|
|
2406
|
+
while True:
|
|
2407
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
2408
|
+
if ftype == TType.STOP:
|
|
2409
|
+
break
|
|
2410
|
+
if fid == 1:
|
|
2411
|
+
if ftype == TType.LIST:
|
|
2412
|
+
self.labels = []
|
|
2413
|
+
(_etype226, _size223) = iprot.readListBegin()
|
|
2414
|
+
for _i227 in range(_size223):
|
|
2415
|
+
_elem228 = JoinPart()
|
|
2416
|
+
_elem228.read(iprot)
|
|
2417
|
+
self.labels.append(_elem228)
|
|
2418
|
+
iprot.readListEnd()
|
|
2419
|
+
else:
|
|
2420
|
+
iprot.skip(ftype)
|
|
2421
|
+
elif fid == 2:
|
|
2422
|
+
if ftype == TType.I32:
|
|
2423
|
+
self.leftStartOffset = iprot.readI32()
|
|
2424
|
+
else:
|
|
2425
|
+
iprot.skip(ftype)
|
|
2426
|
+
elif fid == 3:
|
|
2427
|
+
if ftype == TType.I32:
|
|
2428
|
+
self.leftEndOffset = iprot.readI32()
|
|
2429
|
+
else:
|
|
2430
|
+
iprot.skip(ftype)
|
|
2431
|
+
elif fid == 4:
|
|
2432
|
+
if ftype == TType.STRUCT:
|
|
2433
|
+
self.metaData = MetaData()
|
|
2434
|
+
self.metaData.read(iprot)
|
|
2435
|
+
else:
|
|
2436
|
+
iprot.skip(ftype)
|
|
2437
|
+
else:
|
|
2438
|
+
iprot.skip(ftype)
|
|
2439
|
+
iprot.readFieldEnd()
|
|
2440
|
+
iprot.readStructEnd()
|
|
2441
|
+
|
|
2442
|
+
def write(self, oprot):
|
|
2443
|
+
self.validate()
|
|
2444
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
2445
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
2446
|
+
return
|
|
2447
|
+
oprot.writeStructBegin('LabelParts')
|
|
2448
|
+
if self.labels is not None:
|
|
2449
|
+
oprot.writeFieldBegin('labels', TType.LIST, 1)
|
|
2450
|
+
oprot.writeListBegin(TType.STRUCT, len(self.labels))
|
|
2451
|
+
for iter229 in self.labels:
|
|
2452
|
+
iter229.write(oprot)
|
|
2453
|
+
oprot.writeListEnd()
|
|
2454
|
+
oprot.writeFieldEnd()
|
|
2455
|
+
if self.leftStartOffset is not None:
|
|
2456
|
+
oprot.writeFieldBegin('leftStartOffset', TType.I32, 2)
|
|
2457
|
+
oprot.writeI32(self.leftStartOffset)
|
|
2458
|
+
oprot.writeFieldEnd()
|
|
2459
|
+
if self.leftEndOffset is not None:
|
|
2460
|
+
oprot.writeFieldBegin('leftEndOffset', TType.I32, 3)
|
|
2461
|
+
oprot.writeI32(self.leftEndOffset)
|
|
2462
|
+
oprot.writeFieldEnd()
|
|
2463
|
+
if self.metaData is not None:
|
|
2464
|
+
oprot.writeFieldBegin('metaData', TType.STRUCT, 4)
|
|
2465
|
+
self.metaData.write(oprot)
|
|
2466
|
+
oprot.writeFieldEnd()
|
|
2467
|
+
oprot.writeFieldStop()
|
|
2468
|
+
oprot.writeStructEnd()
|
|
2469
|
+
|
|
2470
|
+
def validate(self):
|
|
2471
|
+
return
|
|
2472
|
+
|
|
2473
|
+
def __repr__(self):
|
|
2474
|
+
L = ['%s=%r' % (key, value)
|
|
2475
|
+
for key, value in self.__dict__.items()]
|
|
2476
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
2477
|
+
|
|
2478
|
+
def __eq__(self, other):
|
|
2479
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
2480
|
+
|
|
2481
|
+
def __ne__(self, other):
|
|
2482
|
+
return not (self == other)
|
|
2483
|
+
|
|
2484
|
+
|
|
2485
|
+
class GroupByServingInfo(object):
|
|
2486
|
+
"""
|
|
2487
|
+
Attributes:
|
|
2488
|
+
- groupBy
|
|
2489
|
+
- inputAvroSchema
|
|
2490
|
+
- selectedAvroSchema
|
|
2491
|
+
- keyAvroSchema
|
|
2492
|
+
- batchEndDate
|
|
2493
|
+
- dateFormat
|
|
2494
|
+
|
|
2495
|
+
"""
|
|
2496
|
+
thrift_spec = None
|
|
2497
|
+
|
|
2498
|
+
|
|
2499
|
+
def __init__(self, groupBy = None, inputAvroSchema = None, selectedAvroSchema = None, keyAvroSchema = None, batchEndDate = None, dateFormat = None,):
|
|
2500
|
+
self.groupBy = groupBy
|
|
2501
|
+
self.inputAvroSchema = inputAvroSchema
|
|
2502
|
+
self.selectedAvroSchema = selectedAvroSchema
|
|
2503
|
+
self.keyAvroSchema = keyAvroSchema
|
|
2504
|
+
self.batchEndDate = batchEndDate
|
|
2505
|
+
self.dateFormat = dateFormat
|
|
2506
|
+
|
|
2507
|
+
def read(self, iprot):
|
|
2508
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
2509
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
2510
|
+
return
|
|
2511
|
+
iprot.readStructBegin()
|
|
2512
|
+
while True:
|
|
2513
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
2514
|
+
if ftype == TType.STOP:
|
|
2515
|
+
break
|
|
2516
|
+
if fid == 1:
|
|
2517
|
+
if ftype == TType.STRUCT:
|
|
2518
|
+
self.groupBy = GroupBy()
|
|
2519
|
+
self.groupBy.read(iprot)
|
|
2520
|
+
else:
|
|
2521
|
+
iprot.skip(ftype)
|
|
2522
|
+
elif fid == 2:
|
|
2523
|
+
if ftype == TType.STRING:
|
|
2524
|
+
self.inputAvroSchema = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2525
|
+
else:
|
|
2526
|
+
iprot.skip(ftype)
|
|
2527
|
+
elif fid == 3:
|
|
2528
|
+
if ftype == TType.STRING:
|
|
2529
|
+
self.selectedAvroSchema = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2530
|
+
else:
|
|
2531
|
+
iprot.skip(ftype)
|
|
2532
|
+
elif fid == 4:
|
|
2533
|
+
if ftype == TType.STRING:
|
|
2534
|
+
self.keyAvroSchema = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2535
|
+
else:
|
|
2536
|
+
iprot.skip(ftype)
|
|
2537
|
+
elif fid == 5:
|
|
2538
|
+
if ftype == TType.STRING:
|
|
2539
|
+
self.batchEndDate = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2540
|
+
else:
|
|
2541
|
+
iprot.skip(ftype)
|
|
2542
|
+
elif fid == 6:
|
|
2543
|
+
if ftype == TType.STRING:
|
|
2544
|
+
self.dateFormat = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2545
|
+
else:
|
|
2546
|
+
iprot.skip(ftype)
|
|
2547
|
+
else:
|
|
2548
|
+
iprot.skip(ftype)
|
|
2549
|
+
iprot.readFieldEnd()
|
|
2550
|
+
iprot.readStructEnd()
|
|
2551
|
+
|
|
2552
|
+
def write(self, oprot):
|
|
2553
|
+
self.validate()
|
|
2554
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
2555
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
2556
|
+
return
|
|
2557
|
+
oprot.writeStructBegin('GroupByServingInfo')
|
|
2558
|
+
if self.groupBy is not None:
|
|
2559
|
+
oprot.writeFieldBegin('groupBy', TType.STRUCT, 1)
|
|
2560
|
+
self.groupBy.write(oprot)
|
|
2561
|
+
oprot.writeFieldEnd()
|
|
2562
|
+
if self.inputAvroSchema is not None:
|
|
2563
|
+
oprot.writeFieldBegin('inputAvroSchema', TType.STRING, 2)
|
|
2564
|
+
oprot.writeString(self.inputAvroSchema.encode('utf-8') if sys.version_info[0] == 2 else self.inputAvroSchema)
|
|
2565
|
+
oprot.writeFieldEnd()
|
|
2566
|
+
if self.selectedAvroSchema is not None:
|
|
2567
|
+
oprot.writeFieldBegin('selectedAvroSchema', TType.STRING, 3)
|
|
2568
|
+
oprot.writeString(self.selectedAvroSchema.encode('utf-8') if sys.version_info[0] == 2 else self.selectedAvroSchema)
|
|
2569
|
+
oprot.writeFieldEnd()
|
|
2570
|
+
if self.keyAvroSchema is not None:
|
|
2571
|
+
oprot.writeFieldBegin('keyAvroSchema', TType.STRING, 4)
|
|
2572
|
+
oprot.writeString(self.keyAvroSchema.encode('utf-8') if sys.version_info[0] == 2 else self.keyAvroSchema)
|
|
2573
|
+
oprot.writeFieldEnd()
|
|
2574
|
+
if self.batchEndDate is not None:
|
|
2575
|
+
oprot.writeFieldBegin('batchEndDate', TType.STRING, 5)
|
|
2576
|
+
oprot.writeString(self.batchEndDate.encode('utf-8') if sys.version_info[0] == 2 else self.batchEndDate)
|
|
2577
|
+
oprot.writeFieldEnd()
|
|
2578
|
+
if self.dateFormat is not None:
|
|
2579
|
+
oprot.writeFieldBegin('dateFormat', TType.STRING, 6)
|
|
2580
|
+
oprot.writeString(self.dateFormat.encode('utf-8') if sys.version_info[0] == 2 else self.dateFormat)
|
|
2581
|
+
oprot.writeFieldEnd()
|
|
2582
|
+
oprot.writeFieldStop()
|
|
2583
|
+
oprot.writeStructEnd()
|
|
2584
|
+
|
|
2585
|
+
def validate(self):
|
|
2586
|
+
return
|
|
2587
|
+
|
|
2588
|
+
def __repr__(self):
|
|
2589
|
+
L = ['%s=%r' % (key, value)
|
|
2590
|
+
for key, value in self.__dict__.items()]
|
|
2591
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
2592
|
+
|
|
2593
|
+
def __eq__(self, other):
|
|
2594
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
2595
|
+
|
|
2596
|
+
def __ne__(self, other):
|
|
2597
|
+
return not (self == other)
|
|
2598
|
+
|
|
2599
|
+
|
|
2600
|
+
class DataField(object):
|
|
2601
|
+
"""
|
|
2602
|
+
Attributes:
|
|
2603
|
+
- name
|
|
2604
|
+
- dataType
|
|
2605
|
+
|
|
2606
|
+
"""
|
|
2607
|
+
thrift_spec = None
|
|
2608
|
+
|
|
2609
|
+
|
|
2610
|
+
def __init__(self, name = None, dataType = None,):
|
|
2611
|
+
self.name = name
|
|
2612
|
+
self.dataType = dataType
|
|
2613
|
+
|
|
2614
|
+
def read(self, iprot):
|
|
2615
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
2616
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
2617
|
+
return
|
|
2618
|
+
iprot.readStructBegin()
|
|
2619
|
+
while True:
|
|
2620
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
2621
|
+
if ftype == TType.STOP:
|
|
2622
|
+
break
|
|
2623
|
+
if fid == 1:
|
|
2624
|
+
if ftype == TType.STRING:
|
|
2625
|
+
self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2626
|
+
else:
|
|
2627
|
+
iprot.skip(ftype)
|
|
2628
|
+
elif fid == 2:
|
|
2629
|
+
if ftype == TType.STRUCT:
|
|
2630
|
+
self.dataType = TDataType()
|
|
2631
|
+
self.dataType.read(iprot)
|
|
2632
|
+
else:
|
|
2633
|
+
iprot.skip(ftype)
|
|
2634
|
+
else:
|
|
2635
|
+
iprot.skip(ftype)
|
|
2636
|
+
iprot.readFieldEnd()
|
|
2637
|
+
iprot.readStructEnd()
|
|
2638
|
+
|
|
2639
|
+
def write(self, oprot):
|
|
2640
|
+
self.validate()
|
|
2641
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
2642
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
2643
|
+
return
|
|
2644
|
+
oprot.writeStructBegin('DataField')
|
|
2645
|
+
if self.name is not None:
|
|
2646
|
+
oprot.writeFieldBegin('name', TType.STRING, 1)
|
|
2647
|
+
oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name)
|
|
2648
|
+
oprot.writeFieldEnd()
|
|
2649
|
+
if self.dataType is not None:
|
|
2650
|
+
oprot.writeFieldBegin('dataType', TType.STRUCT, 2)
|
|
2651
|
+
self.dataType.write(oprot)
|
|
2652
|
+
oprot.writeFieldEnd()
|
|
2653
|
+
oprot.writeFieldStop()
|
|
2654
|
+
oprot.writeStructEnd()
|
|
2655
|
+
|
|
2656
|
+
def validate(self):
|
|
2657
|
+
return
|
|
2658
|
+
|
|
2659
|
+
def __repr__(self):
|
|
2660
|
+
L = ['%s=%r' % (key, value)
|
|
2661
|
+
for key, value in self.__dict__.items()]
|
|
2662
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
2663
|
+
|
|
2664
|
+
def __eq__(self, other):
|
|
2665
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
2666
|
+
|
|
2667
|
+
def __ne__(self, other):
|
|
2668
|
+
return not (self == other)
|
|
2669
|
+
|
|
2670
|
+
|
|
2671
|
+
class TDataType(object):
|
|
2672
|
+
"""
|
|
2673
|
+
Attributes:
|
|
2674
|
+
- kind
|
|
2675
|
+
- params
|
|
2676
|
+
- name
|
|
2677
|
+
|
|
2678
|
+
"""
|
|
2679
|
+
thrift_spec = None
|
|
2680
|
+
|
|
2681
|
+
|
|
2682
|
+
def __init__(self, kind = None, params = None, name = None,):
|
|
2683
|
+
self.kind = kind
|
|
2684
|
+
self.params = params
|
|
2685
|
+
self.name = name
|
|
2686
|
+
|
|
2687
|
+
def read(self, iprot):
|
|
2688
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
2689
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
2690
|
+
return
|
|
2691
|
+
iprot.readStructBegin()
|
|
2692
|
+
while True:
|
|
2693
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
2694
|
+
if ftype == TType.STOP:
|
|
2695
|
+
break
|
|
2696
|
+
if fid == 1:
|
|
2697
|
+
if ftype == TType.I32:
|
|
2698
|
+
self.kind = iprot.readI32()
|
|
2699
|
+
else:
|
|
2700
|
+
iprot.skip(ftype)
|
|
2701
|
+
elif fid == 2:
|
|
2702
|
+
if ftype == TType.LIST:
|
|
2703
|
+
self.params = []
|
|
2704
|
+
(_etype233, _size230) = iprot.readListBegin()
|
|
2705
|
+
for _i234 in range(_size230):
|
|
2706
|
+
_elem235 = DataField()
|
|
2707
|
+
_elem235.read(iprot)
|
|
2708
|
+
self.params.append(_elem235)
|
|
2709
|
+
iprot.readListEnd()
|
|
2710
|
+
else:
|
|
2711
|
+
iprot.skip(ftype)
|
|
2712
|
+
elif fid == 3:
|
|
2713
|
+
if ftype == TType.STRING:
|
|
2714
|
+
self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2715
|
+
else:
|
|
2716
|
+
iprot.skip(ftype)
|
|
2717
|
+
else:
|
|
2718
|
+
iprot.skip(ftype)
|
|
2719
|
+
iprot.readFieldEnd()
|
|
2720
|
+
iprot.readStructEnd()
|
|
2721
|
+
|
|
2722
|
+
def write(self, oprot):
|
|
2723
|
+
self.validate()
|
|
2724
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
2725
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
2726
|
+
return
|
|
2727
|
+
oprot.writeStructBegin('TDataType')
|
|
2728
|
+
if self.kind is not None:
|
|
2729
|
+
oprot.writeFieldBegin('kind', TType.I32, 1)
|
|
2730
|
+
oprot.writeI32(self.kind)
|
|
2731
|
+
oprot.writeFieldEnd()
|
|
2732
|
+
if self.params is not None:
|
|
2733
|
+
oprot.writeFieldBegin('params', TType.LIST, 2)
|
|
2734
|
+
oprot.writeListBegin(TType.STRUCT, len(self.params))
|
|
2735
|
+
for iter236 in self.params:
|
|
2736
|
+
iter236.write(oprot)
|
|
2737
|
+
oprot.writeListEnd()
|
|
2738
|
+
oprot.writeFieldEnd()
|
|
2739
|
+
if self.name is not None:
|
|
2740
|
+
oprot.writeFieldBegin('name', TType.STRING, 3)
|
|
2741
|
+
oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name)
|
|
2742
|
+
oprot.writeFieldEnd()
|
|
2743
|
+
oprot.writeFieldStop()
|
|
2744
|
+
oprot.writeStructEnd()
|
|
2745
|
+
|
|
2746
|
+
def validate(self):
|
|
2747
|
+
return
|
|
2748
|
+
|
|
2749
|
+
def __repr__(self):
|
|
2750
|
+
L = ['%s=%r' % (key, value)
|
|
2751
|
+
for key, value in self.__dict__.items()]
|
|
2752
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
2753
|
+
|
|
2754
|
+
def __eq__(self, other):
|
|
2755
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
2756
|
+
|
|
2757
|
+
def __ne__(self, other):
|
|
2758
|
+
return not (self == other)
|
|
2759
|
+
|
|
2760
|
+
|
|
2761
|
+
class DataSpec(object):
|
|
2762
|
+
"""
|
|
2763
|
+
Attributes:
|
|
2764
|
+
- schema
|
|
2765
|
+
- partitionColumns
|
|
2766
|
+
- retentionDays
|
|
2767
|
+
- props
|
|
2768
|
+
|
|
2769
|
+
"""
|
|
2770
|
+
thrift_spec = None
|
|
2771
|
+
|
|
2772
|
+
|
|
2773
|
+
def __init__(self, schema = None, partitionColumns = None, retentionDays = None, props = None,):
|
|
2774
|
+
self.schema = schema
|
|
2775
|
+
self.partitionColumns = partitionColumns
|
|
2776
|
+
self.retentionDays = retentionDays
|
|
2777
|
+
self.props = props
|
|
2778
|
+
|
|
2779
|
+
def read(self, iprot):
|
|
2780
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
2781
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
2782
|
+
return
|
|
2783
|
+
iprot.readStructBegin()
|
|
2784
|
+
while True:
|
|
2785
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
2786
|
+
if ftype == TType.STOP:
|
|
2787
|
+
break
|
|
2788
|
+
if fid == 1:
|
|
2789
|
+
if ftype == TType.STRUCT:
|
|
2790
|
+
self.schema = TDataType()
|
|
2791
|
+
self.schema.read(iprot)
|
|
2792
|
+
else:
|
|
2793
|
+
iprot.skip(ftype)
|
|
2794
|
+
elif fid == 2:
|
|
2795
|
+
if ftype == TType.LIST:
|
|
2796
|
+
self.partitionColumns = []
|
|
2797
|
+
(_etype240, _size237) = iprot.readListBegin()
|
|
2798
|
+
for _i241 in range(_size237):
|
|
2799
|
+
_elem242 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2800
|
+
self.partitionColumns.append(_elem242)
|
|
2801
|
+
iprot.readListEnd()
|
|
2802
|
+
else:
|
|
2803
|
+
iprot.skip(ftype)
|
|
2804
|
+
elif fid == 3:
|
|
2805
|
+
if ftype == TType.I32:
|
|
2806
|
+
self.retentionDays = iprot.readI32()
|
|
2807
|
+
else:
|
|
2808
|
+
iprot.skip(ftype)
|
|
2809
|
+
elif fid == 4:
|
|
2810
|
+
if ftype == TType.MAP:
|
|
2811
|
+
self.props = {}
|
|
2812
|
+
(_ktype244, _vtype245, _size243) = iprot.readMapBegin()
|
|
2813
|
+
for _i247 in range(_size243):
|
|
2814
|
+
_key248 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2815
|
+
_val249 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2816
|
+
self.props[_key248] = _val249
|
|
2817
|
+
iprot.readMapEnd()
|
|
2818
|
+
else:
|
|
2819
|
+
iprot.skip(ftype)
|
|
2820
|
+
else:
|
|
2821
|
+
iprot.skip(ftype)
|
|
2822
|
+
iprot.readFieldEnd()
|
|
2823
|
+
iprot.readStructEnd()
|
|
2824
|
+
|
|
2825
|
+
def write(self, oprot):
|
|
2826
|
+
self.validate()
|
|
2827
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
2828
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
2829
|
+
return
|
|
2830
|
+
oprot.writeStructBegin('DataSpec')
|
|
2831
|
+
if self.schema is not None:
|
|
2832
|
+
oprot.writeFieldBegin('schema', TType.STRUCT, 1)
|
|
2833
|
+
self.schema.write(oprot)
|
|
2834
|
+
oprot.writeFieldEnd()
|
|
2835
|
+
if self.partitionColumns is not None:
|
|
2836
|
+
oprot.writeFieldBegin('partitionColumns', TType.LIST, 2)
|
|
2837
|
+
oprot.writeListBegin(TType.STRING, len(self.partitionColumns))
|
|
2838
|
+
for iter250 in self.partitionColumns:
|
|
2839
|
+
oprot.writeString(iter250.encode('utf-8') if sys.version_info[0] == 2 else iter250)
|
|
2840
|
+
oprot.writeListEnd()
|
|
2841
|
+
oprot.writeFieldEnd()
|
|
2842
|
+
if self.retentionDays is not None:
|
|
2843
|
+
oprot.writeFieldBegin('retentionDays', TType.I32, 3)
|
|
2844
|
+
oprot.writeI32(self.retentionDays)
|
|
2845
|
+
oprot.writeFieldEnd()
|
|
2846
|
+
if self.props is not None:
|
|
2847
|
+
oprot.writeFieldBegin('props', TType.MAP, 4)
|
|
2848
|
+
oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.props))
|
|
2849
|
+
for kiter251, viter252 in self.props.items():
|
|
2850
|
+
oprot.writeString(kiter251.encode('utf-8') if sys.version_info[0] == 2 else kiter251)
|
|
2851
|
+
oprot.writeString(viter252.encode('utf-8') if sys.version_info[0] == 2 else viter252)
|
|
2852
|
+
oprot.writeMapEnd()
|
|
2853
|
+
oprot.writeFieldEnd()
|
|
2854
|
+
oprot.writeFieldStop()
|
|
2855
|
+
oprot.writeStructEnd()
|
|
2856
|
+
|
|
2857
|
+
def validate(self):
|
|
2858
|
+
return
|
|
2859
|
+
|
|
2860
|
+
def __repr__(self):
|
|
2861
|
+
L = ['%s=%r' % (key, value)
|
|
2862
|
+
for key, value in self.__dict__.items()]
|
|
2863
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
2864
|
+
|
|
2865
|
+
def __eq__(self, other):
|
|
2866
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
2867
|
+
|
|
2868
|
+
def __ne__(self, other):
|
|
2869
|
+
return not (self == other)
|
|
2870
|
+
|
|
2871
|
+
|
|
2872
|
+
class Model(object):
|
|
2873
|
+
"""
|
|
2874
|
+
Attributes:
|
|
2875
|
+
- metaData
|
|
2876
|
+
- modelType
|
|
2877
|
+
- outputSchema
|
|
2878
|
+
- source
|
|
2879
|
+
- modelParams
|
|
2880
|
+
|
|
2881
|
+
"""
|
|
2882
|
+
thrift_spec = None
|
|
2883
|
+
|
|
2884
|
+
|
|
2885
|
+
def __init__(self, metaData = None, modelType = None, outputSchema = None, source = None, modelParams = None,):
|
|
2886
|
+
self.metaData = metaData
|
|
2887
|
+
self.modelType = modelType
|
|
2888
|
+
self.outputSchema = outputSchema
|
|
2889
|
+
self.source = source
|
|
2890
|
+
self.modelParams = modelParams
|
|
2891
|
+
|
|
2892
|
+
def read(self, iprot):
|
|
2893
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
2894
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
2895
|
+
return
|
|
2896
|
+
iprot.readStructBegin()
|
|
2897
|
+
while True:
|
|
2898
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
2899
|
+
if ftype == TType.STOP:
|
|
2900
|
+
break
|
|
2901
|
+
if fid == 1:
|
|
2902
|
+
if ftype == TType.STRUCT:
|
|
2903
|
+
self.metaData = MetaData()
|
|
2904
|
+
self.metaData.read(iprot)
|
|
2905
|
+
else:
|
|
2906
|
+
iprot.skip(ftype)
|
|
2907
|
+
elif fid == 2:
|
|
2908
|
+
if ftype == TType.I32:
|
|
2909
|
+
self.modelType = iprot.readI32()
|
|
2910
|
+
else:
|
|
2911
|
+
iprot.skip(ftype)
|
|
2912
|
+
elif fid == 3:
|
|
2913
|
+
if ftype == TType.STRUCT:
|
|
2914
|
+
self.outputSchema = TDataType()
|
|
2915
|
+
self.outputSchema.read(iprot)
|
|
2916
|
+
else:
|
|
2917
|
+
iprot.skip(ftype)
|
|
2918
|
+
elif fid == 4:
|
|
2919
|
+
if ftype == TType.STRUCT:
|
|
2920
|
+
self.source = Source()
|
|
2921
|
+
self.source.read(iprot)
|
|
2922
|
+
else:
|
|
2923
|
+
iprot.skip(ftype)
|
|
2924
|
+
elif fid == 5:
|
|
2925
|
+
if ftype == TType.MAP:
|
|
2926
|
+
self.modelParams = {}
|
|
2927
|
+
(_ktype254, _vtype255, _size253) = iprot.readMapBegin()
|
|
2928
|
+
for _i257 in range(_size253):
|
|
2929
|
+
_key258 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2930
|
+
_val259 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
2931
|
+
self.modelParams[_key258] = _val259
|
|
2932
|
+
iprot.readMapEnd()
|
|
2933
|
+
else:
|
|
2934
|
+
iprot.skip(ftype)
|
|
2935
|
+
else:
|
|
2936
|
+
iprot.skip(ftype)
|
|
2937
|
+
iprot.readFieldEnd()
|
|
2938
|
+
iprot.readStructEnd()
|
|
2939
|
+
|
|
2940
|
+
def write(self, oprot):
|
|
2941
|
+
self.validate()
|
|
2942
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
2943
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
2944
|
+
return
|
|
2945
|
+
oprot.writeStructBegin('Model')
|
|
2946
|
+
if self.metaData is not None:
|
|
2947
|
+
oprot.writeFieldBegin('metaData', TType.STRUCT, 1)
|
|
2948
|
+
self.metaData.write(oprot)
|
|
2949
|
+
oprot.writeFieldEnd()
|
|
2950
|
+
if self.modelType is not None:
|
|
2951
|
+
oprot.writeFieldBegin('modelType', TType.I32, 2)
|
|
2952
|
+
oprot.writeI32(self.modelType)
|
|
2953
|
+
oprot.writeFieldEnd()
|
|
2954
|
+
if self.outputSchema is not None:
|
|
2955
|
+
oprot.writeFieldBegin('outputSchema', TType.STRUCT, 3)
|
|
2956
|
+
self.outputSchema.write(oprot)
|
|
2957
|
+
oprot.writeFieldEnd()
|
|
2958
|
+
if self.source is not None:
|
|
2959
|
+
oprot.writeFieldBegin('source', TType.STRUCT, 4)
|
|
2960
|
+
self.source.write(oprot)
|
|
2961
|
+
oprot.writeFieldEnd()
|
|
2962
|
+
if self.modelParams is not None:
|
|
2963
|
+
oprot.writeFieldBegin('modelParams', TType.MAP, 5)
|
|
2964
|
+
oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.modelParams))
|
|
2965
|
+
for kiter260, viter261 in self.modelParams.items():
|
|
2966
|
+
oprot.writeString(kiter260.encode('utf-8') if sys.version_info[0] == 2 else kiter260)
|
|
2967
|
+
oprot.writeString(viter261.encode('utf-8') if sys.version_info[0] == 2 else viter261)
|
|
2968
|
+
oprot.writeMapEnd()
|
|
2969
|
+
oprot.writeFieldEnd()
|
|
2970
|
+
oprot.writeFieldStop()
|
|
2971
|
+
oprot.writeStructEnd()
|
|
2972
|
+
|
|
2973
|
+
def validate(self):
|
|
2974
|
+
return
|
|
2975
|
+
|
|
2976
|
+
def __repr__(self):
|
|
2977
|
+
L = ['%s=%r' % (key, value)
|
|
2978
|
+
for key, value in self.__dict__.items()]
|
|
2979
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
2980
|
+
|
|
2981
|
+
def __eq__(self, other):
|
|
2982
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
2983
|
+
|
|
2984
|
+
def __ne__(self, other):
|
|
2985
|
+
return not (self == other)
|
|
2986
|
+
|
|
2987
|
+
|
|
2988
|
+
class Team(object):
|
|
2989
|
+
"""
|
|
2990
|
+
Attributes:
|
|
2991
|
+
- name
|
|
2992
|
+
- description
|
|
2993
|
+
- email
|
|
2994
|
+
- outputNamespace
|
|
2995
|
+
- tableProperties
|
|
2996
|
+
- env
|
|
2997
|
+
- conf
|
|
2998
|
+
- clusterConf
|
|
2999
|
+
|
|
3000
|
+
"""
|
|
3001
|
+
thrift_spec = None
|
|
3002
|
+
|
|
3003
|
+
|
|
3004
|
+
def __init__(self, name = None, description = None, email = None, outputNamespace = None, tableProperties = None, env = None, conf = None, clusterConf = None,):
|
|
3005
|
+
self.name = name
|
|
3006
|
+
self.description = description
|
|
3007
|
+
self.email = email
|
|
3008
|
+
self.outputNamespace = outputNamespace
|
|
3009
|
+
self.tableProperties = tableProperties
|
|
3010
|
+
self.env = env
|
|
3011
|
+
self.conf = conf
|
|
3012
|
+
self.clusterConf = clusterConf
|
|
3013
|
+
|
|
3014
|
+
def read(self, iprot):
|
|
3015
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
3016
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
3017
|
+
return
|
|
3018
|
+
iprot.readStructBegin()
|
|
3019
|
+
while True:
|
|
3020
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
3021
|
+
if ftype == TType.STOP:
|
|
3022
|
+
break
|
|
3023
|
+
if fid == 1:
|
|
3024
|
+
if ftype == TType.STRING:
|
|
3025
|
+
self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
3026
|
+
else:
|
|
3027
|
+
iprot.skip(ftype)
|
|
3028
|
+
elif fid == 2:
|
|
3029
|
+
if ftype == TType.STRING:
|
|
3030
|
+
self.description = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
3031
|
+
else:
|
|
3032
|
+
iprot.skip(ftype)
|
|
3033
|
+
elif fid == 3:
|
|
3034
|
+
if ftype == TType.STRING:
|
|
3035
|
+
self.email = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
3036
|
+
else:
|
|
3037
|
+
iprot.skip(ftype)
|
|
3038
|
+
elif fid == 10:
|
|
3039
|
+
if ftype == TType.STRING:
|
|
3040
|
+
self.outputNamespace = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
3041
|
+
else:
|
|
3042
|
+
iprot.skip(ftype)
|
|
3043
|
+
elif fid == 11:
|
|
3044
|
+
if ftype == TType.MAP:
|
|
3045
|
+
self.tableProperties = {}
|
|
3046
|
+
(_ktype263, _vtype264, _size262) = iprot.readMapBegin()
|
|
3047
|
+
for _i266 in range(_size262):
|
|
3048
|
+
_key267 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
3049
|
+
_val268 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
3050
|
+
self.tableProperties[_key267] = _val268
|
|
3051
|
+
iprot.readMapEnd()
|
|
3052
|
+
else:
|
|
3053
|
+
iprot.skip(ftype)
|
|
3054
|
+
elif fid == 20:
|
|
3055
|
+
if ftype == TType.STRUCT:
|
|
3056
|
+
self.env = ai.chronon.api.common.ttypes.EnvironmentVariables()
|
|
3057
|
+
self.env.read(iprot)
|
|
3058
|
+
else:
|
|
3059
|
+
iprot.skip(ftype)
|
|
3060
|
+
elif fid == 21:
|
|
3061
|
+
if ftype == TType.STRUCT:
|
|
3062
|
+
self.conf = ai.chronon.api.common.ttypes.ConfigProperties()
|
|
3063
|
+
self.conf.read(iprot)
|
|
3064
|
+
else:
|
|
3065
|
+
iprot.skip(ftype)
|
|
3066
|
+
elif fid == 22:
|
|
3067
|
+
if ftype == TType.STRUCT:
|
|
3068
|
+
self.clusterConf = ai.chronon.api.common.ttypes.ClusterConfigProperties()
|
|
3069
|
+
self.clusterConf.read(iprot)
|
|
3070
|
+
else:
|
|
3071
|
+
iprot.skip(ftype)
|
|
3072
|
+
else:
|
|
3073
|
+
iprot.skip(ftype)
|
|
3074
|
+
iprot.readFieldEnd()
|
|
3075
|
+
iprot.readStructEnd()
|
|
3076
|
+
|
|
3077
|
+
def write(self, oprot):
|
|
3078
|
+
self.validate()
|
|
3079
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
3080
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
3081
|
+
return
|
|
3082
|
+
oprot.writeStructBegin('Team')
|
|
3083
|
+
if self.name is not None:
|
|
3084
|
+
oprot.writeFieldBegin('name', TType.STRING, 1)
|
|
3085
|
+
oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name)
|
|
3086
|
+
oprot.writeFieldEnd()
|
|
3087
|
+
if self.description is not None:
|
|
3088
|
+
oprot.writeFieldBegin('description', TType.STRING, 2)
|
|
3089
|
+
oprot.writeString(self.description.encode('utf-8') if sys.version_info[0] == 2 else self.description)
|
|
3090
|
+
oprot.writeFieldEnd()
|
|
3091
|
+
if self.email is not None:
|
|
3092
|
+
oprot.writeFieldBegin('email', TType.STRING, 3)
|
|
3093
|
+
oprot.writeString(self.email.encode('utf-8') if sys.version_info[0] == 2 else self.email)
|
|
3094
|
+
oprot.writeFieldEnd()
|
|
3095
|
+
if self.outputNamespace is not None:
|
|
3096
|
+
oprot.writeFieldBegin('outputNamespace', TType.STRING, 10)
|
|
3097
|
+
oprot.writeString(self.outputNamespace.encode('utf-8') if sys.version_info[0] == 2 else self.outputNamespace)
|
|
3098
|
+
oprot.writeFieldEnd()
|
|
3099
|
+
if self.tableProperties is not None:
|
|
3100
|
+
oprot.writeFieldBegin('tableProperties', TType.MAP, 11)
|
|
3101
|
+
oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.tableProperties))
|
|
3102
|
+
for kiter269, viter270 in self.tableProperties.items():
|
|
3103
|
+
oprot.writeString(kiter269.encode('utf-8') if sys.version_info[0] == 2 else kiter269)
|
|
3104
|
+
oprot.writeString(viter270.encode('utf-8') if sys.version_info[0] == 2 else viter270)
|
|
3105
|
+
oprot.writeMapEnd()
|
|
3106
|
+
oprot.writeFieldEnd()
|
|
3107
|
+
if self.env is not None:
|
|
3108
|
+
oprot.writeFieldBegin('env', TType.STRUCT, 20)
|
|
3109
|
+
self.env.write(oprot)
|
|
3110
|
+
oprot.writeFieldEnd()
|
|
3111
|
+
if self.conf is not None:
|
|
3112
|
+
oprot.writeFieldBegin('conf', TType.STRUCT, 21)
|
|
3113
|
+
self.conf.write(oprot)
|
|
3114
|
+
oprot.writeFieldEnd()
|
|
3115
|
+
if self.clusterConf is not None:
|
|
3116
|
+
oprot.writeFieldBegin('clusterConf', TType.STRUCT, 22)
|
|
3117
|
+
self.clusterConf.write(oprot)
|
|
3118
|
+
oprot.writeFieldEnd()
|
|
3119
|
+
oprot.writeFieldStop()
|
|
3120
|
+
oprot.writeStructEnd()
|
|
3121
|
+
|
|
3122
|
+
def validate(self):
|
|
3123
|
+
return
|
|
3124
|
+
|
|
3125
|
+
def __repr__(self):
|
|
3126
|
+
L = ['%s=%r' % (key, value)
|
|
3127
|
+
for key, value in self.__dict__.items()]
|
|
3128
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
3129
|
+
|
|
3130
|
+
def __eq__(self, other):
|
|
3131
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
3132
|
+
|
|
3133
|
+
def __ne__(self, other):
|
|
3134
|
+
return not (self == other)
|
|
3135
|
+
all_structs.append(Query)
|
|
3136
|
+
Query.thrift_spec = (
|
|
3137
|
+
None, # 0
|
|
3138
|
+
(1, TType.MAP, 'selects', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 1
|
|
3139
|
+
(2, TType.LIST, 'wheres', (TType.STRING, 'UTF8', False), None, ), # 2
|
|
3140
|
+
(3, TType.STRING, 'startPartition', 'UTF8', None, ), # 3
|
|
3141
|
+
(4, TType.STRING, 'endPartition', 'UTF8', None, ), # 4
|
|
3142
|
+
(5, TType.STRING, 'timeColumn', 'UTF8', None, ), # 5
|
|
3143
|
+
(6, TType.LIST, 'setups', (TType.STRING, 'UTF8', False), None, ), # 6
|
|
3144
|
+
(7, TType.STRING, 'mutationTimeColumn', 'UTF8', None, ), # 7
|
|
3145
|
+
(8, TType.STRING, 'reversalColumn', 'UTF8', None, ), # 8
|
|
3146
|
+
None, # 9
|
|
3147
|
+
None, # 10
|
|
3148
|
+
None, # 11
|
|
3149
|
+
None, # 12
|
|
3150
|
+
None, # 13
|
|
3151
|
+
None, # 14
|
|
3152
|
+
None, # 15
|
|
3153
|
+
None, # 16
|
|
3154
|
+
None, # 17
|
|
3155
|
+
None, # 18
|
|
3156
|
+
None, # 19
|
|
3157
|
+
(20, TType.STRING, 'partitionColumn', 'UTF8', None, ), # 20
|
|
3158
|
+
(21, TType.STRING, 'partitionFormat', 'UTF8', None, ), # 21
|
|
3159
|
+
(22, TType.STRUCT, 'partitionInterval', [ai.chronon.api.common.ttypes.Window, None], None, ), # 22
|
|
3160
|
+
(23, TType.STRUCT, 'partitionLag', [ai.chronon.api.common.ttypes.Window, None], None, ), # 23
|
|
3161
|
+
(24, TType.LIST, 'subPartitionsToWaitFor', (TType.STRING, 'UTF8', False), None, ), # 24
|
|
3162
|
+
)
|
|
3163
|
+
all_structs.append(StagingQuery)
|
|
3164
|
+
StagingQuery.thrift_spec = (
|
|
3165
|
+
None, # 0
|
|
3166
|
+
(1, TType.STRUCT, 'metaData', [MetaData, None], None, ), # 1
|
|
3167
|
+
(2, TType.STRING, 'query', 'UTF8', None, ), # 2
|
|
3168
|
+
(3, TType.STRING, 'startPartition', 'UTF8', None, ), # 3
|
|
3169
|
+
(4, TType.LIST, 'setups', (TType.STRING, 'UTF8', False), None, ), # 4
|
|
3170
|
+
(5, TType.STRING, 'partitionColumn', 'UTF8', None, ), # 5
|
|
3171
|
+
(6, TType.I32, 'engineType', None, None, ), # 6
|
|
3172
|
+
(7, TType.LIST, 'tableDependencies', (TType.STRUCT, [ai.chronon.api.common.ttypes.TableDependency, None], False), None, ), # 7
|
|
3173
|
+
None, # 8
|
|
3174
|
+
None, # 9
|
|
3175
|
+
None, # 10
|
|
3176
|
+
None, # 11
|
|
3177
|
+
None, # 12
|
|
3178
|
+
None, # 13
|
|
3179
|
+
None, # 14
|
|
3180
|
+
None, # 15
|
|
3181
|
+
None, # 16
|
|
3182
|
+
None, # 17
|
|
3183
|
+
None, # 18
|
|
3184
|
+
None, # 19
|
|
3185
|
+
(20, TType.I32, 'recomputeDays', None, None, ), # 20
|
|
3186
|
+
)
|
|
3187
|
+
all_structs.append(EventSource)
|
|
3188
|
+
EventSource.thrift_spec = (
|
|
3189
|
+
None, # 0
|
|
3190
|
+
(1, TType.STRING, 'table', 'UTF8', None, ), # 1
|
|
3191
|
+
(2, TType.STRING, 'topic', 'UTF8', None, ), # 2
|
|
3192
|
+
(3, TType.STRUCT, 'query', [Query, None], None, ), # 3
|
|
3193
|
+
(4, TType.BOOL, 'isCumulative', None, None, ), # 4
|
|
3194
|
+
)
|
|
3195
|
+
all_structs.append(EntitySource)
|
|
3196
|
+
EntitySource.thrift_spec = (
|
|
3197
|
+
None, # 0
|
|
3198
|
+
(1, TType.STRING, 'snapshotTable', 'UTF8', None, ), # 1
|
|
3199
|
+
(2, TType.STRING, 'mutationTable', 'UTF8', None, ), # 2
|
|
3200
|
+
(3, TType.STRING, 'mutationTopic', 'UTF8', None, ), # 3
|
|
3201
|
+
(4, TType.STRUCT, 'query', [Query, None], None, ), # 4
|
|
3202
|
+
)
|
|
3203
|
+
all_structs.append(ExternalSource)
|
|
3204
|
+
ExternalSource.thrift_spec = (
|
|
3205
|
+
None, # 0
|
|
3206
|
+
(1, TType.STRUCT, 'metadata', [MetaData, None], None, ), # 1
|
|
3207
|
+
(2, TType.STRUCT, 'keySchema', [TDataType, None], None, ), # 2
|
|
3208
|
+
(3, TType.STRUCT, 'valueSchema', [TDataType, None], None, ), # 3
|
|
3209
|
+
)
|
|
3210
|
+
all_structs.append(JoinSource)
|
|
3211
|
+
JoinSource.thrift_spec = (
|
|
3212
|
+
None, # 0
|
|
3213
|
+
(1, TType.STRUCT, 'join', [Join, None], None, ), # 1
|
|
3214
|
+
(2, TType.STRUCT, 'query', [Query, None], None, ), # 2
|
|
3215
|
+
)
|
|
3216
|
+
all_structs.append(Source)
|
|
3217
|
+
Source.thrift_spec = (
|
|
3218
|
+
None, # 0
|
|
3219
|
+
(1, TType.STRUCT, 'events', [EventSource, None], None, ), # 1
|
|
3220
|
+
(2, TType.STRUCT, 'entities', [EntitySource, None], None, ), # 2
|
|
3221
|
+
(3, TType.STRUCT, 'joinSource', [JoinSource, None], None, ), # 3
|
|
3222
|
+
)
|
|
3223
|
+
all_structs.append(Aggregation)
|
|
3224
|
+
Aggregation.thrift_spec = (
|
|
3225
|
+
None, # 0
|
|
3226
|
+
(1, TType.STRING, 'inputColumn', 'UTF8', None, ), # 1
|
|
3227
|
+
(2, TType.I32, 'operation', None, None, ), # 2
|
|
3228
|
+
(3, TType.MAP, 'argMap', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 3
|
|
3229
|
+
(4, TType.LIST, 'windows', (TType.STRUCT, [ai.chronon.api.common.ttypes.Window, None], False), None, ), # 4
|
|
3230
|
+
(5, TType.LIST, 'buckets', (TType.STRING, 'UTF8', False), None, ), # 5
|
|
3231
|
+
)
|
|
3232
|
+
all_structs.append(AggregationPart)
|
|
3233
|
+
AggregationPart.thrift_spec = (
|
|
3234
|
+
None, # 0
|
|
3235
|
+
(1, TType.STRING, 'inputColumn', 'UTF8', None, ), # 1
|
|
3236
|
+
(2, TType.I32, 'operation', None, None, ), # 2
|
|
3237
|
+
(3, TType.MAP, 'argMap', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 3
|
|
3238
|
+
(4, TType.STRUCT, 'window', [ai.chronon.api.common.ttypes.Window, None], None, ), # 4
|
|
3239
|
+
(5, TType.STRING, 'bucket', 'UTF8', None, ), # 5
|
|
3240
|
+
)
|
|
3241
|
+
all_structs.append(MetaData)
|
|
3242
|
+
MetaData.thrift_spec = (
|
|
3243
|
+
None, # 0
|
|
3244
|
+
(1, TType.STRING, 'name', 'UTF8', None, ), # 1
|
|
3245
|
+
(2, TType.STRING, 'team', 'UTF8', None, ), # 2
|
|
3246
|
+
(3, TType.STRING, 'version', 'UTF8', None, ), # 3
|
|
3247
|
+
(4, TType.STRING, 'outputNamespace', 'UTF8', None, ), # 4
|
|
3248
|
+
(5, TType.LIST, 'additionalOutputPartitionColumns', (TType.STRING, 'UTF8', False), None, ), # 5
|
|
3249
|
+
(6, TType.MAP, 'tableProperties', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 6
|
|
3250
|
+
None, # 7
|
|
3251
|
+
None, # 8
|
|
3252
|
+
None, # 9
|
|
3253
|
+
None, # 10
|
|
3254
|
+
None, # 11
|
|
3255
|
+
None, # 12
|
|
3256
|
+
None, # 13
|
|
3257
|
+
None, # 14
|
|
3258
|
+
None, # 15
|
|
3259
|
+
None, # 16
|
|
3260
|
+
None, # 17
|
|
3261
|
+
None, # 18
|
|
3262
|
+
None, # 19
|
|
3263
|
+
(20, TType.MAP, 'tags', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 20
|
|
3264
|
+
(21, TType.MAP, 'columnTags', (TType.STRING, 'UTF8', TType.MAP, (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), False), None, ), # 21
|
|
3265
|
+
None, # 22
|
|
3266
|
+
None, # 23
|
|
3267
|
+
None, # 24
|
|
3268
|
+
None, # 25
|
|
3269
|
+
None, # 26
|
|
3270
|
+
None, # 27
|
|
3271
|
+
None, # 28
|
|
3272
|
+
None, # 29
|
|
3273
|
+
None, # 30
|
|
3274
|
+
None, # 31
|
|
3275
|
+
None, # 32
|
|
3276
|
+
None, # 33
|
|
3277
|
+
None, # 34
|
|
3278
|
+
None, # 35
|
|
3279
|
+
None, # 36
|
|
3280
|
+
None, # 37
|
|
3281
|
+
None, # 38
|
|
3282
|
+
None, # 39
|
|
3283
|
+
None, # 40
|
|
3284
|
+
None, # 41
|
|
3285
|
+
None, # 42
|
|
3286
|
+
None, # 43
|
|
3287
|
+
None, # 44
|
|
3288
|
+
None, # 45
|
|
3289
|
+
None, # 46
|
|
3290
|
+
None, # 47
|
|
3291
|
+
None, # 48
|
|
3292
|
+
None, # 49
|
|
3293
|
+
None, # 50
|
|
3294
|
+
None, # 51
|
|
3295
|
+
None, # 52
|
|
3296
|
+
None, # 53
|
|
3297
|
+
None, # 54
|
|
3298
|
+
None, # 55
|
|
3299
|
+
None, # 56
|
|
3300
|
+
None, # 57
|
|
3301
|
+
None, # 58
|
|
3302
|
+
None, # 59
|
|
3303
|
+
None, # 60
|
|
3304
|
+
None, # 61
|
|
3305
|
+
None, # 62
|
|
3306
|
+
None, # 63
|
|
3307
|
+
None, # 64
|
|
3308
|
+
None, # 65
|
|
3309
|
+
None, # 66
|
|
3310
|
+
None, # 67
|
|
3311
|
+
None, # 68
|
|
3312
|
+
None, # 69
|
|
3313
|
+
None, # 70
|
|
3314
|
+
None, # 71
|
|
3315
|
+
None, # 72
|
|
3316
|
+
None, # 73
|
|
3317
|
+
None, # 74
|
|
3318
|
+
None, # 75
|
|
3319
|
+
None, # 76
|
|
3320
|
+
None, # 77
|
|
3321
|
+
None, # 78
|
|
3322
|
+
None, # 79
|
|
3323
|
+
None, # 80
|
|
3324
|
+
None, # 81
|
|
3325
|
+
None, # 82
|
|
3326
|
+
None, # 83
|
|
3327
|
+
None, # 84
|
|
3328
|
+
None, # 85
|
|
3329
|
+
None, # 86
|
|
3330
|
+
None, # 87
|
|
3331
|
+
None, # 88
|
|
3332
|
+
None, # 89
|
|
3333
|
+
None, # 90
|
|
3334
|
+
None, # 91
|
|
3335
|
+
None, # 92
|
|
3336
|
+
None, # 93
|
|
3337
|
+
None, # 94
|
|
3338
|
+
None, # 95
|
|
3339
|
+
None, # 96
|
|
3340
|
+
None, # 97
|
|
3341
|
+
None, # 98
|
|
3342
|
+
None, # 99
|
|
3343
|
+
(100, TType.BOOL, 'online', None, None, ), # 100
|
|
3344
|
+
(101, TType.BOOL, 'production', None, None, ), # 101
|
|
3345
|
+
(102, TType.STRING, 'sourceFile', 'UTF8', None, ), # 102
|
|
3346
|
+
(103, TType.STRING, 'customJson', 'UTF8', None, ), # 103
|
|
3347
|
+
None, # 104
|
|
3348
|
+
None, # 105
|
|
3349
|
+
None, # 106
|
|
3350
|
+
None, # 107
|
|
3351
|
+
None, # 108
|
|
3352
|
+
None, # 109
|
|
3353
|
+
None, # 110
|
|
3354
|
+
None, # 111
|
|
3355
|
+
None, # 112
|
|
3356
|
+
None, # 113
|
|
3357
|
+
None, # 114
|
|
3358
|
+
None, # 115
|
|
3359
|
+
None, # 116
|
|
3360
|
+
None, # 117
|
|
3361
|
+
None, # 118
|
|
3362
|
+
None, # 119
|
|
3363
|
+
None, # 120
|
|
3364
|
+
None, # 121
|
|
3365
|
+
None, # 122
|
|
3366
|
+
None, # 123
|
|
3367
|
+
None, # 124
|
|
3368
|
+
None, # 125
|
|
3369
|
+
None, # 126
|
|
3370
|
+
None, # 127
|
|
3371
|
+
None, # 128
|
|
3372
|
+
None, # 129
|
|
3373
|
+
None, # 130
|
|
3374
|
+
None, # 131
|
|
3375
|
+
None, # 132
|
|
3376
|
+
None, # 133
|
|
3377
|
+
None, # 134
|
|
3378
|
+
None, # 135
|
|
3379
|
+
None, # 136
|
|
3380
|
+
None, # 137
|
|
3381
|
+
None, # 138
|
|
3382
|
+
None, # 139
|
|
3383
|
+
None, # 140
|
|
3384
|
+
None, # 141
|
|
3385
|
+
None, # 142
|
|
3386
|
+
None, # 143
|
|
3387
|
+
None, # 144
|
|
3388
|
+
None, # 145
|
|
3389
|
+
None, # 146
|
|
3390
|
+
None, # 147
|
|
3391
|
+
None, # 148
|
|
3392
|
+
None, # 149
|
|
3393
|
+
None, # 150
|
|
3394
|
+
None, # 151
|
|
3395
|
+
None, # 152
|
|
3396
|
+
None, # 153
|
|
3397
|
+
None, # 154
|
|
3398
|
+
None, # 155
|
|
3399
|
+
None, # 156
|
|
3400
|
+
None, # 157
|
|
3401
|
+
None, # 158
|
|
3402
|
+
None, # 159
|
|
3403
|
+
None, # 160
|
|
3404
|
+
None, # 161
|
|
3405
|
+
None, # 162
|
|
3406
|
+
None, # 163
|
|
3407
|
+
None, # 164
|
|
3408
|
+
None, # 165
|
|
3409
|
+
None, # 166
|
|
3410
|
+
None, # 167
|
|
3411
|
+
None, # 168
|
|
3412
|
+
None, # 169
|
|
3413
|
+
None, # 170
|
|
3414
|
+
None, # 171
|
|
3415
|
+
None, # 172
|
|
3416
|
+
None, # 173
|
|
3417
|
+
None, # 174
|
|
3418
|
+
None, # 175
|
|
3419
|
+
None, # 176
|
|
3420
|
+
None, # 177
|
|
3421
|
+
None, # 178
|
|
3422
|
+
None, # 179
|
|
3423
|
+
None, # 180
|
|
3424
|
+
None, # 181
|
|
3425
|
+
None, # 182
|
|
3426
|
+
None, # 183
|
|
3427
|
+
None, # 184
|
|
3428
|
+
None, # 185
|
|
3429
|
+
None, # 186
|
|
3430
|
+
None, # 187
|
|
3431
|
+
None, # 188
|
|
3432
|
+
None, # 189
|
|
3433
|
+
None, # 190
|
|
3434
|
+
None, # 191
|
|
3435
|
+
None, # 192
|
|
3436
|
+
None, # 193
|
|
3437
|
+
None, # 194
|
|
3438
|
+
None, # 195
|
|
3439
|
+
None, # 196
|
|
3440
|
+
None, # 197
|
|
3441
|
+
None, # 198
|
|
3442
|
+
None, # 199
|
|
3443
|
+
(200, TType.BOOL, 'consistencyCheck', None, None, ), # 200
|
|
3444
|
+
(201, TType.DOUBLE, 'samplePercent', None, None, ), # 201
|
|
3445
|
+
(202, TType.DOUBLE, 'consistencySamplePercent', None, None, ), # 202
|
|
3446
|
+
(203, TType.STRUCT, 'driftSpec', [ai.chronon.observability.ttypes.DriftSpec, None], None, ), # 203
|
|
3447
|
+
(204, TType.STRUCT, 'executionInfo', [ai.chronon.api.common.ttypes.ExecutionInfo, None], None, ), # 204
|
|
3448
|
+
)
|
|
3449
|
+
all_structs.append(GroupBy)
|
|
3450
|
+
GroupBy.thrift_spec = (
|
|
3451
|
+
None, # 0
|
|
3452
|
+
(1, TType.STRUCT, 'metaData', [MetaData, None], None, ), # 1
|
|
3453
|
+
(2, TType.LIST, 'sources', (TType.STRUCT, [Source, None], False), None, ), # 2
|
|
3454
|
+
(3, TType.LIST, 'keyColumns', (TType.STRING, 'UTF8', False), None, ), # 3
|
|
3455
|
+
(4, TType.LIST, 'aggregations', (TType.STRUCT, [Aggregation, None], False), None, ), # 4
|
|
3456
|
+
(5, TType.I32, 'accuracy', None, None, ), # 5
|
|
3457
|
+
(6, TType.STRING, 'backfillStartDate', 'UTF8', None, ), # 6
|
|
3458
|
+
(7, TType.LIST, 'derivations', (TType.STRUCT, [Derivation, None], False), None, ), # 7
|
|
3459
|
+
)
|
|
3460
|
+
all_structs.append(JoinPart)
|
|
3461
|
+
JoinPart.thrift_spec = (
|
|
3462
|
+
None, # 0
|
|
3463
|
+
(1, TType.STRUCT, 'groupBy', [GroupBy, None], None, ), # 1
|
|
3464
|
+
(2, TType.MAP, 'keyMapping', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 2
|
|
3465
|
+
(3, TType.STRING, 'prefix', 'UTF8', None, ), # 3
|
|
3466
|
+
None, # 4
|
|
3467
|
+
None, # 5
|
|
3468
|
+
None, # 6
|
|
3469
|
+
None, # 7
|
|
3470
|
+
None, # 8
|
|
3471
|
+
None, # 9
|
|
3472
|
+
(10, TType.BOOL, 'useLongNames', None, None, ), # 10
|
|
3473
|
+
)
|
|
3474
|
+
all_structs.append(ExternalPart)
|
|
3475
|
+
ExternalPart.thrift_spec = (
|
|
3476
|
+
None, # 0
|
|
3477
|
+
(1, TType.STRUCT, 'source', [ExternalSource, None], None, ), # 1
|
|
3478
|
+
(2, TType.MAP, 'keyMapping', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 2
|
|
3479
|
+
(3, TType.STRING, 'prefix', 'UTF8', None, ), # 3
|
|
3480
|
+
)
|
|
3481
|
+
all_structs.append(Derivation)
|
|
3482
|
+
Derivation.thrift_spec = (
|
|
3483
|
+
None, # 0
|
|
3484
|
+
(1, TType.STRING, 'name', 'UTF8', None, ), # 1
|
|
3485
|
+
(2, TType.STRING, 'expression', 'UTF8', None, ), # 2
|
|
3486
|
+
)
|
|
3487
|
+
all_structs.append(Join)
|
|
3488
|
+
Join.thrift_spec = (
|
|
3489
|
+
None, # 0
|
|
3490
|
+
(1, TType.STRUCT, 'metaData', [MetaData, None], None, ), # 1
|
|
3491
|
+
(2, TType.STRUCT, 'left', [Source, None], None, ), # 2
|
|
3492
|
+
(3, TType.LIST, 'joinParts', (TType.STRUCT, [JoinPart, None], False), None, ), # 3
|
|
3493
|
+
(4, TType.MAP, 'skewKeys', (TType.STRING, 'UTF8', TType.LIST, (TType.STRING, 'UTF8', False), False), None, ), # 4
|
|
3494
|
+
(5, TType.LIST, 'onlineExternalParts', (TType.STRUCT, [ExternalPart, None], False), None, ), # 5
|
|
3495
|
+
(6, TType.STRUCT, 'labelParts', [LabelParts, None], None, ), # 6
|
|
3496
|
+
(7, TType.LIST, 'bootstrapParts', (TType.STRUCT, [BootstrapPart, None], False), None, ), # 7
|
|
3497
|
+
(8, TType.LIST, 'rowIds', (TType.STRING, 'UTF8', False), None, ), # 8
|
|
3498
|
+
(9, TType.LIST, 'derivations', (TType.STRUCT, [Derivation, None], False), None, ), # 9
|
|
3499
|
+
None, # 10
|
|
3500
|
+
None, # 11
|
|
3501
|
+
None, # 12
|
|
3502
|
+
None, # 13
|
|
3503
|
+
None, # 14
|
|
3504
|
+
None, # 15
|
|
3505
|
+
None, # 16
|
|
3506
|
+
None, # 17
|
|
3507
|
+
None, # 18
|
|
3508
|
+
None, # 19
|
|
3509
|
+
None, # 20
|
|
3510
|
+
None, # 21
|
|
3511
|
+
None, # 22
|
|
3512
|
+
None, # 23
|
|
3513
|
+
None, # 24
|
|
3514
|
+
None, # 25
|
|
3515
|
+
None, # 26
|
|
3516
|
+
None, # 27
|
|
3517
|
+
None, # 28
|
|
3518
|
+
None, # 29
|
|
3519
|
+
None, # 30
|
|
3520
|
+
None, # 31
|
|
3521
|
+
None, # 32
|
|
3522
|
+
None, # 33
|
|
3523
|
+
None, # 34
|
|
3524
|
+
None, # 35
|
|
3525
|
+
None, # 36
|
|
3526
|
+
None, # 37
|
|
3527
|
+
None, # 38
|
|
3528
|
+
None, # 39
|
|
3529
|
+
None, # 40
|
|
3530
|
+
None, # 41
|
|
3531
|
+
None, # 42
|
|
3532
|
+
None, # 43
|
|
3533
|
+
None, # 44
|
|
3534
|
+
None, # 45
|
|
3535
|
+
None, # 46
|
|
3536
|
+
None, # 47
|
|
3537
|
+
None, # 48
|
|
3538
|
+
None, # 49
|
|
3539
|
+
(50, TType.BOOL, 'useLongNames', None, None, ), # 50
|
|
3540
|
+
)
|
|
3541
|
+
all_structs.append(BootstrapPart)
|
|
3542
|
+
BootstrapPart.thrift_spec = (
|
|
3543
|
+
None, # 0
|
|
3544
|
+
(1, TType.STRUCT, 'metaData', [MetaData, None], None, ), # 1
|
|
3545
|
+
(2, TType.STRING, 'table', 'UTF8', None, ), # 2
|
|
3546
|
+
(3, TType.STRUCT, 'query', [Query, None], None, ), # 3
|
|
3547
|
+
(4, TType.LIST, 'keyColumns', (TType.STRING, 'UTF8', False), None, ), # 4
|
|
3548
|
+
)
|
|
3549
|
+
all_structs.append(LabelParts)
|
|
3550
|
+
LabelParts.thrift_spec = (
|
|
3551
|
+
None, # 0
|
|
3552
|
+
(1, TType.LIST, 'labels', (TType.STRUCT, [JoinPart, None], False), None, ), # 1
|
|
3553
|
+
(2, TType.I32, 'leftStartOffset', None, None, ), # 2
|
|
3554
|
+
(3, TType.I32, 'leftEndOffset', None, None, ), # 3
|
|
3555
|
+
(4, TType.STRUCT, 'metaData', [MetaData, None], None, ), # 4
|
|
3556
|
+
)
|
|
3557
|
+
all_structs.append(GroupByServingInfo)
|
|
3558
|
+
GroupByServingInfo.thrift_spec = (
|
|
3559
|
+
None, # 0
|
|
3560
|
+
(1, TType.STRUCT, 'groupBy', [GroupBy, None], None, ), # 1
|
|
3561
|
+
(2, TType.STRING, 'inputAvroSchema', 'UTF8', None, ), # 2
|
|
3562
|
+
(3, TType.STRING, 'selectedAvroSchema', 'UTF8', None, ), # 3
|
|
3563
|
+
(4, TType.STRING, 'keyAvroSchema', 'UTF8', None, ), # 4
|
|
3564
|
+
(5, TType.STRING, 'batchEndDate', 'UTF8', None, ), # 5
|
|
3565
|
+
(6, TType.STRING, 'dateFormat', 'UTF8', None, ), # 6
|
|
3566
|
+
)
|
|
3567
|
+
all_structs.append(DataField)
|
|
3568
|
+
DataField.thrift_spec = (
|
|
3569
|
+
None, # 0
|
|
3570
|
+
(1, TType.STRING, 'name', 'UTF8', None, ), # 1
|
|
3571
|
+
(2, TType.STRUCT, 'dataType', [TDataType, None], None, ), # 2
|
|
3572
|
+
)
|
|
3573
|
+
all_structs.append(TDataType)
|
|
3574
|
+
TDataType.thrift_spec = (
|
|
3575
|
+
None, # 0
|
|
3576
|
+
(1, TType.I32, 'kind', None, None, ), # 1
|
|
3577
|
+
(2, TType.LIST, 'params', (TType.STRUCT, [DataField, None], False), None, ), # 2
|
|
3578
|
+
(3, TType.STRING, 'name', 'UTF8', None, ), # 3
|
|
3579
|
+
)
|
|
3580
|
+
all_structs.append(DataSpec)
|
|
3581
|
+
DataSpec.thrift_spec = (
|
|
3582
|
+
None, # 0
|
|
3583
|
+
(1, TType.STRUCT, 'schema', [TDataType, None], None, ), # 1
|
|
3584
|
+
(2, TType.LIST, 'partitionColumns', (TType.STRING, 'UTF8', False), None, ), # 2
|
|
3585
|
+
(3, TType.I32, 'retentionDays', None, None, ), # 3
|
|
3586
|
+
(4, TType.MAP, 'props', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 4
|
|
3587
|
+
)
|
|
3588
|
+
all_structs.append(Model)
|
|
3589
|
+
Model.thrift_spec = (
|
|
3590
|
+
None, # 0
|
|
3591
|
+
(1, TType.STRUCT, 'metaData', [MetaData, None], None, ), # 1
|
|
3592
|
+
(2, TType.I32, 'modelType', None, None, ), # 2
|
|
3593
|
+
(3, TType.STRUCT, 'outputSchema', [TDataType, None], None, ), # 3
|
|
3594
|
+
(4, TType.STRUCT, 'source', [Source, None], None, ), # 4
|
|
3595
|
+
(5, TType.MAP, 'modelParams', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 5
|
|
3596
|
+
)
|
|
3597
|
+
all_structs.append(Team)
|
|
3598
|
+
Team.thrift_spec = (
|
|
3599
|
+
None, # 0
|
|
3600
|
+
(1, TType.STRING, 'name', 'UTF8', None, ), # 1
|
|
3601
|
+
(2, TType.STRING, 'description', 'UTF8', None, ), # 2
|
|
3602
|
+
(3, TType.STRING, 'email', 'UTF8', None, ), # 3
|
|
3603
|
+
None, # 4
|
|
3604
|
+
None, # 5
|
|
3605
|
+
None, # 6
|
|
3606
|
+
None, # 7
|
|
3607
|
+
None, # 8
|
|
3608
|
+
None, # 9
|
|
3609
|
+
(10, TType.STRING, 'outputNamespace', 'UTF8', None, ), # 10
|
|
3610
|
+
(11, TType.MAP, 'tableProperties', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 11
|
|
3611
|
+
None, # 12
|
|
3612
|
+
None, # 13
|
|
3613
|
+
None, # 14
|
|
3614
|
+
None, # 15
|
|
3615
|
+
None, # 16
|
|
3616
|
+
None, # 17
|
|
3617
|
+
None, # 18
|
|
3618
|
+
None, # 19
|
|
3619
|
+
(20, TType.STRUCT, 'env', [ai.chronon.api.common.ttypes.EnvironmentVariables, None], None, ), # 20
|
|
3620
|
+
(21, TType.STRUCT, 'conf', [ai.chronon.api.common.ttypes.ConfigProperties, None], None, ), # 21
|
|
3621
|
+
(22, TType.STRUCT, 'clusterConf', [ai.chronon.api.common.ttypes.ClusterConfigProperties, None], None, ), # 22
|
|
3622
|
+
)
|
|
3623
|
+
fix_spec(all_structs)
|
|
3624
|
+
del all_structs
|