awx-zipline-ai 0.0.32__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (96) hide show
  1. __init__.py +0 -0
  2. agent/__init__.py +1 -0
  3. agent/constants.py +15 -0
  4. agent/ttypes.py +1684 -0
  5. ai/__init__.py +0 -0
  6. ai/chronon/__init__.py +0 -0
  7. ai/chronon/airflow_helpers.py +248 -0
  8. ai/chronon/cli/__init__.py +0 -0
  9. ai/chronon/cli/compile/__init__.py +0 -0
  10. ai/chronon/cli/compile/column_hashing.py +336 -0
  11. ai/chronon/cli/compile/compile_context.py +173 -0
  12. ai/chronon/cli/compile/compiler.py +183 -0
  13. ai/chronon/cli/compile/conf_validator.py +742 -0
  14. ai/chronon/cli/compile/display/__init__.py +0 -0
  15. ai/chronon/cli/compile/display/class_tracker.py +102 -0
  16. ai/chronon/cli/compile/display/compile_status.py +95 -0
  17. ai/chronon/cli/compile/display/compiled_obj.py +12 -0
  18. ai/chronon/cli/compile/display/console.py +3 -0
  19. ai/chronon/cli/compile/display/diff_result.py +111 -0
  20. ai/chronon/cli/compile/fill_templates.py +35 -0
  21. ai/chronon/cli/compile/parse_configs.py +134 -0
  22. ai/chronon/cli/compile/parse_teams.py +242 -0
  23. ai/chronon/cli/compile/serializer.py +109 -0
  24. ai/chronon/cli/compile/version_utils.py +42 -0
  25. ai/chronon/cli/git_utils.py +145 -0
  26. ai/chronon/cli/logger.py +59 -0
  27. ai/chronon/constants.py +3 -0
  28. ai/chronon/group_by.py +692 -0
  29. ai/chronon/join.py +580 -0
  30. ai/chronon/logger.py +23 -0
  31. ai/chronon/model.py +40 -0
  32. ai/chronon/query.py +126 -0
  33. ai/chronon/repo/__init__.py +39 -0
  34. ai/chronon/repo/aws.py +284 -0
  35. ai/chronon/repo/cluster.py +136 -0
  36. ai/chronon/repo/compile.py +62 -0
  37. ai/chronon/repo/constants.py +164 -0
  38. ai/chronon/repo/default_runner.py +269 -0
  39. ai/chronon/repo/explore.py +418 -0
  40. ai/chronon/repo/extract_objects.py +134 -0
  41. ai/chronon/repo/gcp.py +586 -0
  42. ai/chronon/repo/gitpython_utils.py +15 -0
  43. ai/chronon/repo/hub_runner.py +261 -0
  44. ai/chronon/repo/hub_uploader.py +109 -0
  45. ai/chronon/repo/init.py +60 -0
  46. ai/chronon/repo/join_backfill.py +119 -0
  47. ai/chronon/repo/run.py +296 -0
  48. ai/chronon/repo/serializer.py +133 -0
  49. ai/chronon/repo/team_json_utils.py +46 -0
  50. ai/chronon/repo/utils.py +481 -0
  51. ai/chronon/repo/zipline.py +35 -0
  52. ai/chronon/repo/zipline_hub.py +277 -0
  53. ai/chronon/resources/__init__.py +0 -0
  54. ai/chronon/resources/gcp/__init__.py +0 -0
  55. ai/chronon/resources/gcp/group_bys/__init__.py +0 -0
  56. ai/chronon/resources/gcp/group_bys/test/__init__.py +0 -0
  57. ai/chronon/resources/gcp/group_bys/test/data.py +30 -0
  58. ai/chronon/resources/gcp/joins/__init__.py +0 -0
  59. ai/chronon/resources/gcp/joins/test/__init__.py +0 -0
  60. ai/chronon/resources/gcp/joins/test/data.py +26 -0
  61. ai/chronon/resources/gcp/sources/__init__.py +0 -0
  62. ai/chronon/resources/gcp/sources/test/__init__.py +0 -0
  63. ai/chronon/resources/gcp/sources/test/data.py +26 -0
  64. ai/chronon/resources/gcp/teams.py +58 -0
  65. ai/chronon/source.py +86 -0
  66. ai/chronon/staging_query.py +226 -0
  67. ai/chronon/types.py +58 -0
  68. ai/chronon/utils.py +510 -0
  69. ai/chronon/windows.py +48 -0
  70. awx_zipline_ai-0.0.32.dist-info/METADATA +197 -0
  71. awx_zipline_ai-0.0.32.dist-info/RECORD +96 -0
  72. awx_zipline_ai-0.0.32.dist-info/WHEEL +5 -0
  73. awx_zipline_ai-0.0.32.dist-info/entry_points.txt +2 -0
  74. awx_zipline_ai-0.0.32.dist-info/top_level.txt +4 -0
  75. gen_thrift/__init__.py +0 -0
  76. gen_thrift/api/__init__.py +1 -0
  77. gen_thrift/api/constants.py +15 -0
  78. gen_thrift/api/ttypes.py +3754 -0
  79. gen_thrift/common/__init__.py +1 -0
  80. gen_thrift/common/constants.py +15 -0
  81. gen_thrift/common/ttypes.py +1814 -0
  82. gen_thrift/eval/__init__.py +1 -0
  83. gen_thrift/eval/constants.py +15 -0
  84. gen_thrift/eval/ttypes.py +660 -0
  85. gen_thrift/fetcher/__init__.py +1 -0
  86. gen_thrift/fetcher/constants.py +15 -0
  87. gen_thrift/fetcher/ttypes.py +127 -0
  88. gen_thrift/hub/__init__.py +1 -0
  89. gen_thrift/hub/constants.py +15 -0
  90. gen_thrift/hub/ttypes.py +1109 -0
  91. gen_thrift/observability/__init__.py +1 -0
  92. gen_thrift/observability/constants.py +15 -0
  93. gen_thrift/observability/ttypes.py +2355 -0
  94. gen_thrift/planner/__init__.py +1 -0
  95. gen_thrift/planner/constants.py +15 -0
  96. gen_thrift/planner/ttypes.py +1967 -0
@@ -0,0 +1,3754 @@
1
+ #
2
+ # Autogenerated by Thrift Compiler (0.22.0)
3
+ #
4
+ # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
5
+ #
6
+ # options string: py
7
+ #
8
+
9
+ from thrift.Thrift import TType, TMessageType, TFrozenDict, TException, TApplicationException
10
+ from thrift.protocol.TProtocol import TProtocolException
11
+ from thrift.TRecursive import fix_spec
12
+ from uuid import UUID
13
+
14
+ import sys
15
+ import gen_thrift.common.ttypes
16
+ import gen_thrift.observability.ttypes
17
+
18
+ from thrift.transport import TTransport
19
+ all_structs = []
20
+
21
+
22
+ class Operation(object):
23
+ MIN = 0
24
+ MAX = 1
25
+ FIRST = 2
26
+ LAST = 3
27
+ UNIQUE_COUNT = 4
28
+ APPROX_UNIQUE_COUNT = 5
29
+ COUNT = 6
30
+ SUM = 7
31
+ AVERAGE = 8
32
+ VARIANCE = 9
33
+ SKEW = 10
34
+ KURTOSIS = 11
35
+ APPROX_PERCENTILE = 12
36
+ LAST_K = 13
37
+ FIRST_K = 14
38
+ TOP_K = 15
39
+ BOTTOM_K = 16
40
+ HISTOGRAM = 17
41
+ APPROX_FREQUENT_K = 18
42
+ APPROX_HEAVY_HITTERS_K = 19
43
+ UNIQUE_TOP_K = 20
44
+
45
+ _VALUES_TO_NAMES = {
46
+ 0: "MIN",
47
+ 1: "MAX",
48
+ 2: "FIRST",
49
+ 3: "LAST",
50
+ 4: "UNIQUE_COUNT",
51
+ 5: "APPROX_UNIQUE_COUNT",
52
+ 6: "COUNT",
53
+ 7: "SUM",
54
+ 8: "AVERAGE",
55
+ 9: "VARIANCE",
56
+ 10: "SKEW",
57
+ 11: "KURTOSIS",
58
+ 12: "APPROX_PERCENTILE",
59
+ 13: "LAST_K",
60
+ 14: "FIRST_K",
61
+ 15: "TOP_K",
62
+ 16: "BOTTOM_K",
63
+ 17: "HISTOGRAM",
64
+ 18: "APPROX_FREQUENT_K",
65
+ 19: "APPROX_HEAVY_HITTERS_K",
66
+ 20: "UNIQUE_TOP_K",
67
+ }
68
+
69
+ _NAMES_TO_VALUES = {
70
+ "MIN": 0,
71
+ "MAX": 1,
72
+ "FIRST": 2,
73
+ "LAST": 3,
74
+ "UNIQUE_COUNT": 4,
75
+ "APPROX_UNIQUE_COUNT": 5,
76
+ "COUNT": 6,
77
+ "SUM": 7,
78
+ "AVERAGE": 8,
79
+ "VARIANCE": 9,
80
+ "SKEW": 10,
81
+ "KURTOSIS": 11,
82
+ "APPROX_PERCENTILE": 12,
83
+ "LAST_K": 13,
84
+ "FIRST_K": 14,
85
+ "TOP_K": 15,
86
+ "BOTTOM_K": 16,
87
+ "HISTOGRAM": 17,
88
+ "APPROX_FREQUENT_K": 18,
89
+ "APPROX_HEAVY_HITTERS_K": 19,
90
+ "UNIQUE_TOP_K": 20,
91
+ }
92
+
93
+
94
+ class Accuracy(object):
95
+ TEMPORAL = 0
96
+ SNAPSHOT = 1
97
+
98
+ _VALUES_TO_NAMES = {
99
+ 0: "TEMPORAL",
100
+ 1: "SNAPSHOT",
101
+ }
102
+
103
+ _NAMES_TO_VALUES = {
104
+ "TEMPORAL": 0,
105
+ "SNAPSHOT": 1,
106
+ }
107
+
108
+
109
+ class EngineType(object):
110
+ SPARK = 0
111
+ BIGQUERY = 1
112
+
113
+ _VALUES_TO_NAMES = {
114
+ 0: "SPARK",
115
+ 1: "BIGQUERY",
116
+ }
117
+
118
+ _NAMES_TO_VALUES = {
119
+ "SPARK": 0,
120
+ "BIGQUERY": 1,
121
+ }
122
+
123
+
124
+ class ConfType(object):
125
+ GROUP_BY = 0
126
+ JOIN = 1
127
+ STAGING_QUERY = 2
128
+ MODEL = 3
129
+
130
+ _VALUES_TO_NAMES = {
131
+ 0: "GROUP_BY",
132
+ 1: "JOIN",
133
+ 2: "STAGING_QUERY",
134
+ 3: "MODEL",
135
+ }
136
+
137
+ _NAMES_TO_VALUES = {
138
+ "GROUP_BY": 0,
139
+ "JOIN": 1,
140
+ "STAGING_QUERY": 2,
141
+ "MODEL": 3,
142
+ }
143
+
144
+
145
+ class DataKind(object):
146
+ BOOLEAN = 0
147
+ BYTE = 1
148
+ SHORT = 2
149
+ INT = 3
150
+ LONG = 4
151
+ FLOAT = 5
152
+ DOUBLE = 6
153
+ STRING = 7
154
+ BINARY = 8
155
+ DATE = 9
156
+ TIMESTAMP = 10
157
+ MAP = 11
158
+ LIST = 12
159
+ STRUCT = 13
160
+
161
+ _VALUES_TO_NAMES = {
162
+ 0: "BOOLEAN",
163
+ 1: "BYTE",
164
+ 2: "SHORT",
165
+ 3: "INT",
166
+ 4: "LONG",
167
+ 5: "FLOAT",
168
+ 6: "DOUBLE",
169
+ 7: "STRING",
170
+ 8: "BINARY",
171
+ 9: "DATE",
172
+ 10: "TIMESTAMP",
173
+ 11: "MAP",
174
+ 12: "LIST",
175
+ 13: "STRUCT",
176
+ }
177
+
178
+ _NAMES_TO_VALUES = {
179
+ "BOOLEAN": 0,
180
+ "BYTE": 1,
181
+ "SHORT": 2,
182
+ "INT": 3,
183
+ "LONG": 4,
184
+ "FLOAT": 5,
185
+ "DOUBLE": 6,
186
+ "STRING": 7,
187
+ "BINARY": 8,
188
+ "DATE": 9,
189
+ "TIMESTAMP": 10,
190
+ "MAP": 11,
191
+ "LIST": 12,
192
+ "STRUCT": 13,
193
+ }
194
+
195
+
196
+ class ModelType(object):
197
+ XGBoost = 0
198
+ PyTorch = 1
199
+ TensorFlow = 2
200
+ ScikitLearn = 3
201
+ LightGBM = 4
202
+ Other = 100
203
+
204
+ _VALUES_TO_NAMES = {
205
+ 0: "XGBoost",
206
+ 1: "PyTorch",
207
+ 2: "TensorFlow",
208
+ 3: "ScikitLearn",
209
+ 4: "LightGBM",
210
+ 100: "Other",
211
+ }
212
+
213
+ _NAMES_TO_VALUES = {
214
+ "XGBoost": 0,
215
+ "PyTorch": 1,
216
+ "TensorFlow": 2,
217
+ "ScikitLearn": 3,
218
+ "LightGBM": 4,
219
+ "Other": 100,
220
+ }
221
+
222
+
223
+ class DataModel(object):
224
+ ENTITIES = 0
225
+ EVENTS = 1
226
+
227
+ _VALUES_TO_NAMES = {
228
+ 0: "ENTITIES",
229
+ 1: "EVENTS",
230
+ }
231
+
232
+ _NAMES_TO_VALUES = {
233
+ "ENTITIES": 0,
234
+ "EVENTS": 1,
235
+ }
236
+
237
+
238
+ class Query(object):
239
+ """
240
+ Attributes:
241
+ - selects
242
+ - wheres
243
+ - startPartition
244
+ - endPartition
245
+ - timeColumn
246
+ - setups
247
+ - mutationTimeColumn
248
+ - reversalColumn
249
+ - partitionColumn: Chronon expects all its batch input data to be date/time partitioned.
250
+ We in-turn produce partitioned outputs.
251
+
252
+ - partitionFormat: Partition format in the java DateFormatter spec:
253
+ see: https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html
254
+
255
+ - partitionInterval: Indicates the timespan of a given interval of the source
256
+
257
+ - partitionLag: Indicates how often this source is typically delayed by.
258
+ Should be a multiple of partitionInterval
259
+
260
+ - subPartitionsToWaitFor: Additional partitions to be used in sensing that the source data has landed.
261
+ Should be a full partition string, such as `hr=23:00'
262
+
263
+
264
+ """
265
+ thrift_spec = None
266
+
267
+
268
+ def __init__(self, selects = None, wheres = None, startPartition = None, endPartition = None, timeColumn = None, setups = None, mutationTimeColumn = None, reversalColumn = None, partitionColumn = None, partitionFormat = None, partitionInterval = None, partitionLag = None, subPartitionsToWaitFor = None,):
269
+ self.selects = selects
270
+ self.wheres = wheres
271
+ self.startPartition = startPartition
272
+ self.endPartition = endPartition
273
+ self.timeColumn = timeColumn
274
+ self.setups = setups
275
+ self.mutationTimeColumn = mutationTimeColumn
276
+ self.reversalColumn = reversalColumn
277
+ self.partitionColumn = partitionColumn
278
+ self.partitionFormat = partitionFormat
279
+ self.partitionInterval = partitionInterval
280
+ self.partitionLag = partitionLag
281
+ self.subPartitionsToWaitFor = subPartitionsToWaitFor
282
+
283
+ def read(self, iprot):
284
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
285
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
286
+ return
287
+ iprot.readStructBegin()
288
+ while True:
289
+ (fname, ftype, fid) = iprot.readFieldBegin()
290
+ if ftype == TType.STOP:
291
+ break
292
+ if fid == 1:
293
+ if ftype == TType.MAP:
294
+ self.selects = {}
295
+ (_ktype1, _vtype2, _size0) = iprot.readMapBegin()
296
+ for _i4 in range(_size0):
297
+ _key5 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
298
+ _val6 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
299
+ self.selects[_key5] = _val6
300
+ iprot.readMapEnd()
301
+ else:
302
+ iprot.skip(ftype)
303
+ elif fid == 2:
304
+ if ftype == TType.LIST:
305
+ self.wheres = []
306
+ (_etype10, _size7) = iprot.readListBegin()
307
+ for _i11 in range(_size7):
308
+ _elem12 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
309
+ self.wheres.append(_elem12)
310
+ iprot.readListEnd()
311
+ else:
312
+ iprot.skip(ftype)
313
+ elif fid == 3:
314
+ if ftype == TType.STRING:
315
+ self.startPartition = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
316
+ else:
317
+ iprot.skip(ftype)
318
+ elif fid == 4:
319
+ if ftype == TType.STRING:
320
+ self.endPartition = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
321
+ else:
322
+ iprot.skip(ftype)
323
+ elif fid == 5:
324
+ if ftype == TType.STRING:
325
+ self.timeColumn = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
326
+ else:
327
+ iprot.skip(ftype)
328
+ elif fid == 6:
329
+ if ftype == TType.LIST:
330
+ self.setups = []
331
+ (_etype16, _size13) = iprot.readListBegin()
332
+ for _i17 in range(_size13):
333
+ _elem18 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
334
+ self.setups.append(_elem18)
335
+ iprot.readListEnd()
336
+ else:
337
+ iprot.skip(ftype)
338
+ elif fid == 7:
339
+ if ftype == TType.STRING:
340
+ self.mutationTimeColumn = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
341
+ else:
342
+ iprot.skip(ftype)
343
+ elif fid == 8:
344
+ if ftype == TType.STRING:
345
+ self.reversalColumn = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
346
+ else:
347
+ iprot.skip(ftype)
348
+ elif fid == 20:
349
+ if ftype == TType.STRING:
350
+ self.partitionColumn = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
351
+ else:
352
+ iprot.skip(ftype)
353
+ elif fid == 21:
354
+ if ftype == TType.STRING:
355
+ self.partitionFormat = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
356
+ else:
357
+ iprot.skip(ftype)
358
+ elif fid == 22:
359
+ if ftype == TType.STRUCT:
360
+ self.partitionInterval = gen_thrift.common.ttypes.Window()
361
+ self.partitionInterval.read(iprot)
362
+ else:
363
+ iprot.skip(ftype)
364
+ elif fid == 23:
365
+ if ftype == TType.STRUCT:
366
+ self.partitionLag = gen_thrift.common.ttypes.Window()
367
+ self.partitionLag.read(iprot)
368
+ else:
369
+ iprot.skip(ftype)
370
+ elif fid == 24:
371
+ if ftype == TType.LIST:
372
+ self.subPartitionsToWaitFor = []
373
+ (_etype22, _size19) = iprot.readListBegin()
374
+ for _i23 in range(_size19):
375
+ _elem24 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
376
+ self.subPartitionsToWaitFor.append(_elem24)
377
+ iprot.readListEnd()
378
+ else:
379
+ iprot.skip(ftype)
380
+ else:
381
+ iprot.skip(ftype)
382
+ iprot.readFieldEnd()
383
+ iprot.readStructEnd()
384
+
385
+ def write(self, oprot):
386
+ self.validate()
387
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
388
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
389
+ return
390
+ oprot.writeStructBegin('Query')
391
+ if self.selects is not None:
392
+ oprot.writeFieldBegin('selects', TType.MAP, 1)
393
+ oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.selects))
394
+ for kiter25, viter26 in self.selects.items():
395
+ oprot.writeString(kiter25.encode('utf-8') if sys.version_info[0] == 2 else kiter25)
396
+ oprot.writeString(viter26.encode('utf-8') if sys.version_info[0] == 2 else viter26)
397
+ oprot.writeMapEnd()
398
+ oprot.writeFieldEnd()
399
+ if self.wheres is not None:
400
+ oprot.writeFieldBegin('wheres', TType.LIST, 2)
401
+ oprot.writeListBegin(TType.STRING, len(self.wheres))
402
+ for iter27 in self.wheres:
403
+ oprot.writeString(iter27.encode('utf-8') if sys.version_info[0] == 2 else iter27)
404
+ oprot.writeListEnd()
405
+ oprot.writeFieldEnd()
406
+ if self.startPartition is not None:
407
+ oprot.writeFieldBegin('startPartition', TType.STRING, 3)
408
+ oprot.writeString(self.startPartition.encode('utf-8') if sys.version_info[0] == 2 else self.startPartition)
409
+ oprot.writeFieldEnd()
410
+ if self.endPartition is not None:
411
+ oprot.writeFieldBegin('endPartition', TType.STRING, 4)
412
+ oprot.writeString(self.endPartition.encode('utf-8') if sys.version_info[0] == 2 else self.endPartition)
413
+ oprot.writeFieldEnd()
414
+ if self.timeColumn is not None:
415
+ oprot.writeFieldBegin('timeColumn', TType.STRING, 5)
416
+ oprot.writeString(self.timeColumn.encode('utf-8') if sys.version_info[0] == 2 else self.timeColumn)
417
+ oprot.writeFieldEnd()
418
+ if self.setups is not None:
419
+ oprot.writeFieldBegin('setups', TType.LIST, 6)
420
+ oprot.writeListBegin(TType.STRING, len(self.setups))
421
+ for iter28 in self.setups:
422
+ oprot.writeString(iter28.encode('utf-8') if sys.version_info[0] == 2 else iter28)
423
+ oprot.writeListEnd()
424
+ oprot.writeFieldEnd()
425
+ if self.mutationTimeColumn is not None:
426
+ oprot.writeFieldBegin('mutationTimeColumn', TType.STRING, 7)
427
+ oprot.writeString(self.mutationTimeColumn.encode('utf-8') if sys.version_info[0] == 2 else self.mutationTimeColumn)
428
+ oprot.writeFieldEnd()
429
+ if self.reversalColumn is not None:
430
+ oprot.writeFieldBegin('reversalColumn', TType.STRING, 8)
431
+ oprot.writeString(self.reversalColumn.encode('utf-8') if sys.version_info[0] == 2 else self.reversalColumn)
432
+ oprot.writeFieldEnd()
433
+ if self.partitionColumn is not None:
434
+ oprot.writeFieldBegin('partitionColumn', TType.STRING, 20)
435
+ oprot.writeString(self.partitionColumn.encode('utf-8') if sys.version_info[0] == 2 else self.partitionColumn)
436
+ oprot.writeFieldEnd()
437
+ if self.partitionFormat is not None:
438
+ oprot.writeFieldBegin('partitionFormat', TType.STRING, 21)
439
+ oprot.writeString(self.partitionFormat.encode('utf-8') if sys.version_info[0] == 2 else self.partitionFormat)
440
+ oprot.writeFieldEnd()
441
+ if self.partitionInterval is not None:
442
+ oprot.writeFieldBegin('partitionInterval', TType.STRUCT, 22)
443
+ self.partitionInterval.write(oprot)
444
+ oprot.writeFieldEnd()
445
+ if self.partitionLag is not None:
446
+ oprot.writeFieldBegin('partitionLag', TType.STRUCT, 23)
447
+ self.partitionLag.write(oprot)
448
+ oprot.writeFieldEnd()
449
+ if self.subPartitionsToWaitFor is not None:
450
+ oprot.writeFieldBegin('subPartitionsToWaitFor', TType.LIST, 24)
451
+ oprot.writeListBegin(TType.STRING, len(self.subPartitionsToWaitFor))
452
+ for iter29 in self.subPartitionsToWaitFor:
453
+ oprot.writeString(iter29.encode('utf-8') if sys.version_info[0] == 2 else iter29)
454
+ oprot.writeListEnd()
455
+ oprot.writeFieldEnd()
456
+ oprot.writeFieldStop()
457
+ oprot.writeStructEnd()
458
+
459
+ def validate(self):
460
+ return
461
+
462
+ def __repr__(self):
463
+ L = ['%s=%r' % (key, value)
464
+ for key, value in self.__dict__.items()]
465
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
466
+
467
+ def __eq__(self, other):
468
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
469
+
470
+ def __ne__(self, other):
471
+ return not (self == other)
472
+
473
+
474
+ class StagingQuery(object):
475
+ """
476
+ Staging Query encapsulates arbitrary spark computation. One key feature is that the computation follows a
477
+ "fill-what's-missing" pattern. Basically instead of explicitly specifying dates you specify two macros.
478
+ `{{ start_date }}` and `{{end_date}}`. Chronon will pass in earliest-missing-partition for `start_date` and
479
+ execution-date / today for `end_date`. So the query will compute multiple partitions at once.
480
+
481
+ Attributes:
482
+ - metaData: Contains name, team, output_namespace, execution parameters etc. Things that don't change the semantics of the computation itself.
483
+
484
+ - query: Arbitrary spark query that should be written with `{{ start_date }}`, `{{ end_date }}` and `{{ latest_date }}` templates
485
+ - `{{ start_date }}` will be set to this user provided start date, future incremental runs will set it to the latest existing partition + 1 day.
486
+ - `{{ start_date(offset=-10, lower_bound='2023-01-01', upper_bound='2024-01-01') }}` will shift the date back one day and bound it with lower and upper bounds.
487
+ - `{{ end_date }}` is the end partition of the computing range. offsetting and bounding the end_date also works as described above.
488
+ - `{{ latest_date }}` is the end partition independent of the computing range (meant for cumulative sources). offsetting and bounding the end_date also works as described above.
489
+ - `{{ max_date(table=namespace.my_table) }}` is the max partition available for a given table.
490
+
491
+
492
+ - startPartition: on the first run, `{{ start_date }}` will be set to this user provided start date, future incremental runs will set it to the latest existing partition + 1 day.
493
+
494
+ - setups: Spark SQL setup statements. Used typically to register UDFs.
495
+
496
+ - engineType: By default, spark is the compute engine. You can specify an override (eg. bigquery, etc.)
497
+
498
+ - tableDependencies
499
+ - recomputeDays
500
+
501
+ """
502
+ thrift_spec = None
503
+
504
+
505
+ def __init__(self, metaData = None, query = None, startPartition = None, setups = None, engineType = None, tableDependencies = None, recomputeDays = None,):
506
+ self.metaData = metaData
507
+ self.query = query
508
+ self.startPartition = startPartition
509
+ self.setups = setups
510
+ self.engineType = engineType
511
+ self.tableDependencies = tableDependencies
512
+ self.recomputeDays = recomputeDays
513
+
514
+ def read(self, iprot):
515
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
516
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
517
+ return
518
+ iprot.readStructBegin()
519
+ while True:
520
+ (fname, ftype, fid) = iprot.readFieldBegin()
521
+ if ftype == TType.STOP:
522
+ break
523
+ if fid == 1:
524
+ if ftype == TType.STRUCT:
525
+ self.metaData = MetaData()
526
+ self.metaData.read(iprot)
527
+ else:
528
+ iprot.skip(ftype)
529
+ elif fid == 2:
530
+ if ftype == TType.STRING:
531
+ self.query = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
532
+ else:
533
+ iprot.skip(ftype)
534
+ elif fid == 3:
535
+ if ftype == TType.STRING:
536
+ self.startPartition = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
537
+ else:
538
+ iprot.skip(ftype)
539
+ elif fid == 4:
540
+ if ftype == TType.LIST:
541
+ self.setups = []
542
+ (_etype33, _size30) = iprot.readListBegin()
543
+ for _i34 in range(_size30):
544
+ _elem35 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
545
+ self.setups.append(_elem35)
546
+ iprot.readListEnd()
547
+ else:
548
+ iprot.skip(ftype)
549
+ elif fid == 5:
550
+ if ftype == TType.I32:
551
+ self.engineType = iprot.readI32()
552
+ else:
553
+ iprot.skip(ftype)
554
+ elif fid == 6:
555
+ if ftype == TType.LIST:
556
+ self.tableDependencies = []
557
+ (_etype39, _size36) = iprot.readListBegin()
558
+ for _i40 in range(_size36):
559
+ _elem41 = gen_thrift.common.ttypes.TableDependency()
560
+ _elem41.read(iprot)
561
+ self.tableDependencies.append(_elem41)
562
+ iprot.readListEnd()
563
+ else:
564
+ iprot.skip(ftype)
565
+ elif fid == 20:
566
+ if ftype == TType.I32:
567
+ self.recomputeDays = iprot.readI32()
568
+ else:
569
+ iprot.skip(ftype)
570
+ else:
571
+ iprot.skip(ftype)
572
+ iprot.readFieldEnd()
573
+ iprot.readStructEnd()
574
+
575
+ def write(self, oprot):
576
+ self.validate()
577
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
578
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
579
+ return
580
+ oprot.writeStructBegin('StagingQuery')
581
+ if self.metaData is not None:
582
+ oprot.writeFieldBegin('metaData', TType.STRUCT, 1)
583
+ self.metaData.write(oprot)
584
+ oprot.writeFieldEnd()
585
+ if self.query is not None:
586
+ oprot.writeFieldBegin('query', TType.STRING, 2)
587
+ oprot.writeString(self.query.encode('utf-8') if sys.version_info[0] == 2 else self.query)
588
+ oprot.writeFieldEnd()
589
+ if self.startPartition is not None:
590
+ oprot.writeFieldBegin('startPartition', TType.STRING, 3)
591
+ oprot.writeString(self.startPartition.encode('utf-8') if sys.version_info[0] == 2 else self.startPartition)
592
+ oprot.writeFieldEnd()
593
+ if self.setups is not None:
594
+ oprot.writeFieldBegin('setups', TType.LIST, 4)
595
+ oprot.writeListBegin(TType.STRING, len(self.setups))
596
+ for iter42 in self.setups:
597
+ oprot.writeString(iter42.encode('utf-8') if sys.version_info[0] == 2 else iter42)
598
+ oprot.writeListEnd()
599
+ oprot.writeFieldEnd()
600
+ if self.engineType is not None:
601
+ oprot.writeFieldBegin('engineType', TType.I32, 5)
602
+ oprot.writeI32(self.engineType)
603
+ oprot.writeFieldEnd()
604
+ if self.tableDependencies is not None:
605
+ oprot.writeFieldBegin('tableDependencies', TType.LIST, 6)
606
+ oprot.writeListBegin(TType.STRUCT, len(self.tableDependencies))
607
+ for iter43 in self.tableDependencies:
608
+ iter43.write(oprot)
609
+ oprot.writeListEnd()
610
+ oprot.writeFieldEnd()
611
+ if self.recomputeDays is not None:
612
+ oprot.writeFieldBegin('recomputeDays', TType.I32, 20)
613
+ oprot.writeI32(self.recomputeDays)
614
+ oprot.writeFieldEnd()
615
+ oprot.writeFieldStop()
616
+ oprot.writeStructEnd()
617
+
618
+ def validate(self):
619
+ return
620
+
621
+ def __repr__(self):
622
+ L = ['%s=%r' % (key, value)
623
+ for key, value in self.__dict__.items()]
624
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
625
+
626
+ def __eq__(self, other):
627
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
628
+
629
+ def __ne__(self, other):
630
+ return not (self == other)
631
+
632
+
633
+ class EventSource(object):
634
+ """
635
+ Attributes:
636
+ - table: Table currently needs to be a 'ds' (date string - yyyy-MM-dd) partitioned hive table. Table names can contain subpartition specs, example db.table/system=mobile/currency=USD
637
+
638
+ - topic: Topic is a kafka table. The table contains all the events historically came through this topic.
639
+
640
+ - query: The logic used to scan both the table and the topic. Contains row level transformations and filtering expressed as Spark SQL statements.
641
+
642
+ - isCumulative: If each new hive partition contains not just the current day's events but the entire set of events since the begininng. The key property is that the events are not mutated across partitions.
643
+
644
+
645
+ """
646
+ thrift_spec = None
647
+
648
+
649
+ def __init__(self, table = None, topic = None, query = None, isCumulative = None,):
650
+ self.table = table
651
+ self.topic = topic
652
+ self.query = query
653
+ self.isCumulative = isCumulative
654
+
655
+ def read(self, iprot):
656
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
657
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
658
+ return
659
+ iprot.readStructBegin()
660
+ while True:
661
+ (fname, ftype, fid) = iprot.readFieldBegin()
662
+ if ftype == TType.STOP:
663
+ break
664
+ if fid == 1:
665
+ if ftype == TType.STRING:
666
+ self.table = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
667
+ else:
668
+ iprot.skip(ftype)
669
+ elif fid == 2:
670
+ if ftype == TType.STRING:
671
+ self.topic = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
672
+ else:
673
+ iprot.skip(ftype)
674
+ elif fid == 3:
675
+ if ftype == TType.STRUCT:
676
+ self.query = Query()
677
+ self.query.read(iprot)
678
+ else:
679
+ iprot.skip(ftype)
680
+ elif fid == 4:
681
+ if ftype == TType.BOOL:
682
+ self.isCumulative = iprot.readBool()
683
+ else:
684
+ iprot.skip(ftype)
685
+ else:
686
+ iprot.skip(ftype)
687
+ iprot.readFieldEnd()
688
+ iprot.readStructEnd()
689
+
690
+ def write(self, oprot):
691
+ self.validate()
692
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
693
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
694
+ return
695
+ oprot.writeStructBegin('EventSource')
696
+ if self.table is not None:
697
+ oprot.writeFieldBegin('table', TType.STRING, 1)
698
+ oprot.writeString(self.table.encode('utf-8') if sys.version_info[0] == 2 else self.table)
699
+ oprot.writeFieldEnd()
700
+ if self.topic is not None:
701
+ oprot.writeFieldBegin('topic', TType.STRING, 2)
702
+ oprot.writeString(self.topic.encode('utf-8') if sys.version_info[0] == 2 else self.topic)
703
+ oprot.writeFieldEnd()
704
+ if self.query is not None:
705
+ oprot.writeFieldBegin('query', TType.STRUCT, 3)
706
+ self.query.write(oprot)
707
+ oprot.writeFieldEnd()
708
+ if self.isCumulative is not None:
709
+ oprot.writeFieldBegin('isCumulative', TType.BOOL, 4)
710
+ oprot.writeBool(self.isCumulative)
711
+ oprot.writeFieldEnd()
712
+ oprot.writeFieldStop()
713
+ oprot.writeStructEnd()
714
+
715
+ def validate(self):
716
+ return
717
+
718
+ def __repr__(self):
719
+ L = ['%s=%r' % (key, value)
720
+ for key, value in self.__dict__.items()]
721
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
722
+
723
+ def __eq__(self, other):
724
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
725
+
726
+ def __ne__(self, other):
727
+ return not (self == other)
728
+
729
+
730
+ class EntitySource(object):
731
+ """
732
+ Entity Sources represent data that gets mutated over-time - at row-level. This is a group of three data elements.
733
+ snapshotTable, mutationTable and mutationTopic. mutationTable and mutationTopic are only necessary if we are trying
734
+ to create realtime or point-in-time aggregations over these sources. Entity sources usually map 1:1 with a database
735
+ tables in your OLTP store that typically serves live application traffic. When mutation data is absent they map 1:1
736
+ to `dim` tables in star schema.
737
+
738
+ Attributes:
739
+ - snapshotTable: Snapshot table currently needs to be a 'ds' (date string - yyyy-MM-dd) partitioned hive table.
740
+ - mutationTable: Topic is a kafka table. The table contains all the events that historically came through this topic.
741
+ - mutationTopic: The logic used to scan both the table and the topic. Contains row level transformations and filtering expressed as Spark SQL statements.
742
+ - query: If each new hive partition contains not just the current day's events but the entire set of events since the begininng. The key property is that the events are not mutated across partitions.
743
+
744
+ """
745
+ thrift_spec = None
746
+
747
+
748
+ def __init__(self, snapshotTable = None, mutationTable = None, mutationTopic = None, query = None,):
749
+ self.snapshotTable = snapshotTable
750
+ self.mutationTable = mutationTable
751
+ self.mutationTopic = mutationTopic
752
+ self.query = query
753
+
754
+ def read(self, iprot):
755
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
756
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
757
+ return
758
+ iprot.readStructBegin()
759
+ while True:
760
+ (fname, ftype, fid) = iprot.readFieldBegin()
761
+ if ftype == TType.STOP:
762
+ break
763
+ if fid == 1:
764
+ if ftype == TType.STRING:
765
+ self.snapshotTable = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
766
+ else:
767
+ iprot.skip(ftype)
768
+ elif fid == 2:
769
+ if ftype == TType.STRING:
770
+ self.mutationTable = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
771
+ else:
772
+ iprot.skip(ftype)
773
+ elif fid == 3:
774
+ if ftype == TType.STRING:
775
+ self.mutationTopic = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
776
+ else:
777
+ iprot.skip(ftype)
778
+ elif fid == 4:
779
+ if ftype == TType.STRUCT:
780
+ self.query = Query()
781
+ self.query.read(iprot)
782
+ else:
783
+ iprot.skip(ftype)
784
+ else:
785
+ iprot.skip(ftype)
786
+ iprot.readFieldEnd()
787
+ iprot.readStructEnd()
788
+
789
+ def write(self, oprot):
790
+ self.validate()
791
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
792
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
793
+ return
794
+ oprot.writeStructBegin('EntitySource')
795
+ if self.snapshotTable is not None:
796
+ oprot.writeFieldBegin('snapshotTable', TType.STRING, 1)
797
+ oprot.writeString(self.snapshotTable.encode('utf-8') if sys.version_info[0] == 2 else self.snapshotTable)
798
+ oprot.writeFieldEnd()
799
+ if self.mutationTable is not None:
800
+ oprot.writeFieldBegin('mutationTable', TType.STRING, 2)
801
+ oprot.writeString(self.mutationTable.encode('utf-8') if sys.version_info[0] == 2 else self.mutationTable)
802
+ oprot.writeFieldEnd()
803
+ if self.mutationTopic is not None:
804
+ oprot.writeFieldBegin('mutationTopic', TType.STRING, 3)
805
+ oprot.writeString(self.mutationTopic.encode('utf-8') if sys.version_info[0] == 2 else self.mutationTopic)
806
+ oprot.writeFieldEnd()
807
+ if self.query is not None:
808
+ oprot.writeFieldBegin('query', TType.STRUCT, 4)
809
+ self.query.write(oprot)
810
+ oprot.writeFieldEnd()
811
+ oprot.writeFieldStop()
812
+ oprot.writeStructEnd()
813
+
814
+ def validate(self):
815
+ return
816
+
817
+ def __repr__(self):
818
+ L = ['%s=%r' % (key, value)
819
+ for key, value in self.__dict__.items()]
820
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
821
+
822
+ def __eq__(self, other):
823
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
824
+
825
+ def __ne__(self, other):
826
+ return not (self == other)
827
+
828
+
829
+ class ExternalSource(object):
830
+ """
831
+ Attributes:
832
+ - metadata
833
+ - keySchema
834
+ - valueSchema
835
+
836
+ """
837
+ thrift_spec = None
838
+
839
+
840
+ def __init__(self, metadata = None, keySchema = None, valueSchema = None,):
841
+ self.metadata = metadata
842
+ self.keySchema = keySchema
843
+ self.valueSchema = valueSchema
844
+
845
+ def read(self, iprot):
846
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
847
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
848
+ return
849
+ iprot.readStructBegin()
850
+ while True:
851
+ (fname, ftype, fid) = iprot.readFieldBegin()
852
+ if ftype == TType.STOP:
853
+ break
854
+ if fid == 1:
855
+ if ftype == TType.STRUCT:
856
+ self.metadata = MetaData()
857
+ self.metadata.read(iprot)
858
+ else:
859
+ iprot.skip(ftype)
860
+ elif fid == 2:
861
+ if ftype == TType.STRUCT:
862
+ self.keySchema = TDataType()
863
+ self.keySchema.read(iprot)
864
+ else:
865
+ iprot.skip(ftype)
866
+ elif fid == 3:
867
+ if ftype == TType.STRUCT:
868
+ self.valueSchema = TDataType()
869
+ self.valueSchema.read(iprot)
870
+ else:
871
+ iprot.skip(ftype)
872
+ else:
873
+ iprot.skip(ftype)
874
+ iprot.readFieldEnd()
875
+ iprot.readStructEnd()
876
+
877
+ def write(self, oprot):
878
+ self.validate()
879
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
880
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
881
+ return
882
+ oprot.writeStructBegin('ExternalSource')
883
+ if self.metadata is not None:
884
+ oprot.writeFieldBegin('metadata', TType.STRUCT, 1)
885
+ self.metadata.write(oprot)
886
+ oprot.writeFieldEnd()
887
+ if self.keySchema is not None:
888
+ oprot.writeFieldBegin('keySchema', TType.STRUCT, 2)
889
+ self.keySchema.write(oprot)
890
+ oprot.writeFieldEnd()
891
+ if self.valueSchema is not None:
892
+ oprot.writeFieldBegin('valueSchema', TType.STRUCT, 3)
893
+ self.valueSchema.write(oprot)
894
+ oprot.writeFieldEnd()
895
+ oprot.writeFieldStop()
896
+ oprot.writeStructEnd()
897
+
898
+ def validate(self):
899
+ return
900
+
901
+ def __repr__(self):
902
+ L = ['%s=%r' % (key, value)
903
+ for key, value in self.__dict__.items()]
904
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
905
+
906
+ def __eq__(self, other):
907
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
908
+
909
+ def __ne__(self, other):
910
+ return not (self == other)
911
+
912
+
913
+ class JoinSource(object):
914
+ """
915
+ Output of a Join can be used as input to downstream computations like GroupBy or a Join.
916
+ Below is a short description of each of the cases we handle.
917
+ Case #1: a join's source is another join [TODO]
918
+ - while serving, we expect the keys for the upstream join to be passed in the request.
919
+ we will query upstream first, and use the result to query downstream
920
+ - while backfill, we will backfill the upstream first, and use the table as the left of the subsequent join
921
+ - this is currently a "to do" because users can achieve this by themselves unlike case 2:
922
+ Case #2: a join is the source of another GroupBy
923
+ - We will support arbitrarily long transformation chains with this.
924
+ - for batch (Accuracy.SNAPSHOT), we simply backfill the join first and compute groupBy as usual
925
+ - will substitute the joinSource with the resulting table and continue computation
926
+ - we will add a "resolve source" step prior to backfills that will compute the parent join and update the source
927
+ - for realtime (Accuracy.TEMPORAL), we need to do "stream enrichment"
928
+ - we will simply issue "fetchJoin" and create an enriched source. Note the join left should be of type "events".
929
+
930
+
931
+ Attributes:
932
+ - join
933
+ - query
934
+
935
+ """
936
+ thrift_spec = None
937
+
938
+
939
+ def __init__(self, join = None, query = None,):
940
+ self.join = join
941
+ self.query = query
942
+
943
+ def read(self, iprot):
944
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
945
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
946
+ return
947
+ iprot.readStructBegin()
948
+ while True:
949
+ (fname, ftype, fid) = iprot.readFieldBegin()
950
+ if ftype == TType.STOP:
951
+ break
952
+ if fid == 1:
953
+ if ftype == TType.STRUCT:
954
+ self.join = Join()
955
+ self.join.read(iprot)
956
+ else:
957
+ iprot.skip(ftype)
958
+ elif fid == 2:
959
+ if ftype == TType.STRUCT:
960
+ self.query = Query()
961
+ self.query.read(iprot)
962
+ else:
963
+ iprot.skip(ftype)
964
+ else:
965
+ iprot.skip(ftype)
966
+ iprot.readFieldEnd()
967
+ iprot.readStructEnd()
968
+
969
+ def write(self, oprot):
970
+ self.validate()
971
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
972
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
973
+ return
974
+ oprot.writeStructBegin('JoinSource')
975
+ if self.join is not None:
976
+ oprot.writeFieldBegin('join', TType.STRUCT, 1)
977
+ self.join.write(oprot)
978
+ oprot.writeFieldEnd()
979
+ if self.query is not None:
980
+ oprot.writeFieldBegin('query', TType.STRUCT, 2)
981
+ self.query.write(oprot)
982
+ oprot.writeFieldEnd()
983
+ oprot.writeFieldStop()
984
+ oprot.writeStructEnd()
985
+
986
+ def validate(self):
987
+ return
988
+
989
+ def __repr__(self):
990
+ L = ['%s=%r' % (key, value)
991
+ for key, value in self.__dict__.items()]
992
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
993
+
994
+ def __eq__(self, other):
995
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
996
+
997
+ def __ne__(self, other):
998
+ return not (self == other)
999
+
1000
+
1001
+ class Source(object):
1002
+ """
1003
+ Attributes:
1004
+ - events
1005
+ - entities
1006
+ - joinSource
1007
+
1008
+ """
1009
+ thrift_spec = None
1010
+
1011
+
1012
+ def __init__(self, events = None, entities = None, joinSource = None,):
1013
+ self.events = events
1014
+ self.entities = entities
1015
+ self.joinSource = joinSource
1016
+
1017
+ def read(self, iprot):
1018
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
1019
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
1020
+ return
1021
+ iprot.readStructBegin()
1022
+ while True:
1023
+ (fname, ftype, fid) = iprot.readFieldBegin()
1024
+ if ftype == TType.STOP:
1025
+ break
1026
+ if fid == 1:
1027
+ if ftype == TType.STRUCT:
1028
+ self.events = EventSource()
1029
+ self.events.read(iprot)
1030
+ else:
1031
+ iprot.skip(ftype)
1032
+ elif fid == 2:
1033
+ if ftype == TType.STRUCT:
1034
+ self.entities = EntitySource()
1035
+ self.entities.read(iprot)
1036
+ else:
1037
+ iprot.skip(ftype)
1038
+ elif fid == 3:
1039
+ if ftype == TType.STRUCT:
1040
+ self.joinSource = JoinSource()
1041
+ self.joinSource.read(iprot)
1042
+ else:
1043
+ iprot.skip(ftype)
1044
+ else:
1045
+ iprot.skip(ftype)
1046
+ iprot.readFieldEnd()
1047
+ iprot.readStructEnd()
1048
+
1049
+ def write(self, oprot):
1050
+ self.validate()
1051
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
1052
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
1053
+ return
1054
+ oprot.writeStructBegin('Source')
1055
+ if self.events is not None:
1056
+ oprot.writeFieldBegin('events', TType.STRUCT, 1)
1057
+ self.events.write(oprot)
1058
+ oprot.writeFieldEnd()
1059
+ if self.entities is not None:
1060
+ oprot.writeFieldBegin('entities', TType.STRUCT, 2)
1061
+ self.entities.write(oprot)
1062
+ oprot.writeFieldEnd()
1063
+ if self.joinSource is not None:
1064
+ oprot.writeFieldBegin('joinSource', TType.STRUCT, 3)
1065
+ self.joinSource.write(oprot)
1066
+ oprot.writeFieldEnd()
1067
+ oprot.writeFieldStop()
1068
+ oprot.writeStructEnd()
1069
+
1070
+ def validate(self):
1071
+ return
1072
+
1073
+ def __repr__(self):
1074
+ L = ['%s=%r' % (key, value)
1075
+ for key, value in self.__dict__.items()]
1076
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
1077
+
1078
+ def __eq__(self, other):
1079
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
1080
+
1081
+ def __ne__(self, other):
1082
+ return not (self == other)
1083
+
1084
+
1085
+ class Aggregation(object):
1086
+ """
1087
+ Chronon provides a powerful aggregations primitive - that takes the familiar aggregation operation, via groupBy in
1088
+ SQL and extends it with three things - windowing, bucketing and auto-explode.
1089
+
1090
+ Attributes:
1091
+ - inputColumn: The column as specified in source.query.selects - on which we need to aggregate with.
1092
+
1093
+ - operation: The type of aggregation that needs to be performed on the inputColumn.
1094
+
1095
+ - argMap: Extra arguments that needs to be passed to some of the operations like LAST_K, APPROX_PERCENTILE.
1096
+
1097
+ - windows: For TEMPORAL case windows are sawtooth. Meaning head slides ahead continuously in time, whereas, the tail only hops ahead, at discrete points in time. Hop is determined by the window size automatically. The maximum hop size is 1/12 of window size. You can specify multiple such windows at once.
1098
+ - Window > 12 days -> Hop Size = 1 day
1099
+ - Window > 12 hours -> Hop Size = 1 hr
1100
+ - Window > 1hr -> Hop Size = 5 minutes
1101
+ - buckets: This is an additional layer of aggregation. You can key a group_by by user, and bucket a “item_view” count by “item_category”. This will produce one row per user, with column containing map of “item_category” to “view_count”. You can specify multiple such buckets at once
1102
+
1103
+ """
1104
+ thrift_spec = None
1105
+
1106
+
1107
+ def __init__(self, inputColumn = None, operation = None, argMap = None, windows = None, buckets = None,):
1108
+ self.inputColumn = inputColumn
1109
+ self.operation = operation
1110
+ self.argMap = argMap
1111
+ self.windows = windows
1112
+ self.buckets = buckets
1113
+
1114
+ def read(self, iprot):
1115
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
1116
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
1117
+ return
1118
+ iprot.readStructBegin()
1119
+ while True:
1120
+ (fname, ftype, fid) = iprot.readFieldBegin()
1121
+ if ftype == TType.STOP:
1122
+ break
1123
+ if fid == 1:
1124
+ if ftype == TType.STRING:
1125
+ self.inputColumn = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1126
+ else:
1127
+ iprot.skip(ftype)
1128
+ elif fid == 2:
1129
+ if ftype == TType.I32:
1130
+ self.operation = iprot.readI32()
1131
+ else:
1132
+ iprot.skip(ftype)
1133
+ elif fid == 3:
1134
+ if ftype == TType.MAP:
1135
+ self.argMap = {}
1136
+ (_ktype45, _vtype46, _size44) = iprot.readMapBegin()
1137
+ for _i48 in range(_size44):
1138
+ _key49 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1139
+ _val50 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1140
+ self.argMap[_key49] = _val50
1141
+ iprot.readMapEnd()
1142
+ else:
1143
+ iprot.skip(ftype)
1144
+ elif fid == 4:
1145
+ if ftype == TType.LIST:
1146
+ self.windows = []
1147
+ (_etype54, _size51) = iprot.readListBegin()
1148
+ for _i55 in range(_size51):
1149
+ _elem56 = gen_thrift.common.ttypes.Window()
1150
+ _elem56.read(iprot)
1151
+ self.windows.append(_elem56)
1152
+ iprot.readListEnd()
1153
+ else:
1154
+ iprot.skip(ftype)
1155
+ elif fid == 5:
1156
+ if ftype == TType.LIST:
1157
+ self.buckets = []
1158
+ (_etype60, _size57) = iprot.readListBegin()
1159
+ for _i61 in range(_size57):
1160
+ _elem62 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1161
+ self.buckets.append(_elem62)
1162
+ iprot.readListEnd()
1163
+ else:
1164
+ iprot.skip(ftype)
1165
+ else:
1166
+ iprot.skip(ftype)
1167
+ iprot.readFieldEnd()
1168
+ iprot.readStructEnd()
1169
+
1170
+ def write(self, oprot):
1171
+ self.validate()
1172
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
1173
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
1174
+ return
1175
+ oprot.writeStructBegin('Aggregation')
1176
+ if self.inputColumn is not None:
1177
+ oprot.writeFieldBegin('inputColumn', TType.STRING, 1)
1178
+ oprot.writeString(self.inputColumn.encode('utf-8') if sys.version_info[0] == 2 else self.inputColumn)
1179
+ oprot.writeFieldEnd()
1180
+ if self.operation is not None:
1181
+ oprot.writeFieldBegin('operation', TType.I32, 2)
1182
+ oprot.writeI32(self.operation)
1183
+ oprot.writeFieldEnd()
1184
+ if self.argMap is not None:
1185
+ oprot.writeFieldBegin('argMap', TType.MAP, 3)
1186
+ oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.argMap))
1187
+ for kiter63, viter64 in self.argMap.items():
1188
+ oprot.writeString(kiter63.encode('utf-8') if sys.version_info[0] == 2 else kiter63)
1189
+ oprot.writeString(viter64.encode('utf-8') if sys.version_info[0] == 2 else viter64)
1190
+ oprot.writeMapEnd()
1191
+ oprot.writeFieldEnd()
1192
+ if self.windows is not None:
1193
+ oprot.writeFieldBegin('windows', TType.LIST, 4)
1194
+ oprot.writeListBegin(TType.STRUCT, len(self.windows))
1195
+ for iter65 in self.windows:
1196
+ iter65.write(oprot)
1197
+ oprot.writeListEnd()
1198
+ oprot.writeFieldEnd()
1199
+ if self.buckets is not None:
1200
+ oprot.writeFieldBegin('buckets', TType.LIST, 5)
1201
+ oprot.writeListBegin(TType.STRING, len(self.buckets))
1202
+ for iter66 in self.buckets:
1203
+ oprot.writeString(iter66.encode('utf-8') if sys.version_info[0] == 2 else iter66)
1204
+ oprot.writeListEnd()
1205
+ oprot.writeFieldEnd()
1206
+ oprot.writeFieldStop()
1207
+ oprot.writeStructEnd()
1208
+
1209
+ def validate(self):
1210
+ return
1211
+
1212
+ def __repr__(self):
1213
+ L = ['%s=%r' % (key, value)
1214
+ for key, value in self.__dict__.items()]
1215
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
1216
+
1217
+ def __eq__(self, other):
1218
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
1219
+
1220
+ def __ne__(self, other):
1221
+ return not (self == other)
1222
+
1223
+
1224
+ class AggregationPart(object):
1225
+ """
1226
+ Attributes:
1227
+ - inputColumn
1228
+ - operation
1229
+ - argMap
1230
+ - window
1231
+ - bucket
1232
+
1233
+ """
1234
+ thrift_spec = None
1235
+
1236
+
1237
+ def __init__(self, inputColumn = None, operation = None, argMap = None, window = None, bucket = None,):
1238
+ self.inputColumn = inputColumn
1239
+ self.operation = operation
1240
+ self.argMap = argMap
1241
+ self.window = window
1242
+ self.bucket = bucket
1243
+
1244
+ def read(self, iprot):
1245
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
1246
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
1247
+ return
1248
+ iprot.readStructBegin()
1249
+ while True:
1250
+ (fname, ftype, fid) = iprot.readFieldBegin()
1251
+ if ftype == TType.STOP:
1252
+ break
1253
+ if fid == 1:
1254
+ if ftype == TType.STRING:
1255
+ self.inputColumn = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1256
+ else:
1257
+ iprot.skip(ftype)
1258
+ elif fid == 2:
1259
+ if ftype == TType.I32:
1260
+ self.operation = iprot.readI32()
1261
+ else:
1262
+ iprot.skip(ftype)
1263
+ elif fid == 3:
1264
+ if ftype == TType.MAP:
1265
+ self.argMap = {}
1266
+ (_ktype68, _vtype69, _size67) = iprot.readMapBegin()
1267
+ for _i71 in range(_size67):
1268
+ _key72 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1269
+ _val73 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1270
+ self.argMap[_key72] = _val73
1271
+ iprot.readMapEnd()
1272
+ else:
1273
+ iprot.skip(ftype)
1274
+ elif fid == 4:
1275
+ if ftype == TType.STRUCT:
1276
+ self.window = gen_thrift.common.ttypes.Window()
1277
+ self.window.read(iprot)
1278
+ else:
1279
+ iprot.skip(ftype)
1280
+ elif fid == 5:
1281
+ if ftype == TType.STRING:
1282
+ self.bucket = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1283
+ else:
1284
+ iprot.skip(ftype)
1285
+ else:
1286
+ iprot.skip(ftype)
1287
+ iprot.readFieldEnd()
1288
+ iprot.readStructEnd()
1289
+
1290
+ def write(self, oprot):
1291
+ self.validate()
1292
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
1293
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
1294
+ return
1295
+ oprot.writeStructBegin('AggregationPart')
1296
+ if self.inputColumn is not None:
1297
+ oprot.writeFieldBegin('inputColumn', TType.STRING, 1)
1298
+ oprot.writeString(self.inputColumn.encode('utf-8') if sys.version_info[0] == 2 else self.inputColumn)
1299
+ oprot.writeFieldEnd()
1300
+ if self.operation is not None:
1301
+ oprot.writeFieldBegin('operation', TType.I32, 2)
1302
+ oprot.writeI32(self.operation)
1303
+ oprot.writeFieldEnd()
1304
+ if self.argMap is not None:
1305
+ oprot.writeFieldBegin('argMap', TType.MAP, 3)
1306
+ oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.argMap))
1307
+ for kiter74, viter75 in self.argMap.items():
1308
+ oprot.writeString(kiter74.encode('utf-8') if sys.version_info[0] == 2 else kiter74)
1309
+ oprot.writeString(viter75.encode('utf-8') if sys.version_info[0] == 2 else viter75)
1310
+ oprot.writeMapEnd()
1311
+ oprot.writeFieldEnd()
1312
+ if self.window is not None:
1313
+ oprot.writeFieldBegin('window', TType.STRUCT, 4)
1314
+ self.window.write(oprot)
1315
+ oprot.writeFieldEnd()
1316
+ if self.bucket is not None:
1317
+ oprot.writeFieldBegin('bucket', TType.STRING, 5)
1318
+ oprot.writeString(self.bucket.encode('utf-8') if sys.version_info[0] == 2 else self.bucket)
1319
+ oprot.writeFieldEnd()
1320
+ oprot.writeFieldStop()
1321
+ oprot.writeStructEnd()
1322
+
1323
+ def validate(self):
1324
+ return
1325
+
1326
+ def __repr__(self):
1327
+ L = ['%s=%r' % (key, value)
1328
+ for key, value in self.__dict__.items()]
1329
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
1330
+
1331
+ def __eq__(self, other):
1332
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
1333
+
1334
+ def __ne__(self, other):
1335
+ return not (self == other)
1336
+
1337
+
1338
+ class MetaData(object):
1339
+ """
1340
+ contains configs params that don't change the contents of the output.
1341
+
1342
+
1343
+ Attributes:
1344
+ - name
1345
+ - team
1346
+ - version
1347
+ - outputNamespace
1348
+ - additionalOutputPartitionColumns: By default we will just partition the output by the date column - set via "spark.chronon.partition.column"
1349
+ With this we will partition the output with the specified additional columns
1350
+
1351
+ - tableProperties
1352
+ - tags
1353
+ - columnTags
1354
+ - columnHashes
1355
+ - online
1356
+ - production
1357
+ - sourceFile
1358
+ - customJson
1359
+ - consistencyCheck
1360
+ - samplePercent
1361
+ - consistencySamplePercent
1362
+ - driftSpec
1363
+ - executionInfo
1364
+
1365
+ """
1366
+ thrift_spec = None
1367
+
1368
+
1369
+ def __init__(self, name = None, team = None, version = None, outputNamespace = None, additionalOutputPartitionColumns = None, tableProperties = None, tags = None, columnTags = None, columnHashes = None, online = None, production = None, sourceFile = None, customJson = None, consistencyCheck = None, samplePercent = None, consistencySamplePercent = None, driftSpec = None, executionInfo = None,):
1370
+ self.name = name
1371
+ self.team = team
1372
+ self.version = version
1373
+ self.outputNamespace = outputNamespace
1374
+ self.additionalOutputPartitionColumns = additionalOutputPartitionColumns
1375
+ self.tableProperties = tableProperties
1376
+ self.tags = tags
1377
+ self.columnTags = columnTags
1378
+ self.columnHashes = columnHashes
1379
+ self.online = online
1380
+ self.production = production
1381
+ self.sourceFile = sourceFile
1382
+ self.customJson = customJson
1383
+ self.consistencyCheck = consistencyCheck
1384
+ self.samplePercent = samplePercent
1385
+ self.consistencySamplePercent = consistencySamplePercent
1386
+ self.driftSpec = driftSpec
1387
+ self.executionInfo = executionInfo
1388
+
1389
+ def read(self, iprot):
1390
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
1391
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
1392
+ return
1393
+ iprot.readStructBegin()
1394
+ while True:
1395
+ (fname, ftype, fid) = iprot.readFieldBegin()
1396
+ if ftype == TType.STOP:
1397
+ break
1398
+ if fid == 1:
1399
+ if ftype == TType.STRING:
1400
+ self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1401
+ else:
1402
+ iprot.skip(ftype)
1403
+ elif fid == 2:
1404
+ if ftype == TType.STRING:
1405
+ self.team = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1406
+ else:
1407
+ iprot.skip(ftype)
1408
+ elif fid == 3:
1409
+ if ftype == TType.STRING:
1410
+ self.version = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1411
+ else:
1412
+ iprot.skip(ftype)
1413
+ elif fid == 4:
1414
+ if ftype == TType.STRING:
1415
+ self.outputNamespace = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1416
+ else:
1417
+ iprot.skip(ftype)
1418
+ elif fid == 5:
1419
+ if ftype == TType.LIST:
1420
+ self.additionalOutputPartitionColumns = []
1421
+ (_etype79, _size76) = iprot.readListBegin()
1422
+ for _i80 in range(_size76):
1423
+ _elem81 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1424
+ self.additionalOutputPartitionColumns.append(_elem81)
1425
+ iprot.readListEnd()
1426
+ else:
1427
+ iprot.skip(ftype)
1428
+ elif fid == 6:
1429
+ if ftype == TType.MAP:
1430
+ self.tableProperties = {}
1431
+ (_ktype83, _vtype84, _size82) = iprot.readMapBegin()
1432
+ for _i86 in range(_size82):
1433
+ _key87 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1434
+ _val88 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1435
+ self.tableProperties[_key87] = _val88
1436
+ iprot.readMapEnd()
1437
+ else:
1438
+ iprot.skip(ftype)
1439
+ elif fid == 20:
1440
+ if ftype == TType.MAP:
1441
+ self.tags = {}
1442
+ (_ktype90, _vtype91, _size89) = iprot.readMapBegin()
1443
+ for _i93 in range(_size89):
1444
+ _key94 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1445
+ _val95 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1446
+ self.tags[_key94] = _val95
1447
+ iprot.readMapEnd()
1448
+ else:
1449
+ iprot.skip(ftype)
1450
+ elif fid == 21:
1451
+ if ftype == TType.MAP:
1452
+ self.columnTags = {}
1453
+ (_ktype97, _vtype98, _size96) = iprot.readMapBegin()
1454
+ for _i100 in range(_size96):
1455
+ _key101 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1456
+ _val102 = {}
1457
+ (_ktype104, _vtype105, _size103) = iprot.readMapBegin()
1458
+ for _i107 in range(_size103):
1459
+ _key108 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1460
+ _val109 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1461
+ _val102[_key108] = _val109
1462
+ iprot.readMapEnd()
1463
+ self.columnTags[_key101] = _val102
1464
+ iprot.readMapEnd()
1465
+ else:
1466
+ iprot.skip(ftype)
1467
+ elif fid == 22:
1468
+ if ftype == TType.MAP:
1469
+ self.columnHashes = {}
1470
+ (_ktype111, _vtype112, _size110) = iprot.readMapBegin()
1471
+ for _i114 in range(_size110):
1472
+ _key115 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1473
+ _val116 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1474
+ self.columnHashes[_key115] = _val116
1475
+ iprot.readMapEnd()
1476
+ else:
1477
+ iprot.skip(ftype)
1478
+ elif fid == 100:
1479
+ if ftype == TType.BOOL:
1480
+ self.online = iprot.readBool()
1481
+ else:
1482
+ iprot.skip(ftype)
1483
+ elif fid == 101:
1484
+ if ftype == TType.BOOL:
1485
+ self.production = iprot.readBool()
1486
+ else:
1487
+ iprot.skip(ftype)
1488
+ elif fid == 102:
1489
+ if ftype == TType.STRING:
1490
+ self.sourceFile = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1491
+ else:
1492
+ iprot.skip(ftype)
1493
+ elif fid == 103:
1494
+ if ftype == TType.STRING:
1495
+ self.customJson = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1496
+ else:
1497
+ iprot.skip(ftype)
1498
+ elif fid == 200:
1499
+ if ftype == TType.BOOL:
1500
+ self.consistencyCheck = iprot.readBool()
1501
+ else:
1502
+ iprot.skip(ftype)
1503
+ elif fid == 201:
1504
+ if ftype == TType.DOUBLE:
1505
+ self.samplePercent = iprot.readDouble()
1506
+ else:
1507
+ iprot.skip(ftype)
1508
+ elif fid == 202:
1509
+ if ftype == TType.DOUBLE:
1510
+ self.consistencySamplePercent = iprot.readDouble()
1511
+ else:
1512
+ iprot.skip(ftype)
1513
+ elif fid == 203:
1514
+ if ftype == TType.STRUCT:
1515
+ self.driftSpec = gen_thrift.observability.ttypes.DriftSpec()
1516
+ self.driftSpec.read(iprot)
1517
+ else:
1518
+ iprot.skip(ftype)
1519
+ elif fid == 204:
1520
+ if ftype == TType.STRUCT:
1521
+ self.executionInfo = gen_thrift.common.ttypes.ExecutionInfo()
1522
+ self.executionInfo.read(iprot)
1523
+ else:
1524
+ iprot.skip(ftype)
1525
+ else:
1526
+ iprot.skip(ftype)
1527
+ iprot.readFieldEnd()
1528
+ iprot.readStructEnd()
1529
+
1530
+ def write(self, oprot):
1531
+ self.validate()
1532
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
1533
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
1534
+ return
1535
+ oprot.writeStructBegin('MetaData')
1536
+ if self.name is not None:
1537
+ oprot.writeFieldBegin('name', TType.STRING, 1)
1538
+ oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name)
1539
+ oprot.writeFieldEnd()
1540
+ if self.team is not None:
1541
+ oprot.writeFieldBegin('team', TType.STRING, 2)
1542
+ oprot.writeString(self.team.encode('utf-8') if sys.version_info[0] == 2 else self.team)
1543
+ oprot.writeFieldEnd()
1544
+ if self.version is not None:
1545
+ oprot.writeFieldBegin('version', TType.STRING, 3)
1546
+ oprot.writeString(self.version.encode('utf-8') if sys.version_info[0] == 2 else self.version)
1547
+ oprot.writeFieldEnd()
1548
+ if self.outputNamespace is not None:
1549
+ oprot.writeFieldBegin('outputNamespace', TType.STRING, 4)
1550
+ oprot.writeString(self.outputNamespace.encode('utf-8') if sys.version_info[0] == 2 else self.outputNamespace)
1551
+ oprot.writeFieldEnd()
1552
+ if self.additionalOutputPartitionColumns is not None:
1553
+ oprot.writeFieldBegin('additionalOutputPartitionColumns', TType.LIST, 5)
1554
+ oprot.writeListBegin(TType.STRING, len(self.additionalOutputPartitionColumns))
1555
+ for iter117 in self.additionalOutputPartitionColumns:
1556
+ oprot.writeString(iter117.encode('utf-8') if sys.version_info[0] == 2 else iter117)
1557
+ oprot.writeListEnd()
1558
+ oprot.writeFieldEnd()
1559
+ if self.tableProperties is not None:
1560
+ oprot.writeFieldBegin('tableProperties', TType.MAP, 6)
1561
+ oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.tableProperties))
1562
+ for kiter118, viter119 in self.tableProperties.items():
1563
+ oprot.writeString(kiter118.encode('utf-8') if sys.version_info[0] == 2 else kiter118)
1564
+ oprot.writeString(viter119.encode('utf-8') if sys.version_info[0] == 2 else viter119)
1565
+ oprot.writeMapEnd()
1566
+ oprot.writeFieldEnd()
1567
+ if self.tags is not None:
1568
+ oprot.writeFieldBegin('tags', TType.MAP, 20)
1569
+ oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.tags))
1570
+ for kiter120, viter121 in self.tags.items():
1571
+ oprot.writeString(kiter120.encode('utf-8') if sys.version_info[0] == 2 else kiter120)
1572
+ oprot.writeString(viter121.encode('utf-8') if sys.version_info[0] == 2 else viter121)
1573
+ oprot.writeMapEnd()
1574
+ oprot.writeFieldEnd()
1575
+ if self.columnTags is not None:
1576
+ oprot.writeFieldBegin('columnTags', TType.MAP, 21)
1577
+ oprot.writeMapBegin(TType.STRING, TType.MAP, len(self.columnTags))
1578
+ for kiter122, viter123 in self.columnTags.items():
1579
+ oprot.writeString(kiter122.encode('utf-8') if sys.version_info[0] == 2 else kiter122)
1580
+ oprot.writeMapBegin(TType.STRING, TType.STRING, len(viter123))
1581
+ for kiter124, viter125 in viter123.items():
1582
+ oprot.writeString(kiter124.encode('utf-8') if sys.version_info[0] == 2 else kiter124)
1583
+ oprot.writeString(viter125.encode('utf-8') if sys.version_info[0] == 2 else viter125)
1584
+ oprot.writeMapEnd()
1585
+ oprot.writeMapEnd()
1586
+ oprot.writeFieldEnd()
1587
+ if self.columnHashes is not None:
1588
+ oprot.writeFieldBegin('columnHashes', TType.MAP, 22)
1589
+ oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.columnHashes))
1590
+ for kiter126, viter127 in self.columnHashes.items():
1591
+ oprot.writeString(kiter126.encode('utf-8') if sys.version_info[0] == 2 else kiter126)
1592
+ oprot.writeString(viter127.encode('utf-8') if sys.version_info[0] == 2 else viter127)
1593
+ oprot.writeMapEnd()
1594
+ oprot.writeFieldEnd()
1595
+ if self.online is not None:
1596
+ oprot.writeFieldBegin('online', TType.BOOL, 100)
1597
+ oprot.writeBool(self.online)
1598
+ oprot.writeFieldEnd()
1599
+ if self.production is not None:
1600
+ oprot.writeFieldBegin('production', TType.BOOL, 101)
1601
+ oprot.writeBool(self.production)
1602
+ oprot.writeFieldEnd()
1603
+ if self.sourceFile is not None:
1604
+ oprot.writeFieldBegin('sourceFile', TType.STRING, 102)
1605
+ oprot.writeString(self.sourceFile.encode('utf-8') if sys.version_info[0] == 2 else self.sourceFile)
1606
+ oprot.writeFieldEnd()
1607
+ if self.customJson is not None:
1608
+ oprot.writeFieldBegin('customJson', TType.STRING, 103)
1609
+ oprot.writeString(self.customJson.encode('utf-8') if sys.version_info[0] == 2 else self.customJson)
1610
+ oprot.writeFieldEnd()
1611
+ if self.consistencyCheck is not None:
1612
+ oprot.writeFieldBegin('consistencyCheck', TType.BOOL, 200)
1613
+ oprot.writeBool(self.consistencyCheck)
1614
+ oprot.writeFieldEnd()
1615
+ if self.samplePercent is not None:
1616
+ oprot.writeFieldBegin('samplePercent', TType.DOUBLE, 201)
1617
+ oprot.writeDouble(self.samplePercent)
1618
+ oprot.writeFieldEnd()
1619
+ if self.consistencySamplePercent is not None:
1620
+ oprot.writeFieldBegin('consistencySamplePercent', TType.DOUBLE, 202)
1621
+ oprot.writeDouble(self.consistencySamplePercent)
1622
+ oprot.writeFieldEnd()
1623
+ if self.driftSpec is not None:
1624
+ oprot.writeFieldBegin('driftSpec', TType.STRUCT, 203)
1625
+ self.driftSpec.write(oprot)
1626
+ oprot.writeFieldEnd()
1627
+ if self.executionInfo is not None:
1628
+ oprot.writeFieldBegin('executionInfo', TType.STRUCT, 204)
1629
+ self.executionInfo.write(oprot)
1630
+ oprot.writeFieldEnd()
1631
+ oprot.writeFieldStop()
1632
+ oprot.writeStructEnd()
1633
+
1634
+ def validate(self):
1635
+ return
1636
+
1637
+ def __repr__(self):
1638
+ L = ['%s=%r' % (key, value)
1639
+ for key, value in self.__dict__.items()]
1640
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
1641
+
1642
+ def __eq__(self, other):
1643
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
1644
+
1645
+ def __ne__(self, other):
1646
+ return not (self == other)
1647
+
1648
+
1649
+ class Conf(object):
1650
+ """
1651
+ Attributes:
1652
+ - name
1653
+ - hash
1654
+ - contents
1655
+ - confType
1656
+
1657
+ """
1658
+ thrift_spec = None
1659
+
1660
+
1661
+ def __init__(self, name = None, hash = None, contents = None, confType = None,):
1662
+ self.name = name
1663
+ self.hash = hash
1664
+ self.contents = contents
1665
+ self.confType = confType
1666
+
1667
+ def read(self, iprot):
1668
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
1669
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
1670
+ return
1671
+ iprot.readStructBegin()
1672
+ while True:
1673
+ (fname, ftype, fid) = iprot.readFieldBegin()
1674
+ if ftype == TType.STOP:
1675
+ break
1676
+ if fid == 1:
1677
+ if ftype == TType.STRING:
1678
+ self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1679
+ else:
1680
+ iprot.skip(ftype)
1681
+ elif fid == 2:
1682
+ if ftype == TType.STRING:
1683
+ self.hash = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1684
+ else:
1685
+ iprot.skip(ftype)
1686
+ elif fid == 3:
1687
+ if ftype == TType.STRING:
1688
+ self.contents = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1689
+ else:
1690
+ iprot.skip(ftype)
1691
+ elif fid == 4:
1692
+ if ftype == TType.I32:
1693
+ self.confType = iprot.readI32()
1694
+ else:
1695
+ iprot.skip(ftype)
1696
+ else:
1697
+ iprot.skip(ftype)
1698
+ iprot.readFieldEnd()
1699
+ iprot.readStructEnd()
1700
+
1701
+ def write(self, oprot):
1702
+ self.validate()
1703
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
1704
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
1705
+ return
1706
+ oprot.writeStructBegin('Conf')
1707
+ if self.name is not None:
1708
+ oprot.writeFieldBegin('name', TType.STRING, 1)
1709
+ oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name)
1710
+ oprot.writeFieldEnd()
1711
+ if self.hash is not None:
1712
+ oprot.writeFieldBegin('hash', TType.STRING, 2)
1713
+ oprot.writeString(self.hash.encode('utf-8') if sys.version_info[0] == 2 else self.hash)
1714
+ oprot.writeFieldEnd()
1715
+ if self.contents is not None:
1716
+ oprot.writeFieldBegin('contents', TType.STRING, 3)
1717
+ oprot.writeString(self.contents.encode('utf-8') if sys.version_info[0] == 2 else self.contents)
1718
+ oprot.writeFieldEnd()
1719
+ if self.confType is not None:
1720
+ oprot.writeFieldBegin('confType', TType.I32, 4)
1721
+ oprot.writeI32(self.confType)
1722
+ oprot.writeFieldEnd()
1723
+ oprot.writeFieldStop()
1724
+ oprot.writeStructEnd()
1725
+
1726
+ def validate(self):
1727
+ return
1728
+
1729
+ def __repr__(self):
1730
+ L = ['%s=%r' % (key, value)
1731
+ for key, value in self.__dict__.items()]
1732
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
1733
+
1734
+ def __eq__(self, other):
1735
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
1736
+
1737
+ def __ne__(self, other):
1738
+ return not (self == other)
1739
+
1740
+
1741
+ class GroupBy(object):
1742
+ """
1743
+ Attributes:
1744
+ - metaData
1745
+ - sources
1746
+ - keyColumns
1747
+ - aggregations
1748
+ - accuracy
1749
+ - backfillStartDate
1750
+ - derivations
1751
+
1752
+ """
1753
+ thrift_spec = None
1754
+
1755
+
1756
+ def __init__(self, metaData = None, sources = None, keyColumns = None, aggregations = None, accuracy = None, backfillStartDate = None, derivations = None,):
1757
+ self.metaData = metaData
1758
+ self.sources = sources
1759
+ self.keyColumns = keyColumns
1760
+ self.aggregations = aggregations
1761
+ self.accuracy = accuracy
1762
+ self.backfillStartDate = backfillStartDate
1763
+ self.derivations = derivations
1764
+
1765
+ def read(self, iprot):
1766
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
1767
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
1768
+ return
1769
+ iprot.readStructBegin()
1770
+ while True:
1771
+ (fname, ftype, fid) = iprot.readFieldBegin()
1772
+ if ftype == TType.STOP:
1773
+ break
1774
+ if fid == 1:
1775
+ if ftype == TType.STRUCT:
1776
+ self.metaData = MetaData()
1777
+ self.metaData.read(iprot)
1778
+ else:
1779
+ iprot.skip(ftype)
1780
+ elif fid == 2:
1781
+ if ftype == TType.LIST:
1782
+ self.sources = []
1783
+ (_etype131, _size128) = iprot.readListBegin()
1784
+ for _i132 in range(_size128):
1785
+ _elem133 = Source()
1786
+ _elem133.read(iprot)
1787
+ self.sources.append(_elem133)
1788
+ iprot.readListEnd()
1789
+ else:
1790
+ iprot.skip(ftype)
1791
+ elif fid == 3:
1792
+ if ftype == TType.LIST:
1793
+ self.keyColumns = []
1794
+ (_etype137, _size134) = iprot.readListBegin()
1795
+ for _i138 in range(_size134):
1796
+ _elem139 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1797
+ self.keyColumns.append(_elem139)
1798
+ iprot.readListEnd()
1799
+ else:
1800
+ iprot.skip(ftype)
1801
+ elif fid == 4:
1802
+ if ftype == TType.LIST:
1803
+ self.aggregations = []
1804
+ (_etype143, _size140) = iprot.readListBegin()
1805
+ for _i144 in range(_size140):
1806
+ _elem145 = Aggregation()
1807
+ _elem145.read(iprot)
1808
+ self.aggregations.append(_elem145)
1809
+ iprot.readListEnd()
1810
+ else:
1811
+ iprot.skip(ftype)
1812
+ elif fid == 5:
1813
+ if ftype == TType.I32:
1814
+ self.accuracy = iprot.readI32()
1815
+ else:
1816
+ iprot.skip(ftype)
1817
+ elif fid == 6:
1818
+ if ftype == TType.STRING:
1819
+ self.backfillStartDate = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1820
+ else:
1821
+ iprot.skip(ftype)
1822
+ elif fid == 7:
1823
+ if ftype == TType.LIST:
1824
+ self.derivations = []
1825
+ (_etype149, _size146) = iprot.readListBegin()
1826
+ for _i150 in range(_size146):
1827
+ _elem151 = Derivation()
1828
+ _elem151.read(iprot)
1829
+ self.derivations.append(_elem151)
1830
+ iprot.readListEnd()
1831
+ else:
1832
+ iprot.skip(ftype)
1833
+ else:
1834
+ iprot.skip(ftype)
1835
+ iprot.readFieldEnd()
1836
+ iprot.readStructEnd()
1837
+
1838
+ def write(self, oprot):
1839
+ self.validate()
1840
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
1841
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
1842
+ return
1843
+ oprot.writeStructBegin('GroupBy')
1844
+ if self.metaData is not None:
1845
+ oprot.writeFieldBegin('metaData', TType.STRUCT, 1)
1846
+ self.metaData.write(oprot)
1847
+ oprot.writeFieldEnd()
1848
+ if self.sources is not None:
1849
+ oprot.writeFieldBegin('sources', TType.LIST, 2)
1850
+ oprot.writeListBegin(TType.STRUCT, len(self.sources))
1851
+ for iter152 in self.sources:
1852
+ iter152.write(oprot)
1853
+ oprot.writeListEnd()
1854
+ oprot.writeFieldEnd()
1855
+ if self.keyColumns is not None:
1856
+ oprot.writeFieldBegin('keyColumns', TType.LIST, 3)
1857
+ oprot.writeListBegin(TType.STRING, len(self.keyColumns))
1858
+ for iter153 in self.keyColumns:
1859
+ oprot.writeString(iter153.encode('utf-8') if sys.version_info[0] == 2 else iter153)
1860
+ oprot.writeListEnd()
1861
+ oprot.writeFieldEnd()
1862
+ if self.aggregations is not None:
1863
+ oprot.writeFieldBegin('aggregations', TType.LIST, 4)
1864
+ oprot.writeListBegin(TType.STRUCT, len(self.aggregations))
1865
+ for iter154 in self.aggregations:
1866
+ iter154.write(oprot)
1867
+ oprot.writeListEnd()
1868
+ oprot.writeFieldEnd()
1869
+ if self.accuracy is not None:
1870
+ oprot.writeFieldBegin('accuracy', TType.I32, 5)
1871
+ oprot.writeI32(self.accuracy)
1872
+ oprot.writeFieldEnd()
1873
+ if self.backfillStartDate is not None:
1874
+ oprot.writeFieldBegin('backfillStartDate', TType.STRING, 6)
1875
+ oprot.writeString(self.backfillStartDate.encode('utf-8') if sys.version_info[0] == 2 else self.backfillStartDate)
1876
+ oprot.writeFieldEnd()
1877
+ if self.derivations is not None:
1878
+ oprot.writeFieldBegin('derivations', TType.LIST, 7)
1879
+ oprot.writeListBegin(TType.STRUCT, len(self.derivations))
1880
+ for iter155 in self.derivations:
1881
+ iter155.write(oprot)
1882
+ oprot.writeListEnd()
1883
+ oprot.writeFieldEnd()
1884
+ oprot.writeFieldStop()
1885
+ oprot.writeStructEnd()
1886
+
1887
+ def validate(self):
1888
+ return
1889
+
1890
+ def __repr__(self):
1891
+ L = ['%s=%r' % (key, value)
1892
+ for key, value in self.__dict__.items()]
1893
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
1894
+
1895
+ def __eq__(self, other):
1896
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
1897
+
1898
+ def __ne__(self, other):
1899
+ return not (self == other)
1900
+
1901
+
1902
+ class JoinPart(object):
1903
+ """
1904
+ Attributes:
1905
+ - groupBy
1906
+ - keyMapping
1907
+ - prefix
1908
+ - useLongNames
1909
+
1910
+ """
1911
+ thrift_spec = None
1912
+
1913
+
1914
+ def __init__(self, groupBy = None, keyMapping = None, prefix = None, useLongNames = None,):
1915
+ self.groupBy = groupBy
1916
+ self.keyMapping = keyMapping
1917
+ self.prefix = prefix
1918
+ self.useLongNames = useLongNames
1919
+
1920
+ def read(self, iprot):
1921
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
1922
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
1923
+ return
1924
+ iprot.readStructBegin()
1925
+ while True:
1926
+ (fname, ftype, fid) = iprot.readFieldBegin()
1927
+ if ftype == TType.STOP:
1928
+ break
1929
+ if fid == 1:
1930
+ if ftype == TType.STRUCT:
1931
+ self.groupBy = GroupBy()
1932
+ self.groupBy.read(iprot)
1933
+ else:
1934
+ iprot.skip(ftype)
1935
+ elif fid == 2:
1936
+ if ftype == TType.MAP:
1937
+ self.keyMapping = {}
1938
+ (_ktype157, _vtype158, _size156) = iprot.readMapBegin()
1939
+ for _i160 in range(_size156):
1940
+ _key161 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1941
+ _val162 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1942
+ self.keyMapping[_key161] = _val162
1943
+ iprot.readMapEnd()
1944
+ else:
1945
+ iprot.skip(ftype)
1946
+ elif fid == 3:
1947
+ if ftype == TType.STRING:
1948
+ self.prefix = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
1949
+ else:
1950
+ iprot.skip(ftype)
1951
+ elif fid == 10:
1952
+ if ftype == TType.BOOL:
1953
+ self.useLongNames = iprot.readBool()
1954
+ else:
1955
+ iprot.skip(ftype)
1956
+ else:
1957
+ iprot.skip(ftype)
1958
+ iprot.readFieldEnd()
1959
+ iprot.readStructEnd()
1960
+
1961
+ def write(self, oprot):
1962
+ self.validate()
1963
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
1964
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
1965
+ return
1966
+ oprot.writeStructBegin('JoinPart')
1967
+ if self.groupBy is not None:
1968
+ oprot.writeFieldBegin('groupBy', TType.STRUCT, 1)
1969
+ self.groupBy.write(oprot)
1970
+ oprot.writeFieldEnd()
1971
+ if self.keyMapping is not None:
1972
+ oprot.writeFieldBegin('keyMapping', TType.MAP, 2)
1973
+ oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.keyMapping))
1974
+ for kiter163, viter164 in self.keyMapping.items():
1975
+ oprot.writeString(kiter163.encode('utf-8') if sys.version_info[0] == 2 else kiter163)
1976
+ oprot.writeString(viter164.encode('utf-8') if sys.version_info[0] == 2 else viter164)
1977
+ oprot.writeMapEnd()
1978
+ oprot.writeFieldEnd()
1979
+ if self.prefix is not None:
1980
+ oprot.writeFieldBegin('prefix', TType.STRING, 3)
1981
+ oprot.writeString(self.prefix.encode('utf-8') if sys.version_info[0] == 2 else self.prefix)
1982
+ oprot.writeFieldEnd()
1983
+ if self.useLongNames is not None:
1984
+ oprot.writeFieldBegin('useLongNames', TType.BOOL, 10)
1985
+ oprot.writeBool(self.useLongNames)
1986
+ oprot.writeFieldEnd()
1987
+ oprot.writeFieldStop()
1988
+ oprot.writeStructEnd()
1989
+
1990
+ def validate(self):
1991
+ return
1992
+
1993
+ def __repr__(self):
1994
+ L = ['%s=%r' % (key, value)
1995
+ for key, value in self.__dict__.items()]
1996
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
1997
+
1998
+ def __eq__(self, other):
1999
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
2000
+
2001
+ def __ne__(self, other):
2002
+ return not (self == other)
2003
+
2004
+
2005
+ class ExternalPart(object):
2006
+ """
2007
+ Attributes:
2008
+ - source
2009
+ - keyMapping
2010
+ - prefix
2011
+
2012
+ """
2013
+ thrift_spec = None
2014
+
2015
+
2016
+ def __init__(self, source = None, keyMapping = None, prefix = None,):
2017
+ self.source = source
2018
+ self.keyMapping = keyMapping
2019
+ self.prefix = prefix
2020
+
2021
+ def read(self, iprot):
2022
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
2023
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
2024
+ return
2025
+ iprot.readStructBegin()
2026
+ while True:
2027
+ (fname, ftype, fid) = iprot.readFieldBegin()
2028
+ if ftype == TType.STOP:
2029
+ break
2030
+ if fid == 1:
2031
+ if ftype == TType.STRUCT:
2032
+ self.source = ExternalSource()
2033
+ self.source.read(iprot)
2034
+ else:
2035
+ iprot.skip(ftype)
2036
+ elif fid == 2:
2037
+ if ftype == TType.MAP:
2038
+ self.keyMapping = {}
2039
+ (_ktype166, _vtype167, _size165) = iprot.readMapBegin()
2040
+ for _i169 in range(_size165):
2041
+ _key170 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2042
+ _val171 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2043
+ self.keyMapping[_key170] = _val171
2044
+ iprot.readMapEnd()
2045
+ else:
2046
+ iprot.skip(ftype)
2047
+ elif fid == 3:
2048
+ if ftype == TType.STRING:
2049
+ self.prefix = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2050
+ else:
2051
+ iprot.skip(ftype)
2052
+ else:
2053
+ iprot.skip(ftype)
2054
+ iprot.readFieldEnd()
2055
+ iprot.readStructEnd()
2056
+
2057
+ def write(self, oprot):
2058
+ self.validate()
2059
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
2060
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
2061
+ return
2062
+ oprot.writeStructBegin('ExternalPart')
2063
+ if self.source is not None:
2064
+ oprot.writeFieldBegin('source', TType.STRUCT, 1)
2065
+ self.source.write(oprot)
2066
+ oprot.writeFieldEnd()
2067
+ if self.keyMapping is not None:
2068
+ oprot.writeFieldBegin('keyMapping', TType.MAP, 2)
2069
+ oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.keyMapping))
2070
+ for kiter172, viter173 in self.keyMapping.items():
2071
+ oprot.writeString(kiter172.encode('utf-8') if sys.version_info[0] == 2 else kiter172)
2072
+ oprot.writeString(viter173.encode('utf-8') if sys.version_info[0] == 2 else viter173)
2073
+ oprot.writeMapEnd()
2074
+ oprot.writeFieldEnd()
2075
+ if self.prefix is not None:
2076
+ oprot.writeFieldBegin('prefix', TType.STRING, 3)
2077
+ oprot.writeString(self.prefix.encode('utf-8') if sys.version_info[0] == 2 else self.prefix)
2078
+ oprot.writeFieldEnd()
2079
+ oprot.writeFieldStop()
2080
+ oprot.writeStructEnd()
2081
+
2082
+ def validate(self):
2083
+ return
2084
+
2085
+ def __repr__(self):
2086
+ L = ['%s=%r' % (key, value)
2087
+ for key, value in self.__dict__.items()]
2088
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
2089
+
2090
+ def __eq__(self, other):
2091
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
2092
+
2093
+ def __ne__(self, other):
2094
+ return not (self == other)
2095
+
2096
+
2097
+ class Derivation(object):
2098
+ """
2099
+ Attributes:
2100
+ - name
2101
+ - expression
2102
+
2103
+ """
2104
+ thrift_spec = None
2105
+
2106
+
2107
+ def __init__(self, name = None, expression = None,):
2108
+ self.name = name
2109
+ self.expression = expression
2110
+
2111
+ def read(self, iprot):
2112
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
2113
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
2114
+ return
2115
+ iprot.readStructBegin()
2116
+ while True:
2117
+ (fname, ftype, fid) = iprot.readFieldBegin()
2118
+ if ftype == TType.STOP:
2119
+ break
2120
+ if fid == 1:
2121
+ if ftype == TType.STRING:
2122
+ self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2123
+ else:
2124
+ iprot.skip(ftype)
2125
+ elif fid == 2:
2126
+ if ftype == TType.STRING:
2127
+ self.expression = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2128
+ else:
2129
+ iprot.skip(ftype)
2130
+ else:
2131
+ iprot.skip(ftype)
2132
+ iprot.readFieldEnd()
2133
+ iprot.readStructEnd()
2134
+
2135
+ def write(self, oprot):
2136
+ self.validate()
2137
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
2138
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
2139
+ return
2140
+ oprot.writeStructBegin('Derivation')
2141
+ if self.name is not None:
2142
+ oprot.writeFieldBegin('name', TType.STRING, 1)
2143
+ oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name)
2144
+ oprot.writeFieldEnd()
2145
+ if self.expression is not None:
2146
+ oprot.writeFieldBegin('expression', TType.STRING, 2)
2147
+ oprot.writeString(self.expression.encode('utf-8') if sys.version_info[0] == 2 else self.expression)
2148
+ oprot.writeFieldEnd()
2149
+ oprot.writeFieldStop()
2150
+ oprot.writeStructEnd()
2151
+
2152
+ def validate(self):
2153
+ return
2154
+
2155
+ def __repr__(self):
2156
+ L = ['%s=%r' % (key, value)
2157
+ for key, value in self.__dict__.items()]
2158
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
2159
+
2160
+ def __eq__(self, other):
2161
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
2162
+
2163
+ def __ne__(self, other):
2164
+ return not (self == other)
2165
+
2166
+
2167
+ class Join(object):
2168
+ """
2169
+ Attributes:
2170
+ - metaData
2171
+ - left
2172
+ - joinParts
2173
+ - skewKeys
2174
+ - onlineExternalParts
2175
+ - labelParts
2176
+ - bootstrapParts
2177
+ - rowIds
2178
+ - derivations: List of a derived column names to the expression based on joinPart / externalPart columns
2179
+ The expression can be any valid Spark SQL select clause without aggregation functions.
2180
+
2181
+ joinPart column names are automatically constructed according to the below convention
2182
+ `{join_part_prefix}_{group_by_name}_{input_column_name}_{aggregation_operation}_{window}_{by_bucket}`
2183
+ prefix, window and bucket are optional. You can find the type information of columns using the analyzer tool.
2184
+
2185
+ externalPart column names are automatically constructed according to the below convention
2186
+ `ext_{external_source_name}_{value_column}`
2187
+ Types are defined along with the schema by users for external sources.
2188
+
2189
+ Including a column with key "*" and value "*", means that every raw column will be included along with the derived
2190
+ columns.
2191
+
2192
+ - useLongNames
2193
+
2194
+ """
2195
+ thrift_spec = None
2196
+
2197
+
2198
+ def __init__(self, metaData = None, left = None, joinParts = None, skewKeys = None, onlineExternalParts = None, labelParts = None, bootstrapParts = None, rowIds = None, derivations = None, useLongNames = None,):
2199
+ self.metaData = metaData
2200
+ self.left = left
2201
+ self.joinParts = joinParts
2202
+ self.skewKeys = skewKeys
2203
+ self.onlineExternalParts = onlineExternalParts
2204
+ self.labelParts = labelParts
2205
+ self.bootstrapParts = bootstrapParts
2206
+ self.rowIds = rowIds
2207
+ self.derivations = derivations
2208
+ self.useLongNames = useLongNames
2209
+
2210
+ def read(self, iprot):
2211
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
2212
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
2213
+ return
2214
+ iprot.readStructBegin()
2215
+ while True:
2216
+ (fname, ftype, fid) = iprot.readFieldBegin()
2217
+ if ftype == TType.STOP:
2218
+ break
2219
+ if fid == 1:
2220
+ if ftype == TType.STRUCT:
2221
+ self.metaData = MetaData()
2222
+ self.metaData.read(iprot)
2223
+ else:
2224
+ iprot.skip(ftype)
2225
+ elif fid == 2:
2226
+ if ftype == TType.STRUCT:
2227
+ self.left = Source()
2228
+ self.left.read(iprot)
2229
+ else:
2230
+ iprot.skip(ftype)
2231
+ elif fid == 3:
2232
+ if ftype == TType.LIST:
2233
+ self.joinParts = []
2234
+ (_etype177, _size174) = iprot.readListBegin()
2235
+ for _i178 in range(_size174):
2236
+ _elem179 = JoinPart()
2237
+ _elem179.read(iprot)
2238
+ self.joinParts.append(_elem179)
2239
+ iprot.readListEnd()
2240
+ else:
2241
+ iprot.skip(ftype)
2242
+ elif fid == 4:
2243
+ if ftype == TType.MAP:
2244
+ self.skewKeys = {}
2245
+ (_ktype181, _vtype182, _size180) = iprot.readMapBegin()
2246
+ for _i184 in range(_size180):
2247
+ _key185 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2248
+ _val186 = []
2249
+ (_etype190, _size187) = iprot.readListBegin()
2250
+ for _i191 in range(_size187):
2251
+ _elem192 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2252
+ _val186.append(_elem192)
2253
+ iprot.readListEnd()
2254
+ self.skewKeys[_key185] = _val186
2255
+ iprot.readMapEnd()
2256
+ else:
2257
+ iprot.skip(ftype)
2258
+ elif fid == 5:
2259
+ if ftype == TType.LIST:
2260
+ self.onlineExternalParts = []
2261
+ (_etype196, _size193) = iprot.readListBegin()
2262
+ for _i197 in range(_size193):
2263
+ _elem198 = ExternalPart()
2264
+ _elem198.read(iprot)
2265
+ self.onlineExternalParts.append(_elem198)
2266
+ iprot.readListEnd()
2267
+ else:
2268
+ iprot.skip(ftype)
2269
+ elif fid == 6:
2270
+ if ftype == TType.STRUCT:
2271
+ self.labelParts = LabelParts()
2272
+ self.labelParts.read(iprot)
2273
+ else:
2274
+ iprot.skip(ftype)
2275
+ elif fid == 7:
2276
+ if ftype == TType.LIST:
2277
+ self.bootstrapParts = []
2278
+ (_etype202, _size199) = iprot.readListBegin()
2279
+ for _i203 in range(_size199):
2280
+ _elem204 = BootstrapPart()
2281
+ _elem204.read(iprot)
2282
+ self.bootstrapParts.append(_elem204)
2283
+ iprot.readListEnd()
2284
+ else:
2285
+ iprot.skip(ftype)
2286
+ elif fid == 8:
2287
+ if ftype == TType.LIST:
2288
+ self.rowIds = []
2289
+ (_etype208, _size205) = iprot.readListBegin()
2290
+ for _i209 in range(_size205):
2291
+ _elem210 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2292
+ self.rowIds.append(_elem210)
2293
+ iprot.readListEnd()
2294
+ else:
2295
+ iprot.skip(ftype)
2296
+ elif fid == 9:
2297
+ if ftype == TType.LIST:
2298
+ self.derivations = []
2299
+ (_etype214, _size211) = iprot.readListBegin()
2300
+ for _i215 in range(_size211):
2301
+ _elem216 = Derivation()
2302
+ _elem216.read(iprot)
2303
+ self.derivations.append(_elem216)
2304
+ iprot.readListEnd()
2305
+ else:
2306
+ iprot.skip(ftype)
2307
+ elif fid == 50:
2308
+ if ftype == TType.BOOL:
2309
+ self.useLongNames = iprot.readBool()
2310
+ else:
2311
+ iprot.skip(ftype)
2312
+ else:
2313
+ iprot.skip(ftype)
2314
+ iprot.readFieldEnd()
2315
+ iprot.readStructEnd()
2316
+
2317
+ def write(self, oprot):
2318
+ self.validate()
2319
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
2320
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
2321
+ return
2322
+ oprot.writeStructBegin('Join')
2323
+ if self.metaData is not None:
2324
+ oprot.writeFieldBegin('metaData', TType.STRUCT, 1)
2325
+ self.metaData.write(oprot)
2326
+ oprot.writeFieldEnd()
2327
+ if self.left is not None:
2328
+ oprot.writeFieldBegin('left', TType.STRUCT, 2)
2329
+ self.left.write(oprot)
2330
+ oprot.writeFieldEnd()
2331
+ if self.joinParts is not None:
2332
+ oprot.writeFieldBegin('joinParts', TType.LIST, 3)
2333
+ oprot.writeListBegin(TType.STRUCT, len(self.joinParts))
2334
+ for iter217 in self.joinParts:
2335
+ iter217.write(oprot)
2336
+ oprot.writeListEnd()
2337
+ oprot.writeFieldEnd()
2338
+ if self.skewKeys is not None:
2339
+ oprot.writeFieldBegin('skewKeys', TType.MAP, 4)
2340
+ oprot.writeMapBegin(TType.STRING, TType.LIST, len(self.skewKeys))
2341
+ for kiter218, viter219 in self.skewKeys.items():
2342
+ oprot.writeString(kiter218.encode('utf-8') if sys.version_info[0] == 2 else kiter218)
2343
+ oprot.writeListBegin(TType.STRING, len(viter219))
2344
+ for iter220 in viter219:
2345
+ oprot.writeString(iter220.encode('utf-8') if sys.version_info[0] == 2 else iter220)
2346
+ oprot.writeListEnd()
2347
+ oprot.writeMapEnd()
2348
+ oprot.writeFieldEnd()
2349
+ if self.onlineExternalParts is not None:
2350
+ oprot.writeFieldBegin('onlineExternalParts', TType.LIST, 5)
2351
+ oprot.writeListBegin(TType.STRUCT, len(self.onlineExternalParts))
2352
+ for iter221 in self.onlineExternalParts:
2353
+ iter221.write(oprot)
2354
+ oprot.writeListEnd()
2355
+ oprot.writeFieldEnd()
2356
+ if self.labelParts is not None:
2357
+ oprot.writeFieldBegin('labelParts', TType.STRUCT, 6)
2358
+ self.labelParts.write(oprot)
2359
+ oprot.writeFieldEnd()
2360
+ if self.bootstrapParts is not None:
2361
+ oprot.writeFieldBegin('bootstrapParts', TType.LIST, 7)
2362
+ oprot.writeListBegin(TType.STRUCT, len(self.bootstrapParts))
2363
+ for iter222 in self.bootstrapParts:
2364
+ iter222.write(oprot)
2365
+ oprot.writeListEnd()
2366
+ oprot.writeFieldEnd()
2367
+ if self.rowIds is not None:
2368
+ oprot.writeFieldBegin('rowIds', TType.LIST, 8)
2369
+ oprot.writeListBegin(TType.STRING, len(self.rowIds))
2370
+ for iter223 in self.rowIds:
2371
+ oprot.writeString(iter223.encode('utf-8') if sys.version_info[0] == 2 else iter223)
2372
+ oprot.writeListEnd()
2373
+ oprot.writeFieldEnd()
2374
+ if self.derivations is not None:
2375
+ oprot.writeFieldBegin('derivations', TType.LIST, 9)
2376
+ oprot.writeListBegin(TType.STRUCT, len(self.derivations))
2377
+ for iter224 in self.derivations:
2378
+ iter224.write(oprot)
2379
+ oprot.writeListEnd()
2380
+ oprot.writeFieldEnd()
2381
+ if self.useLongNames is not None:
2382
+ oprot.writeFieldBegin('useLongNames', TType.BOOL, 50)
2383
+ oprot.writeBool(self.useLongNames)
2384
+ oprot.writeFieldEnd()
2385
+ oprot.writeFieldStop()
2386
+ oprot.writeStructEnd()
2387
+
2388
+ def validate(self):
2389
+ return
2390
+
2391
+ def __repr__(self):
2392
+ L = ['%s=%r' % (key, value)
2393
+ for key, value in self.__dict__.items()]
2394
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
2395
+
2396
+ def __eq__(self, other):
2397
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
2398
+
2399
+ def __ne__(self, other):
2400
+ return not (self == other)
2401
+
2402
+
2403
+ class BootstrapPart(object):
2404
+ """
2405
+ Attributes:
2406
+ - metaData
2407
+ - table
2408
+ - query
2409
+ - keyColumns
2410
+
2411
+ """
2412
+ thrift_spec = None
2413
+
2414
+
2415
+ def __init__(self, metaData = None, table = None, query = None, keyColumns = None,):
2416
+ self.metaData = metaData
2417
+ self.table = table
2418
+ self.query = query
2419
+ self.keyColumns = keyColumns
2420
+
2421
+ def read(self, iprot):
2422
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
2423
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
2424
+ return
2425
+ iprot.readStructBegin()
2426
+ while True:
2427
+ (fname, ftype, fid) = iprot.readFieldBegin()
2428
+ if ftype == TType.STOP:
2429
+ break
2430
+ if fid == 1:
2431
+ if ftype == TType.STRUCT:
2432
+ self.metaData = MetaData()
2433
+ self.metaData.read(iprot)
2434
+ else:
2435
+ iprot.skip(ftype)
2436
+ elif fid == 2:
2437
+ if ftype == TType.STRING:
2438
+ self.table = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2439
+ else:
2440
+ iprot.skip(ftype)
2441
+ elif fid == 3:
2442
+ if ftype == TType.STRUCT:
2443
+ self.query = Query()
2444
+ self.query.read(iprot)
2445
+ else:
2446
+ iprot.skip(ftype)
2447
+ elif fid == 4:
2448
+ if ftype == TType.LIST:
2449
+ self.keyColumns = []
2450
+ (_etype228, _size225) = iprot.readListBegin()
2451
+ for _i229 in range(_size225):
2452
+ _elem230 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2453
+ self.keyColumns.append(_elem230)
2454
+ iprot.readListEnd()
2455
+ else:
2456
+ iprot.skip(ftype)
2457
+ else:
2458
+ iprot.skip(ftype)
2459
+ iprot.readFieldEnd()
2460
+ iprot.readStructEnd()
2461
+
2462
+ def write(self, oprot):
2463
+ self.validate()
2464
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
2465
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
2466
+ return
2467
+ oprot.writeStructBegin('BootstrapPart')
2468
+ if self.metaData is not None:
2469
+ oprot.writeFieldBegin('metaData', TType.STRUCT, 1)
2470
+ self.metaData.write(oprot)
2471
+ oprot.writeFieldEnd()
2472
+ if self.table is not None:
2473
+ oprot.writeFieldBegin('table', TType.STRING, 2)
2474
+ oprot.writeString(self.table.encode('utf-8') if sys.version_info[0] == 2 else self.table)
2475
+ oprot.writeFieldEnd()
2476
+ if self.query is not None:
2477
+ oprot.writeFieldBegin('query', TType.STRUCT, 3)
2478
+ self.query.write(oprot)
2479
+ oprot.writeFieldEnd()
2480
+ if self.keyColumns is not None:
2481
+ oprot.writeFieldBegin('keyColumns', TType.LIST, 4)
2482
+ oprot.writeListBegin(TType.STRING, len(self.keyColumns))
2483
+ for iter231 in self.keyColumns:
2484
+ oprot.writeString(iter231.encode('utf-8') if sys.version_info[0] == 2 else iter231)
2485
+ oprot.writeListEnd()
2486
+ oprot.writeFieldEnd()
2487
+ oprot.writeFieldStop()
2488
+ oprot.writeStructEnd()
2489
+
2490
+ def validate(self):
2491
+ return
2492
+
2493
+ def __repr__(self):
2494
+ L = ['%s=%r' % (key, value)
2495
+ for key, value in self.__dict__.items()]
2496
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
2497
+
2498
+ def __eq__(self, other):
2499
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
2500
+
2501
+ def __ne__(self, other):
2502
+ return not (self == other)
2503
+
2504
+
2505
+ class LabelParts(object):
2506
+ """
2507
+ Attributes:
2508
+ - labels
2509
+ - leftStartOffset
2510
+ - leftEndOffset
2511
+ - metaData
2512
+
2513
+ """
2514
+ thrift_spec = None
2515
+
2516
+
2517
+ def __init__(self, labels = None, leftStartOffset = None, leftEndOffset = None, metaData = None,):
2518
+ self.labels = labels
2519
+ self.leftStartOffset = leftStartOffset
2520
+ self.leftEndOffset = leftEndOffset
2521
+ self.metaData = metaData
2522
+
2523
+ def read(self, iprot):
2524
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
2525
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
2526
+ return
2527
+ iprot.readStructBegin()
2528
+ while True:
2529
+ (fname, ftype, fid) = iprot.readFieldBegin()
2530
+ if ftype == TType.STOP:
2531
+ break
2532
+ if fid == 1:
2533
+ if ftype == TType.LIST:
2534
+ self.labels = []
2535
+ (_etype235, _size232) = iprot.readListBegin()
2536
+ for _i236 in range(_size232):
2537
+ _elem237 = JoinPart()
2538
+ _elem237.read(iprot)
2539
+ self.labels.append(_elem237)
2540
+ iprot.readListEnd()
2541
+ else:
2542
+ iprot.skip(ftype)
2543
+ elif fid == 2:
2544
+ if ftype == TType.I32:
2545
+ self.leftStartOffset = iprot.readI32()
2546
+ else:
2547
+ iprot.skip(ftype)
2548
+ elif fid == 3:
2549
+ if ftype == TType.I32:
2550
+ self.leftEndOffset = iprot.readI32()
2551
+ else:
2552
+ iprot.skip(ftype)
2553
+ elif fid == 4:
2554
+ if ftype == TType.STRUCT:
2555
+ self.metaData = MetaData()
2556
+ self.metaData.read(iprot)
2557
+ else:
2558
+ iprot.skip(ftype)
2559
+ else:
2560
+ iprot.skip(ftype)
2561
+ iprot.readFieldEnd()
2562
+ iprot.readStructEnd()
2563
+
2564
+ def write(self, oprot):
2565
+ self.validate()
2566
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
2567
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
2568
+ return
2569
+ oprot.writeStructBegin('LabelParts')
2570
+ if self.labels is not None:
2571
+ oprot.writeFieldBegin('labels', TType.LIST, 1)
2572
+ oprot.writeListBegin(TType.STRUCT, len(self.labels))
2573
+ for iter238 in self.labels:
2574
+ iter238.write(oprot)
2575
+ oprot.writeListEnd()
2576
+ oprot.writeFieldEnd()
2577
+ if self.leftStartOffset is not None:
2578
+ oprot.writeFieldBegin('leftStartOffset', TType.I32, 2)
2579
+ oprot.writeI32(self.leftStartOffset)
2580
+ oprot.writeFieldEnd()
2581
+ if self.leftEndOffset is not None:
2582
+ oprot.writeFieldBegin('leftEndOffset', TType.I32, 3)
2583
+ oprot.writeI32(self.leftEndOffset)
2584
+ oprot.writeFieldEnd()
2585
+ if self.metaData is not None:
2586
+ oprot.writeFieldBegin('metaData', TType.STRUCT, 4)
2587
+ self.metaData.write(oprot)
2588
+ oprot.writeFieldEnd()
2589
+ oprot.writeFieldStop()
2590
+ oprot.writeStructEnd()
2591
+
2592
+ def validate(self):
2593
+ return
2594
+
2595
+ def __repr__(self):
2596
+ L = ['%s=%r' % (key, value)
2597
+ for key, value in self.__dict__.items()]
2598
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
2599
+
2600
+ def __eq__(self, other):
2601
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
2602
+
2603
+ def __ne__(self, other):
2604
+ return not (self == other)
2605
+
2606
+
2607
+ class GroupByServingInfo(object):
2608
+ """
2609
+ Attributes:
2610
+ - groupBy
2611
+ - inputAvroSchema
2612
+ - selectedAvroSchema
2613
+ - keyAvroSchema
2614
+ - batchEndDate
2615
+ - dateFormat
2616
+
2617
+ """
2618
+ thrift_spec = None
2619
+
2620
+
2621
+ def __init__(self, groupBy = None, inputAvroSchema = None, selectedAvroSchema = None, keyAvroSchema = None, batchEndDate = None, dateFormat = None,):
2622
+ self.groupBy = groupBy
2623
+ self.inputAvroSchema = inputAvroSchema
2624
+ self.selectedAvroSchema = selectedAvroSchema
2625
+ self.keyAvroSchema = keyAvroSchema
2626
+ self.batchEndDate = batchEndDate
2627
+ self.dateFormat = dateFormat
2628
+
2629
+ def read(self, iprot):
2630
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
2631
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
2632
+ return
2633
+ iprot.readStructBegin()
2634
+ while True:
2635
+ (fname, ftype, fid) = iprot.readFieldBegin()
2636
+ if ftype == TType.STOP:
2637
+ break
2638
+ if fid == 1:
2639
+ if ftype == TType.STRUCT:
2640
+ self.groupBy = GroupBy()
2641
+ self.groupBy.read(iprot)
2642
+ else:
2643
+ iprot.skip(ftype)
2644
+ elif fid == 2:
2645
+ if ftype == TType.STRING:
2646
+ self.inputAvroSchema = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2647
+ else:
2648
+ iprot.skip(ftype)
2649
+ elif fid == 3:
2650
+ if ftype == TType.STRING:
2651
+ self.selectedAvroSchema = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2652
+ else:
2653
+ iprot.skip(ftype)
2654
+ elif fid == 4:
2655
+ if ftype == TType.STRING:
2656
+ self.keyAvroSchema = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2657
+ else:
2658
+ iprot.skip(ftype)
2659
+ elif fid == 5:
2660
+ if ftype == TType.STRING:
2661
+ self.batchEndDate = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2662
+ else:
2663
+ iprot.skip(ftype)
2664
+ elif fid == 6:
2665
+ if ftype == TType.STRING:
2666
+ self.dateFormat = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2667
+ else:
2668
+ iprot.skip(ftype)
2669
+ else:
2670
+ iprot.skip(ftype)
2671
+ iprot.readFieldEnd()
2672
+ iprot.readStructEnd()
2673
+
2674
+ def write(self, oprot):
2675
+ self.validate()
2676
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
2677
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
2678
+ return
2679
+ oprot.writeStructBegin('GroupByServingInfo')
2680
+ if self.groupBy is not None:
2681
+ oprot.writeFieldBegin('groupBy', TType.STRUCT, 1)
2682
+ self.groupBy.write(oprot)
2683
+ oprot.writeFieldEnd()
2684
+ if self.inputAvroSchema is not None:
2685
+ oprot.writeFieldBegin('inputAvroSchema', TType.STRING, 2)
2686
+ oprot.writeString(self.inputAvroSchema.encode('utf-8') if sys.version_info[0] == 2 else self.inputAvroSchema)
2687
+ oprot.writeFieldEnd()
2688
+ if self.selectedAvroSchema is not None:
2689
+ oprot.writeFieldBegin('selectedAvroSchema', TType.STRING, 3)
2690
+ oprot.writeString(self.selectedAvroSchema.encode('utf-8') if sys.version_info[0] == 2 else self.selectedAvroSchema)
2691
+ oprot.writeFieldEnd()
2692
+ if self.keyAvroSchema is not None:
2693
+ oprot.writeFieldBegin('keyAvroSchema', TType.STRING, 4)
2694
+ oprot.writeString(self.keyAvroSchema.encode('utf-8') if sys.version_info[0] == 2 else self.keyAvroSchema)
2695
+ oprot.writeFieldEnd()
2696
+ if self.batchEndDate is not None:
2697
+ oprot.writeFieldBegin('batchEndDate', TType.STRING, 5)
2698
+ oprot.writeString(self.batchEndDate.encode('utf-8') if sys.version_info[0] == 2 else self.batchEndDate)
2699
+ oprot.writeFieldEnd()
2700
+ if self.dateFormat is not None:
2701
+ oprot.writeFieldBegin('dateFormat', TType.STRING, 6)
2702
+ oprot.writeString(self.dateFormat.encode('utf-8') if sys.version_info[0] == 2 else self.dateFormat)
2703
+ oprot.writeFieldEnd()
2704
+ oprot.writeFieldStop()
2705
+ oprot.writeStructEnd()
2706
+
2707
+ def validate(self):
2708
+ return
2709
+
2710
+ def __repr__(self):
2711
+ L = ['%s=%r' % (key, value)
2712
+ for key, value in self.__dict__.items()]
2713
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
2714
+
2715
+ def __eq__(self, other):
2716
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
2717
+
2718
+ def __ne__(self, other):
2719
+ return not (self == other)
2720
+
2721
+
2722
+ class DataField(object):
2723
+ """
2724
+ Attributes:
2725
+ - name
2726
+ - dataType
2727
+
2728
+ """
2729
+ thrift_spec = None
2730
+
2731
+
2732
+ def __init__(self, name = None, dataType = None,):
2733
+ self.name = name
2734
+ self.dataType = dataType
2735
+
2736
+ def read(self, iprot):
2737
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
2738
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
2739
+ return
2740
+ iprot.readStructBegin()
2741
+ while True:
2742
+ (fname, ftype, fid) = iprot.readFieldBegin()
2743
+ if ftype == TType.STOP:
2744
+ break
2745
+ if fid == 1:
2746
+ if ftype == TType.STRING:
2747
+ self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2748
+ else:
2749
+ iprot.skip(ftype)
2750
+ elif fid == 2:
2751
+ if ftype == TType.STRUCT:
2752
+ self.dataType = TDataType()
2753
+ self.dataType.read(iprot)
2754
+ else:
2755
+ iprot.skip(ftype)
2756
+ else:
2757
+ iprot.skip(ftype)
2758
+ iprot.readFieldEnd()
2759
+ iprot.readStructEnd()
2760
+
2761
+ def write(self, oprot):
2762
+ self.validate()
2763
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
2764
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
2765
+ return
2766
+ oprot.writeStructBegin('DataField')
2767
+ if self.name is not None:
2768
+ oprot.writeFieldBegin('name', TType.STRING, 1)
2769
+ oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name)
2770
+ oprot.writeFieldEnd()
2771
+ if self.dataType is not None:
2772
+ oprot.writeFieldBegin('dataType', TType.STRUCT, 2)
2773
+ self.dataType.write(oprot)
2774
+ oprot.writeFieldEnd()
2775
+ oprot.writeFieldStop()
2776
+ oprot.writeStructEnd()
2777
+
2778
+ def validate(self):
2779
+ return
2780
+
2781
+ def __repr__(self):
2782
+ L = ['%s=%r' % (key, value)
2783
+ for key, value in self.__dict__.items()]
2784
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
2785
+
2786
+ def __eq__(self, other):
2787
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
2788
+
2789
+ def __ne__(self, other):
2790
+ return not (self == other)
2791
+
2792
+
2793
+ class TDataType(object):
2794
+ """
2795
+ Attributes:
2796
+ - kind
2797
+ - params
2798
+ - name
2799
+
2800
+ """
2801
+ thrift_spec = None
2802
+
2803
+
2804
+ def __init__(self, kind = None, params = None, name = None,):
2805
+ self.kind = kind
2806
+ self.params = params
2807
+ self.name = name
2808
+
2809
+ def read(self, iprot):
2810
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
2811
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
2812
+ return
2813
+ iprot.readStructBegin()
2814
+ while True:
2815
+ (fname, ftype, fid) = iprot.readFieldBegin()
2816
+ if ftype == TType.STOP:
2817
+ break
2818
+ if fid == 1:
2819
+ if ftype == TType.I32:
2820
+ self.kind = iprot.readI32()
2821
+ else:
2822
+ iprot.skip(ftype)
2823
+ elif fid == 2:
2824
+ if ftype == TType.LIST:
2825
+ self.params = []
2826
+ (_etype242, _size239) = iprot.readListBegin()
2827
+ for _i243 in range(_size239):
2828
+ _elem244 = DataField()
2829
+ _elem244.read(iprot)
2830
+ self.params.append(_elem244)
2831
+ iprot.readListEnd()
2832
+ else:
2833
+ iprot.skip(ftype)
2834
+ elif fid == 3:
2835
+ if ftype == TType.STRING:
2836
+ self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2837
+ else:
2838
+ iprot.skip(ftype)
2839
+ else:
2840
+ iprot.skip(ftype)
2841
+ iprot.readFieldEnd()
2842
+ iprot.readStructEnd()
2843
+
2844
+ def write(self, oprot):
2845
+ self.validate()
2846
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
2847
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
2848
+ return
2849
+ oprot.writeStructBegin('TDataType')
2850
+ if self.kind is not None:
2851
+ oprot.writeFieldBegin('kind', TType.I32, 1)
2852
+ oprot.writeI32(self.kind)
2853
+ oprot.writeFieldEnd()
2854
+ if self.params is not None:
2855
+ oprot.writeFieldBegin('params', TType.LIST, 2)
2856
+ oprot.writeListBegin(TType.STRUCT, len(self.params))
2857
+ for iter245 in self.params:
2858
+ iter245.write(oprot)
2859
+ oprot.writeListEnd()
2860
+ oprot.writeFieldEnd()
2861
+ if self.name is not None:
2862
+ oprot.writeFieldBegin('name', TType.STRING, 3)
2863
+ oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name)
2864
+ oprot.writeFieldEnd()
2865
+ oprot.writeFieldStop()
2866
+ oprot.writeStructEnd()
2867
+
2868
+ def validate(self):
2869
+ return
2870
+
2871
+ def __repr__(self):
2872
+ L = ['%s=%r' % (key, value)
2873
+ for key, value in self.__dict__.items()]
2874
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
2875
+
2876
+ def __eq__(self, other):
2877
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
2878
+
2879
+ def __ne__(self, other):
2880
+ return not (self == other)
2881
+
2882
+
2883
+ class DataSpec(object):
2884
+ """
2885
+ Attributes:
2886
+ - schema
2887
+ - partitionColumns
2888
+ - retentionDays
2889
+ - props
2890
+
2891
+ """
2892
+ thrift_spec = None
2893
+
2894
+
2895
+ def __init__(self, schema = None, partitionColumns = None, retentionDays = None, props = None,):
2896
+ self.schema = schema
2897
+ self.partitionColumns = partitionColumns
2898
+ self.retentionDays = retentionDays
2899
+ self.props = props
2900
+
2901
+ def read(self, iprot):
2902
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
2903
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
2904
+ return
2905
+ iprot.readStructBegin()
2906
+ while True:
2907
+ (fname, ftype, fid) = iprot.readFieldBegin()
2908
+ if ftype == TType.STOP:
2909
+ break
2910
+ if fid == 1:
2911
+ if ftype == TType.STRUCT:
2912
+ self.schema = TDataType()
2913
+ self.schema.read(iprot)
2914
+ else:
2915
+ iprot.skip(ftype)
2916
+ elif fid == 2:
2917
+ if ftype == TType.LIST:
2918
+ self.partitionColumns = []
2919
+ (_etype249, _size246) = iprot.readListBegin()
2920
+ for _i250 in range(_size246):
2921
+ _elem251 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2922
+ self.partitionColumns.append(_elem251)
2923
+ iprot.readListEnd()
2924
+ else:
2925
+ iprot.skip(ftype)
2926
+ elif fid == 3:
2927
+ if ftype == TType.I32:
2928
+ self.retentionDays = iprot.readI32()
2929
+ else:
2930
+ iprot.skip(ftype)
2931
+ elif fid == 4:
2932
+ if ftype == TType.MAP:
2933
+ self.props = {}
2934
+ (_ktype253, _vtype254, _size252) = iprot.readMapBegin()
2935
+ for _i256 in range(_size252):
2936
+ _key257 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2937
+ _val258 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
2938
+ self.props[_key257] = _val258
2939
+ iprot.readMapEnd()
2940
+ else:
2941
+ iprot.skip(ftype)
2942
+ else:
2943
+ iprot.skip(ftype)
2944
+ iprot.readFieldEnd()
2945
+ iprot.readStructEnd()
2946
+
2947
+ def write(self, oprot):
2948
+ self.validate()
2949
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
2950
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
2951
+ return
2952
+ oprot.writeStructBegin('DataSpec')
2953
+ if self.schema is not None:
2954
+ oprot.writeFieldBegin('schema', TType.STRUCT, 1)
2955
+ self.schema.write(oprot)
2956
+ oprot.writeFieldEnd()
2957
+ if self.partitionColumns is not None:
2958
+ oprot.writeFieldBegin('partitionColumns', TType.LIST, 2)
2959
+ oprot.writeListBegin(TType.STRING, len(self.partitionColumns))
2960
+ for iter259 in self.partitionColumns:
2961
+ oprot.writeString(iter259.encode('utf-8') if sys.version_info[0] == 2 else iter259)
2962
+ oprot.writeListEnd()
2963
+ oprot.writeFieldEnd()
2964
+ if self.retentionDays is not None:
2965
+ oprot.writeFieldBegin('retentionDays', TType.I32, 3)
2966
+ oprot.writeI32(self.retentionDays)
2967
+ oprot.writeFieldEnd()
2968
+ if self.props is not None:
2969
+ oprot.writeFieldBegin('props', TType.MAP, 4)
2970
+ oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.props))
2971
+ for kiter260, viter261 in self.props.items():
2972
+ oprot.writeString(kiter260.encode('utf-8') if sys.version_info[0] == 2 else kiter260)
2973
+ oprot.writeString(viter261.encode('utf-8') if sys.version_info[0] == 2 else viter261)
2974
+ oprot.writeMapEnd()
2975
+ oprot.writeFieldEnd()
2976
+ oprot.writeFieldStop()
2977
+ oprot.writeStructEnd()
2978
+
2979
+ def validate(self):
2980
+ return
2981
+
2982
+ def __repr__(self):
2983
+ L = ['%s=%r' % (key, value)
2984
+ for key, value in self.__dict__.items()]
2985
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
2986
+
2987
+ def __eq__(self, other):
2988
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
2989
+
2990
+ def __ne__(self, other):
2991
+ return not (self == other)
2992
+
2993
+
2994
+ class Model(object):
2995
+ """
2996
+ Attributes:
2997
+ - metaData
2998
+ - modelType
2999
+ - outputSchema
3000
+ - source
3001
+ - modelParams
3002
+
3003
+ """
3004
+ thrift_spec = None
3005
+
3006
+
3007
+ def __init__(self, metaData = None, modelType = None, outputSchema = None, source = None, modelParams = None,):
3008
+ self.metaData = metaData
3009
+ self.modelType = modelType
3010
+ self.outputSchema = outputSchema
3011
+ self.source = source
3012
+ self.modelParams = modelParams
3013
+
3014
+ def read(self, iprot):
3015
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
3016
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
3017
+ return
3018
+ iprot.readStructBegin()
3019
+ while True:
3020
+ (fname, ftype, fid) = iprot.readFieldBegin()
3021
+ if ftype == TType.STOP:
3022
+ break
3023
+ if fid == 1:
3024
+ if ftype == TType.STRUCT:
3025
+ self.metaData = MetaData()
3026
+ self.metaData.read(iprot)
3027
+ else:
3028
+ iprot.skip(ftype)
3029
+ elif fid == 2:
3030
+ if ftype == TType.I32:
3031
+ self.modelType = iprot.readI32()
3032
+ else:
3033
+ iprot.skip(ftype)
3034
+ elif fid == 3:
3035
+ if ftype == TType.STRUCT:
3036
+ self.outputSchema = TDataType()
3037
+ self.outputSchema.read(iprot)
3038
+ else:
3039
+ iprot.skip(ftype)
3040
+ elif fid == 4:
3041
+ if ftype == TType.STRUCT:
3042
+ self.source = Source()
3043
+ self.source.read(iprot)
3044
+ else:
3045
+ iprot.skip(ftype)
3046
+ elif fid == 5:
3047
+ if ftype == TType.MAP:
3048
+ self.modelParams = {}
3049
+ (_ktype263, _vtype264, _size262) = iprot.readMapBegin()
3050
+ for _i266 in range(_size262):
3051
+ _key267 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
3052
+ _val268 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
3053
+ self.modelParams[_key267] = _val268
3054
+ iprot.readMapEnd()
3055
+ else:
3056
+ iprot.skip(ftype)
3057
+ else:
3058
+ iprot.skip(ftype)
3059
+ iprot.readFieldEnd()
3060
+ iprot.readStructEnd()
3061
+
3062
+ def write(self, oprot):
3063
+ self.validate()
3064
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
3065
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
3066
+ return
3067
+ oprot.writeStructBegin('Model')
3068
+ if self.metaData is not None:
3069
+ oprot.writeFieldBegin('metaData', TType.STRUCT, 1)
3070
+ self.metaData.write(oprot)
3071
+ oprot.writeFieldEnd()
3072
+ if self.modelType is not None:
3073
+ oprot.writeFieldBegin('modelType', TType.I32, 2)
3074
+ oprot.writeI32(self.modelType)
3075
+ oprot.writeFieldEnd()
3076
+ if self.outputSchema is not None:
3077
+ oprot.writeFieldBegin('outputSchema', TType.STRUCT, 3)
3078
+ self.outputSchema.write(oprot)
3079
+ oprot.writeFieldEnd()
3080
+ if self.source is not None:
3081
+ oprot.writeFieldBegin('source', TType.STRUCT, 4)
3082
+ self.source.write(oprot)
3083
+ oprot.writeFieldEnd()
3084
+ if self.modelParams is not None:
3085
+ oprot.writeFieldBegin('modelParams', TType.MAP, 5)
3086
+ oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.modelParams))
3087
+ for kiter269, viter270 in self.modelParams.items():
3088
+ oprot.writeString(kiter269.encode('utf-8') if sys.version_info[0] == 2 else kiter269)
3089
+ oprot.writeString(viter270.encode('utf-8') if sys.version_info[0] == 2 else viter270)
3090
+ oprot.writeMapEnd()
3091
+ oprot.writeFieldEnd()
3092
+ oprot.writeFieldStop()
3093
+ oprot.writeStructEnd()
3094
+
3095
+ def validate(self):
3096
+ return
3097
+
3098
+ def __repr__(self):
3099
+ L = ['%s=%r' % (key, value)
3100
+ for key, value in self.__dict__.items()]
3101
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
3102
+
3103
+ def __eq__(self, other):
3104
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
3105
+
3106
+ def __ne__(self, other):
3107
+ return not (self == other)
3108
+
3109
+
3110
+ class Team(object):
3111
+ """
3112
+ Attributes:
3113
+ - name
3114
+ - description
3115
+ - email
3116
+ - outputNamespace
3117
+ - tableProperties
3118
+ - env
3119
+ - conf
3120
+ - clusterConf
3121
+
3122
+ """
3123
+ thrift_spec = None
3124
+
3125
+
3126
+ def __init__(self, name = None, description = None, email = None, outputNamespace = None, tableProperties = None, env = None, conf = None, clusterConf = None,):
3127
+ self.name = name
3128
+ self.description = description
3129
+ self.email = email
3130
+ self.outputNamespace = outputNamespace
3131
+ self.tableProperties = tableProperties
3132
+ self.env = env
3133
+ self.conf = conf
3134
+ self.clusterConf = clusterConf
3135
+
3136
+ def read(self, iprot):
3137
+ if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
3138
+ iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
3139
+ return
3140
+ iprot.readStructBegin()
3141
+ while True:
3142
+ (fname, ftype, fid) = iprot.readFieldBegin()
3143
+ if ftype == TType.STOP:
3144
+ break
3145
+ if fid == 1:
3146
+ if ftype == TType.STRING:
3147
+ self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
3148
+ else:
3149
+ iprot.skip(ftype)
3150
+ elif fid == 2:
3151
+ if ftype == TType.STRING:
3152
+ self.description = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
3153
+ else:
3154
+ iprot.skip(ftype)
3155
+ elif fid == 3:
3156
+ if ftype == TType.STRING:
3157
+ self.email = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
3158
+ else:
3159
+ iprot.skip(ftype)
3160
+ elif fid == 10:
3161
+ if ftype == TType.STRING:
3162
+ self.outputNamespace = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
3163
+ else:
3164
+ iprot.skip(ftype)
3165
+ elif fid == 11:
3166
+ if ftype == TType.MAP:
3167
+ self.tableProperties = {}
3168
+ (_ktype272, _vtype273, _size271) = iprot.readMapBegin()
3169
+ for _i275 in range(_size271):
3170
+ _key276 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
3171
+ _val277 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
3172
+ self.tableProperties[_key276] = _val277
3173
+ iprot.readMapEnd()
3174
+ else:
3175
+ iprot.skip(ftype)
3176
+ elif fid == 20:
3177
+ if ftype == TType.STRUCT:
3178
+ self.env = gen_thrift.common.ttypes.EnvironmentVariables()
3179
+ self.env.read(iprot)
3180
+ else:
3181
+ iprot.skip(ftype)
3182
+ elif fid == 21:
3183
+ if ftype == TType.STRUCT:
3184
+ self.conf = gen_thrift.common.ttypes.ConfigProperties()
3185
+ self.conf.read(iprot)
3186
+ else:
3187
+ iprot.skip(ftype)
3188
+ elif fid == 22:
3189
+ if ftype == TType.STRUCT:
3190
+ self.clusterConf = gen_thrift.common.ttypes.ClusterConfigProperties()
3191
+ self.clusterConf.read(iprot)
3192
+ else:
3193
+ iprot.skip(ftype)
3194
+ else:
3195
+ iprot.skip(ftype)
3196
+ iprot.readFieldEnd()
3197
+ iprot.readStructEnd()
3198
+
3199
+ def write(self, oprot):
3200
+ self.validate()
3201
+ if oprot._fast_encode is not None and self.thrift_spec is not None:
3202
+ oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
3203
+ return
3204
+ oprot.writeStructBegin('Team')
3205
+ if self.name is not None:
3206
+ oprot.writeFieldBegin('name', TType.STRING, 1)
3207
+ oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name)
3208
+ oprot.writeFieldEnd()
3209
+ if self.description is not None:
3210
+ oprot.writeFieldBegin('description', TType.STRING, 2)
3211
+ oprot.writeString(self.description.encode('utf-8') if sys.version_info[0] == 2 else self.description)
3212
+ oprot.writeFieldEnd()
3213
+ if self.email is not None:
3214
+ oprot.writeFieldBegin('email', TType.STRING, 3)
3215
+ oprot.writeString(self.email.encode('utf-8') if sys.version_info[0] == 2 else self.email)
3216
+ oprot.writeFieldEnd()
3217
+ if self.outputNamespace is not None:
3218
+ oprot.writeFieldBegin('outputNamespace', TType.STRING, 10)
3219
+ oprot.writeString(self.outputNamespace.encode('utf-8') if sys.version_info[0] == 2 else self.outputNamespace)
3220
+ oprot.writeFieldEnd()
3221
+ if self.tableProperties is not None:
3222
+ oprot.writeFieldBegin('tableProperties', TType.MAP, 11)
3223
+ oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.tableProperties))
3224
+ for kiter278, viter279 in self.tableProperties.items():
3225
+ oprot.writeString(kiter278.encode('utf-8') if sys.version_info[0] == 2 else kiter278)
3226
+ oprot.writeString(viter279.encode('utf-8') if sys.version_info[0] == 2 else viter279)
3227
+ oprot.writeMapEnd()
3228
+ oprot.writeFieldEnd()
3229
+ if self.env is not None:
3230
+ oprot.writeFieldBegin('env', TType.STRUCT, 20)
3231
+ self.env.write(oprot)
3232
+ oprot.writeFieldEnd()
3233
+ if self.conf is not None:
3234
+ oprot.writeFieldBegin('conf', TType.STRUCT, 21)
3235
+ self.conf.write(oprot)
3236
+ oprot.writeFieldEnd()
3237
+ if self.clusterConf is not None:
3238
+ oprot.writeFieldBegin('clusterConf', TType.STRUCT, 22)
3239
+ self.clusterConf.write(oprot)
3240
+ oprot.writeFieldEnd()
3241
+ oprot.writeFieldStop()
3242
+ oprot.writeStructEnd()
3243
+
3244
+ def validate(self):
3245
+ return
3246
+
3247
+ def __repr__(self):
3248
+ L = ['%s=%r' % (key, value)
3249
+ for key, value in self.__dict__.items()]
3250
+ return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
3251
+
3252
+ def __eq__(self, other):
3253
+ return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
3254
+
3255
+ def __ne__(self, other):
3256
+ return not (self == other)
3257
+ all_structs.append(Query)
3258
+ Query.thrift_spec = (
3259
+ None, # 0
3260
+ (1, TType.MAP, 'selects', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 1
3261
+ (2, TType.LIST, 'wheres', (TType.STRING, 'UTF8', False), None, ), # 2
3262
+ (3, TType.STRING, 'startPartition', 'UTF8', None, ), # 3
3263
+ (4, TType.STRING, 'endPartition', 'UTF8', None, ), # 4
3264
+ (5, TType.STRING, 'timeColumn', 'UTF8', None, ), # 5
3265
+ (6, TType.LIST, 'setups', (TType.STRING, 'UTF8', False), None, ), # 6
3266
+ (7, TType.STRING, 'mutationTimeColumn', 'UTF8', None, ), # 7
3267
+ (8, TType.STRING, 'reversalColumn', 'UTF8', None, ), # 8
3268
+ None, # 9
3269
+ None, # 10
3270
+ None, # 11
3271
+ None, # 12
3272
+ None, # 13
3273
+ None, # 14
3274
+ None, # 15
3275
+ None, # 16
3276
+ None, # 17
3277
+ None, # 18
3278
+ None, # 19
3279
+ (20, TType.STRING, 'partitionColumn', 'UTF8', None, ), # 20
3280
+ (21, TType.STRING, 'partitionFormat', 'UTF8', None, ), # 21
3281
+ (22, TType.STRUCT, 'partitionInterval', [gen_thrift.common.ttypes.Window, None], None, ), # 22
3282
+ (23, TType.STRUCT, 'partitionLag', [gen_thrift.common.ttypes.Window, None], None, ), # 23
3283
+ (24, TType.LIST, 'subPartitionsToWaitFor', (TType.STRING, 'UTF8', False), None, ), # 24
3284
+ )
3285
+ all_structs.append(StagingQuery)
3286
+ StagingQuery.thrift_spec = (
3287
+ None, # 0
3288
+ (1, TType.STRUCT, 'metaData', [MetaData, None], None, ), # 1
3289
+ (2, TType.STRING, 'query', 'UTF8', None, ), # 2
3290
+ (3, TType.STRING, 'startPartition', 'UTF8', None, ), # 3
3291
+ (4, TType.LIST, 'setups', (TType.STRING, 'UTF8', False), None, ), # 4
3292
+ (5, TType.I32, 'engineType', None, None, ), # 5
3293
+ (6, TType.LIST, 'tableDependencies', (TType.STRUCT, [gen_thrift.common.ttypes.TableDependency, None], False), None, ), # 6
3294
+ None, # 7
3295
+ None, # 8
3296
+ None, # 9
3297
+ None, # 10
3298
+ None, # 11
3299
+ None, # 12
3300
+ None, # 13
3301
+ None, # 14
3302
+ None, # 15
3303
+ None, # 16
3304
+ None, # 17
3305
+ None, # 18
3306
+ None, # 19
3307
+ (20, TType.I32, 'recomputeDays', None, None, ), # 20
3308
+ )
3309
+ all_structs.append(EventSource)
3310
+ EventSource.thrift_spec = (
3311
+ None, # 0
3312
+ (1, TType.STRING, 'table', 'UTF8', None, ), # 1
3313
+ (2, TType.STRING, 'topic', 'UTF8', None, ), # 2
3314
+ (3, TType.STRUCT, 'query', [Query, None], None, ), # 3
3315
+ (4, TType.BOOL, 'isCumulative', None, None, ), # 4
3316
+ )
3317
+ all_structs.append(EntitySource)
3318
+ EntitySource.thrift_spec = (
3319
+ None, # 0
3320
+ (1, TType.STRING, 'snapshotTable', 'UTF8', None, ), # 1
3321
+ (2, TType.STRING, 'mutationTable', 'UTF8', None, ), # 2
3322
+ (3, TType.STRING, 'mutationTopic', 'UTF8', None, ), # 3
3323
+ (4, TType.STRUCT, 'query', [Query, None], None, ), # 4
3324
+ )
3325
+ all_structs.append(ExternalSource)
3326
+ ExternalSource.thrift_spec = (
3327
+ None, # 0
3328
+ (1, TType.STRUCT, 'metadata', [MetaData, None], None, ), # 1
3329
+ (2, TType.STRUCT, 'keySchema', [TDataType, None], None, ), # 2
3330
+ (3, TType.STRUCT, 'valueSchema', [TDataType, None], None, ), # 3
3331
+ )
3332
+ all_structs.append(JoinSource)
3333
+ JoinSource.thrift_spec = (
3334
+ None, # 0
3335
+ (1, TType.STRUCT, 'join', [Join, None], None, ), # 1
3336
+ (2, TType.STRUCT, 'query', [Query, None], None, ), # 2
3337
+ )
3338
+ all_structs.append(Source)
3339
+ Source.thrift_spec = (
3340
+ None, # 0
3341
+ (1, TType.STRUCT, 'events', [EventSource, None], None, ), # 1
3342
+ (2, TType.STRUCT, 'entities', [EntitySource, None], None, ), # 2
3343
+ (3, TType.STRUCT, 'joinSource', [JoinSource, None], None, ), # 3
3344
+ )
3345
+ all_structs.append(Aggregation)
3346
+ Aggregation.thrift_spec = (
3347
+ None, # 0
3348
+ (1, TType.STRING, 'inputColumn', 'UTF8', None, ), # 1
3349
+ (2, TType.I32, 'operation', None, None, ), # 2
3350
+ (3, TType.MAP, 'argMap', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 3
3351
+ (4, TType.LIST, 'windows', (TType.STRUCT, [gen_thrift.common.ttypes.Window, None], False), None, ), # 4
3352
+ (5, TType.LIST, 'buckets', (TType.STRING, 'UTF8', False), None, ), # 5
3353
+ )
3354
+ all_structs.append(AggregationPart)
3355
+ AggregationPart.thrift_spec = (
3356
+ None, # 0
3357
+ (1, TType.STRING, 'inputColumn', 'UTF8', None, ), # 1
3358
+ (2, TType.I32, 'operation', None, None, ), # 2
3359
+ (3, TType.MAP, 'argMap', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 3
3360
+ (4, TType.STRUCT, 'window', [gen_thrift.common.ttypes.Window, None], None, ), # 4
3361
+ (5, TType.STRING, 'bucket', 'UTF8', None, ), # 5
3362
+ )
3363
+ all_structs.append(MetaData)
3364
+ MetaData.thrift_spec = (
3365
+ None, # 0
3366
+ (1, TType.STRING, 'name', 'UTF8', None, ), # 1
3367
+ (2, TType.STRING, 'team', 'UTF8', None, ), # 2
3368
+ (3, TType.STRING, 'version', 'UTF8', None, ), # 3
3369
+ (4, TType.STRING, 'outputNamespace', 'UTF8', None, ), # 4
3370
+ (5, TType.LIST, 'additionalOutputPartitionColumns', (TType.STRING, 'UTF8', False), None, ), # 5
3371
+ (6, TType.MAP, 'tableProperties', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 6
3372
+ None, # 7
3373
+ None, # 8
3374
+ None, # 9
3375
+ None, # 10
3376
+ None, # 11
3377
+ None, # 12
3378
+ None, # 13
3379
+ None, # 14
3380
+ None, # 15
3381
+ None, # 16
3382
+ None, # 17
3383
+ None, # 18
3384
+ None, # 19
3385
+ (20, TType.MAP, 'tags', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 20
3386
+ (21, TType.MAP, 'columnTags', (TType.STRING, 'UTF8', TType.MAP, (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), False), None, ), # 21
3387
+ (22, TType.MAP, 'columnHashes', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 22
3388
+ None, # 23
3389
+ None, # 24
3390
+ None, # 25
3391
+ None, # 26
3392
+ None, # 27
3393
+ None, # 28
3394
+ None, # 29
3395
+ None, # 30
3396
+ None, # 31
3397
+ None, # 32
3398
+ None, # 33
3399
+ None, # 34
3400
+ None, # 35
3401
+ None, # 36
3402
+ None, # 37
3403
+ None, # 38
3404
+ None, # 39
3405
+ None, # 40
3406
+ None, # 41
3407
+ None, # 42
3408
+ None, # 43
3409
+ None, # 44
3410
+ None, # 45
3411
+ None, # 46
3412
+ None, # 47
3413
+ None, # 48
3414
+ None, # 49
3415
+ None, # 50
3416
+ None, # 51
3417
+ None, # 52
3418
+ None, # 53
3419
+ None, # 54
3420
+ None, # 55
3421
+ None, # 56
3422
+ None, # 57
3423
+ None, # 58
3424
+ None, # 59
3425
+ None, # 60
3426
+ None, # 61
3427
+ None, # 62
3428
+ None, # 63
3429
+ None, # 64
3430
+ None, # 65
3431
+ None, # 66
3432
+ None, # 67
3433
+ None, # 68
3434
+ None, # 69
3435
+ None, # 70
3436
+ None, # 71
3437
+ None, # 72
3438
+ None, # 73
3439
+ None, # 74
3440
+ None, # 75
3441
+ None, # 76
3442
+ None, # 77
3443
+ None, # 78
3444
+ None, # 79
3445
+ None, # 80
3446
+ None, # 81
3447
+ None, # 82
3448
+ None, # 83
3449
+ None, # 84
3450
+ None, # 85
3451
+ None, # 86
3452
+ None, # 87
3453
+ None, # 88
3454
+ None, # 89
3455
+ None, # 90
3456
+ None, # 91
3457
+ None, # 92
3458
+ None, # 93
3459
+ None, # 94
3460
+ None, # 95
3461
+ None, # 96
3462
+ None, # 97
3463
+ None, # 98
3464
+ None, # 99
3465
+ (100, TType.BOOL, 'online', None, None, ), # 100
3466
+ (101, TType.BOOL, 'production', None, None, ), # 101
3467
+ (102, TType.STRING, 'sourceFile', 'UTF8', None, ), # 102
3468
+ (103, TType.STRING, 'customJson', 'UTF8', None, ), # 103
3469
+ None, # 104
3470
+ None, # 105
3471
+ None, # 106
3472
+ None, # 107
3473
+ None, # 108
3474
+ None, # 109
3475
+ None, # 110
3476
+ None, # 111
3477
+ None, # 112
3478
+ None, # 113
3479
+ None, # 114
3480
+ None, # 115
3481
+ None, # 116
3482
+ None, # 117
3483
+ None, # 118
3484
+ None, # 119
3485
+ None, # 120
3486
+ None, # 121
3487
+ None, # 122
3488
+ None, # 123
3489
+ None, # 124
3490
+ None, # 125
3491
+ None, # 126
3492
+ None, # 127
3493
+ None, # 128
3494
+ None, # 129
3495
+ None, # 130
3496
+ None, # 131
3497
+ None, # 132
3498
+ None, # 133
3499
+ None, # 134
3500
+ None, # 135
3501
+ None, # 136
3502
+ None, # 137
3503
+ None, # 138
3504
+ None, # 139
3505
+ None, # 140
3506
+ None, # 141
3507
+ None, # 142
3508
+ None, # 143
3509
+ None, # 144
3510
+ None, # 145
3511
+ None, # 146
3512
+ None, # 147
3513
+ None, # 148
3514
+ None, # 149
3515
+ None, # 150
3516
+ None, # 151
3517
+ None, # 152
3518
+ None, # 153
3519
+ None, # 154
3520
+ None, # 155
3521
+ None, # 156
3522
+ None, # 157
3523
+ None, # 158
3524
+ None, # 159
3525
+ None, # 160
3526
+ None, # 161
3527
+ None, # 162
3528
+ None, # 163
3529
+ None, # 164
3530
+ None, # 165
3531
+ None, # 166
3532
+ None, # 167
3533
+ None, # 168
3534
+ None, # 169
3535
+ None, # 170
3536
+ None, # 171
3537
+ None, # 172
3538
+ None, # 173
3539
+ None, # 174
3540
+ None, # 175
3541
+ None, # 176
3542
+ None, # 177
3543
+ None, # 178
3544
+ None, # 179
3545
+ None, # 180
3546
+ None, # 181
3547
+ None, # 182
3548
+ None, # 183
3549
+ None, # 184
3550
+ None, # 185
3551
+ None, # 186
3552
+ None, # 187
3553
+ None, # 188
3554
+ None, # 189
3555
+ None, # 190
3556
+ None, # 191
3557
+ None, # 192
3558
+ None, # 193
3559
+ None, # 194
3560
+ None, # 195
3561
+ None, # 196
3562
+ None, # 197
3563
+ None, # 198
3564
+ None, # 199
3565
+ (200, TType.BOOL, 'consistencyCheck', None, None, ), # 200
3566
+ (201, TType.DOUBLE, 'samplePercent', None, None, ), # 201
3567
+ (202, TType.DOUBLE, 'consistencySamplePercent', None, None, ), # 202
3568
+ (203, TType.STRUCT, 'driftSpec', [gen_thrift.observability.ttypes.DriftSpec, None], None, ), # 203
3569
+ (204, TType.STRUCT, 'executionInfo', [gen_thrift.common.ttypes.ExecutionInfo, None], None, ), # 204
3570
+ )
3571
+ all_structs.append(Conf)
3572
+ Conf.thrift_spec = (
3573
+ None, # 0
3574
+ (1, TType.STRING, 'name', 'UTF8', None, ), # 1
3575
+ (2, TType.STRING, 'hash', 'UTF8', None, ), # 2
3576
+ (3, TType.STRING, 'contents', 'UTF8', None, ), # 3
3577
+ (4, TType.I32, 'confType', None, None, ), # 4
3578
+ )
3579
+ all_structs.append(GroupBy)
3580
+ GroupBy.thrift_spec = (
3581
+ None, # 0
3582
+ (1, TType.STRUCT, 'metaData', [MetaData, None], None, ), # 1
3583
+ (2, TType.LIST, 'sources', (TType.STRUCT, [Source, None], False), None, ), # 2
3584
+ (3, TType.LIST, 'keyColumns', (TType.STRING, 'UTF8', False), None, ), # 3
3585
+ (4, TType.LIST, 'aggregations', (TType.STRUCT, [Aggregation, None], False), None, ), # 4
3586
+ (5, TType.I32, 'accuracy', None, None, ), # 5
3587
+ (6, TType.STRING, 'backfillStartDate', 'UTF8', None, ), # 6
3588
+ (7, TType.LIST, 'derivations', (TType.STRUCT, [Derivation, None], False), None, ), # 7
3589
+ )
3590
+ all_structs.append(JoinPart)
3591
+ JoinPart.thrift_spec = (
3592
+ None, # 0
3593
+ (1, TType.STRUCT, 'groupBy', [GroupBy, None], None, ), # 1
3594
+ (2, TType.MAP, 'keyMapping', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 2
3595
+ (3, TType.STRING, 'prefix', 'UTF8', None, ), # 3
3596
+ None, # 4
3597
+ None, # 5
3598
+ None, # 6
3599
+ None, # 7
3600
+ None, # 8
3601
+ None, # 9
3602
+ (10, TType.BOOL, 'useLongNames', None, None, ), # 10
3603
+ )
3604
+ all_structs.append(ExternalPart)
3605
+ ExternalPart.thrift_spec = (
3606
+ None, # 0
3607
+ (1, TType.STRUCT, 'source', [ExternalSource, None], None, ), # 1
3608
+ (2, TType.MAP, 'keyMapping', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 2
3609
+ (3, TType.STRING, 'prefix', 'UTF8', None, ), # 3
3610
+ )
3611
+ all_structs.append(Derivation)
3612
+ Derivation.thrift_spec = (
3613
+ None, # 0
3614
+ (1, TType.STRING, 'name', 'UTF8', None, ), # 1
3615
+ (2, TType.STRING, 'expression', 'UTF8', None, ), # 2
3616
+ )
3617
+ all_structs.append(Join)
3618
+ Join.thrift_spec = (
3619
+ None, # 0
3620
+ (1, TType.STRUCT, 'metaData', [MetaData, None], None, ), # 1
3621
+ (2, TType.STRUCT, 'left', [Source, None], None, ), # 2
3622
+ (3, TType.LIST, 'joinParts', (TType.STRUCT, [JoinPart, None], False), None, ), # 3
3623
+ (4, TType.MAP, 'skewKeys', (TType.STRING, 'UTF8', TType.LIST, (TType.STRING, 'UTF8', False), False), None, ), # 4
3624
+ (5, TType.LIST, 'onlineExternalParts', (TType.STRUCT, [ExternalPart, None], False), None, ), # 5
3625
+ (6, TType.STRUCT, 'labelParts', [LabelParts, None], None, ), # 6
3626
+ (7, TType.LIST, 'bootstrapParts', (TType.STRUCT, [BootstrapPart, None], False), None, ), # 7
3627
+ (8, TType.LIST, 'rowIds', (TType.STRING, 'UTF8', False), None, ), # 8
3628
+ (9, TType.LIST, 'derivations', (TType.STRUCT, [Derivation, None], False), None, ), # 9
3629
+ None, # 10
3630
+ None, # 11
3631
+ None, # 12
3632
+ None, # 13
3633
+ None, # 14
3634
+ None, # 15
3635
+ None, # 16
3636
+ None, # 17
3637
+ None, # 18
3638
+ None, # 19
3639
+ None, # 20
3640
+ None, # 21
3641
+ None, # 22
3642
+ None, # 23
3643
+ None, # 24
3644
+ None, # 25
3645
+ None, # 26
3646
+ None, # 27
3647
+ None, # 28
3648
+ None, # 29
3649
+ None, # 30
3650
+ None, # 31
3651
+ None, # 32
3652
+ None, # 33
3653
+ None, # 34
3654
+ None, # 35
3655
+ None, # 36
3656
+ None, # 37
3657
+ None, # 38
3658
+ None, # 39
3659
+ None, # 40
3660
+ None, # 41
3661
+ None, # 42
3662
+ None, # 43
3663
+ None, # 44
3664
+ None, # 45
3665
+ None, # 46
3666
+ None, # 47
3667
+ None, # 48
3668
+ None, # 49
3669
+ (50, TType.BOOL, 'useLongNames', None, None, ), # 50
3670
+ )
3671
+ all_structs.append(BootstrapPart)
3672
+ BootstrapPart.thrift_spec = (
3673
+ None, # 0
3674
+ (1, TType.STRUCT, 'metaData', [MetaData, None], None, ), # 1
3675
+ (2, TType.STRING, 'table', 'UTF8', None, ), # 2
3676
+ (3, TType.STRUCT, 'query', [Query, None], None, ), # 3
3677
+ (4, TType.LIST, 'keyColumns', (TType.STRING, 'UTF8', False), None, ), # 4
3678
+ )
3679
+ all_structs.append(LabelParts)
3680
+ LabelParts.thrift_spec = (
3681
+ None, # 0
3682
+ (1, TType.LIST, 'labels', (TType.STRUCT, [JoinPart, None], False), None, ), # 1
3683
+ (2, TType.I32, 'leftStartOffset', None, None, ), # 2
3684
+ (3, TType.I32, 'leftEndOffset', None, None, ), # 3
3685
+ (4, TType.STRUCT, 'metaData', [MetaData, None], None, ), # 4
3686
+ )
3687
+ all_structs.append(GroupByServingInfo)
3688
+ GroupByServingInfo.thrift_spec = (
3689
+ None, # 0
3690
+ (1, TType.STRUCT, 'groupBy', [GroupBy, None], None, ), # 1
3691
+ (2, TType.STRING, 'inputAvroSchema', 'UTF8', None, ), # 2
3692
+ (3, TType.STRING, 'selectedAvroSchema', 'UTF8', None, ), # 3
3693
+ (4, TType.STRING, 'keyAvroSchema', 'UTF8', None, ), # 4
3694
+ (5, TType.STRING, 'batchEndDate', 'UTF8', None, ), # 5
3695
+ (6, TType.STRING, 'dateFormat', 'UTF8', None, ), # 6
3696
+ )
3697
+ all_structs.append(DataField)
3698
+ DataField.thrift_spec = (
3699
+ None, # 0
3700
+ (1, TType.STRING, 'name', 'UTF8', None, ), # 1
3701
+ (2, TType.STRUCT, 'dataType', [TDataType, None], None, ), # 2
3702
+ )
3703
+ all_structs.append(TDataType)
3704
+ TDataType.thrift_spec = (
3705
+ None, # 0
3706
+ (1, TType.I32, 'kind', None, None, ), # 1
3707
+ (2, TType.LIST, 'params', (TType.STRUCT, [DataField, None], False), None, ), # 2
3708
+ (3, TType.STRING, 'name', 'UTF8', None, ), # 3
3709
+ )
3710
+ all_structs.append(DataSpec)
3711
+ DataSpec.thrift_spec = (
3712
+ None, # 0
3713
+ (1, TType.STRUCT, 'schema', [TDataType, None], None, ), # 1
3714
+ (2, TType.LIST, 'partitionColumns', (TType.STRING, 'UTF8', False), None, ), # 2
3715
+ (3, TType.I32, 'retentionDays', None, None, ), # 3
3716
+ (4, TType.MAP, 'props', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 4
3717
+ )
3718
+ all_structs.append(Model)
3719
+ Model.thrift_spec = (
3720
+ None, # 0
3721
+ (1, TType.STRUCT, 'metaData', [MetaData, None], None, ), # 1
3722
+ (2, TType.I32, 'modelType', None, None, ), # 2
3723
+ (3, TType.STRUCT, 'outputSchema', [TDataType, None], None, ), # 3
3724
+ (4, TType.STRUCT, 'source', [Source, None], None, ), # 4
3725
+ (5, TType.MAP, 'modelParams', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 5
3726
+ )
3727
+ all_structs.append(Team)
3728
+ Team.thrift_spec = (
3729
+ None, # 0
3730
+ (1, TType.STRING, 'name', 'UTF8', None, ), # 1
3731
+ (2, TType.STRING, 'description', 'UTF8', None, ), # 2
3732
+ (3, TType.STRING, 'email', 'UTF8', None, ), # 3
3733
+ None, # 4
3734
+ None, # 5
3735
+ None, # 6
3736
+ None, # 7
3737
+ None, # 8
3738
+ None, # 9
3739
+ (10, TType.STRING, 'outputNamespace', 'UTF8', None, ), # 10
3740
+ (11, TType.MAP, 'tableProperties', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 11
3741
+ None, # 12
3742
+ None, # 13
3743
+ None, # 14
3744
+ None, # 15
3745
+ None, # 16
3746
+ None, # 17
3747
+ None, # 18
3748
+ None, # 19
3749
+ (20, TType.STRUCT, 'env', [gen_thrift.common.ttypes.EnvironmentVariables, None], None, ), # 20
3750
+ (21, TType.STRUCT, 'conf', [gen_thrift.common.ttypes.ConfigProperties, None], None, ), # 21
3751
+ (22, TType.STRUCT, 'clusterConf', [gen_thrift.common.ttypes.ClusterConfigProperties, None], None, ), # 22
3752
+ )
3753
+ fix_spec(all_structs)
3754
+ del all_structs