awx-zipline-ai 0.2.1__py3-none-any.whl → 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of awx-zipline-ai might be problematic. Click here for more details.
- agent/ttypes.py +6 -6
- ai/chronon/airflow_helpers.py +20 -23
- ai/chronon/cli/__init__.py +0 -0
- ai/chronon/cli/compile/__init__.py +0 -0
- ai/chronon/cli/compile/column_hashing.py +40 -17
- ai/chronon/cli/compile/compile_context.py +13 -17
- ai/chronon/cli/compile/compiler.py +59 -36
- ai/chronon/cli/compile/conf_validator.py +251 -99
- ai/chronon/cli/compile/display/__init__.py +0 -0
- ai/chronon/cli/compile/display/class_tracker.py +6 -16
- ai/chronon/cli/compile/display/compile_status.py +10 -10
- ai/chronon/cli/compile/display/diff_result.py +79 -14
- ai/chronon/cli/compile/fill_templates.py +3 -8
- ai/chronon/cli/compile/parse_configs.py +10 -17
- ai/chronon/cli/compile/parse_teams.py +38 -34
- ai/chronon/cli/compile/serializer.py +3 -9
- ai/chronon/cli/compile/version_utils.py +42 -0
- ai/chronon/cli/git_utils.py +2 -13
- ai/chronon/cli/logger.py +0 -2
- ai/chronon/constants.py +1 -1
- ai/chronon/group_by.py +47 -47
- ai/chronon/join.py +46 -32
- ai/chronon/logger.py +1 -2
- ai/chronon/model.py +9 -4
- ai/chronon/query.py +2 -2
- ai/chronon/repo/__init__.py +1 -2
- ai/chronon/repo/aws.py +17 -31
- ai/chronon/repo/cluster.py +121 -50
- ai/chronon/repo/compile.py +14 -8
- ai/chronon/repo/constants.py +1 -1
- ai/chronon/repo/default_runner.py +32 -54
- ai/chronon/repo/explore.py +70 -73
- ai/chronon/repo/extract_objects.py +6 -9
- ai/chronon/repo/gcp.py +89 -88
- ai/chronon/repo/gitpython_utils.py +3 -2
- ai/chronon/repo/hub_runner.py +145 -55
- ai/chronon/repo/hub_uploader.py +2 -1
- ai/chronon/repo/init.py +12 -5
- ai/chronon/repo/join_backfill.py +19 -5
- ai/chronon/repo/run.py +42 -39
- ai/chronon/repo/serializer.py +4 -12
- ai/chronon/repo/utils.py +72 -63
- ai/chronon/repo/zipline.py +3 -19
- ai/chronon/repo/zipline_hub.py +211 -39
- ai/chronon/resources/__init__.py +0 -0
- ai/chronon/resources/gcp/__init__.py +0 -0
- ai/chronon/resources/gcp/group_bys/__init__.py +0 -0
- ai/chronon/resources/gcp/group_bys/test/data.py +13 -17
- ai/chronon/resources/gcp/joins/__init__.py +0 -0
- ai/chronon/resources/gcp/joins/test/data.py +4 -8
- ai/chronon/resources/gcp/sources/__init__.py +0 -0
- ai/chronon/resources/gcp/sources/test/data.py +9 -6
- ai/chronon/resources/gcp/teams.py +9 -21
- ai/chronon/source.py +2 -4
- ai/chronon/staging_query.py +60 -19
- ai/chronon/types.py +3 -2
- ai/chronon/utils.py +21 -68
- ai/chronon/windows.py +2 -4
- {awx_zipline_ai-0.2.1.dist-info → awx_zipline_ai-0.3.0.dist-info}/METADATA +47 -24
- awx_zipline_ai-0.3.0.dist-info/RECORD +96 -0
- awx_zipline_ai-0.3.0.dist-info/top_level.txt +4 -0
- gen_thrift/__init__.py +0 -0
- {ai/chronon → gen_thrift}/api/ttypes.py +327 -197
- {ai/chronon/api → gen_thrift}/common/ttypes.py +9 -39
- gen_thrift/eval/ttypes.py +660 -0
- {ai/chronon → gen_thrift}/hub/ttypes.py +12 -131
- {ai/chronon → gen_thrift}/observability/ttypes.py +343 -180
- {ai/chronon → gen_thrift}/planner/ttypes.py +326 -45
- ai/chronon/eval/__init__.py +0 -122
- ai/chronon/eval/query_parsing.py +0 -19
- ai/chronon/eval/sample_tables.py +0 -100
- ai/chronon/eval/table_scan.py +0 -186
- ai/chronon/orchestration/ttypes.py +0 -4406
- ai/chronon/resources/gcp/README.md +0 -174
- ai/chronon/resources/gcp/zipline-cli-install.sh +0 -54
- awx_zipline_ai-0.2.1.dist-info/RECORD +0 -93
- awx_zipline_ai-0.2.1.dist-info/licenses/LICENSE +0 -202
- awx_zipline_ai-0.2.1.dist-info/top_level.txt +0 -3
- /jars/__init__.py → /__init__.py +0 -0
- {awx_zipline_ai-0.2.1.dist-info → awx_zipline_ai-0.3.0.dist-info}/WHEEL +0 -0
- {awx_zipline_ai-0.2.1.dist-info → awx_zipline_ai-0.3.0.dist-info}/entry_points.txt +0 -0
- {ai/chronon → gen_thrift}/api/__init__.py +0 -0
- {ai/chronon/api/common → gen_thrift/api}/constants.py +0 -0
- {ai/chronon/api → gen_thrift}/common/__init__.py +0 -0
- {ai/chronon/api → gen_thrift/common}/constants.py +0 -0
- {ai/chronon/fetcher → gen_thrift/eval}/__init__.py +0 -0
- {ai/chronon/fetcher → gen_thrift/eval}/constants.py +0 -0
- {ai/chronon/hub → gen_thrift/fetcher}/__init__.py +0 -0
- {ai/chronon/hub → gen_thrift/fetcher}/constants.py +0 -0
- {ai/chronon → gen_thrift}/fetcher/ttypes.py +0 -0
- {ai/chronon/observability → gen_thrift/hub}/__init__.py +0 -0
- {ai/chronon/observability → gen_thrift/hub}/constants.py +0 -0
- {ai/chronon/orchestration → gen_thrift/observability}/__init__.py +0 -0
- {ai/chronon/orchestration → gen_thrift/observability}/constants.py +0 -0
- {ai/chronon → gen_thrift}/planner/__init__.py +0 -0
- {ai/chronon → gen_thrift}/planner/constants.py +0 -0
|
@@ -12,8 +12,8 @@ from thrift.TRecursive import fix_spec
|
|
|
12
12
|
from uuid import UUID
|
|
13
13
|
|
|
14
14
|
import sys
|
|
15
|
-
import
|
|
16
|
-
import
|
|
15
|
+
import gen_thrift.common.ttypes
|
|
16
|
+
import gen_thrift.api.ttypes
|
|
17
17
|
|
|
18
18
|
from thrift.transport import TTransport
|
|
19
19
|
all_structs = []
|
|
@@ -62,7 +62,7 @@ class SourceWithFilterNode(object):
|
|
|
62
62
|
break
|
|
63
63
|
if fid == 2:
|
|
64
64
|
if ftype == TType.STRUCT:
|
|
65
|
-
self.source =
|
|
65
|
+
self.source = gen_thrift.api.ttypes.Source()
|
|
66
66
|
self.source.read(iprot)
|
|
67
67
|
else:
|
|
68
68
|
iprot.skip(ftype)
|
|
@@ -149,7 +149,7 @@ class JoinBootstrapNode(object):
|
|
|
149
149
|
break
|
|
150
150
|
if fid == 2:
|
|
151
151
|
if ftype == TType.STRUCT:
|
|
152
|
-
self.join =
|
|
152
|
+
self.join = gen_thrift.api.ttypes.Join()
|
|
153
153
|
self.join.read(iprot)
|
|
154
154
|
else:
|
|
155
155
|
iprot.skip(ftype)
|
|
@@ -190,13 +190,15 @@ class JoinMergeNode(object):
|
|
|
190
190
|
"""
|
|
191
191
|
Attributes:
|
|
192
192
|
- join
|
|
193
|
+
- productionJoin
|
|
193
194
|
|
|
194
195
|
"""
|
|
195
196
|
thrift_spec = None
|
|
196
197
|
|
|
197
198
|
|
|
198
|
-
def __init__(self, join = None,):
|
|
199
|
+
def __init__(self, join = None, productionJoin = None,):
|
|
199
200
|
self.join = join
|
|
201
|
+
self.productionJoin = productionJoin
|
|
200
202
|
|
|
201
203
|
def read(self, iprot):
|
|
202
204
|
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
@@ -209,10 +211,16 @@ class JoinMergeNode(object):
|
|
|
209
211
|
break
|
|
210
212
|
if fid == 2:
|
|
211
213
|
if ftype == TType.STRUCT:
|
|
212
|
-
self.join =
|
|
214
|
+
self.join = gen_thrift.api.ttypes.Join()
|
|
213
215
|
self.join.read(iprot)
|
|
214
216
|
else:
|
|
215
217
|
iprot.skip(ftype)
|
|
218
|
+
elif fid == 3:
|
|
219
|
+
if ftype == TType.STRUCT:
|
|
220
|
+
self.productionJoin = gen_thrift.api.ttypes.Join()
|
|
221
|
+
self.productionJoin.read(iprot)
|
|
222
|
+
else:
|
|
223
|
+
iprot.skip(ftype)
|
|
216
224
|
else:
|
|
217
225
|
iprot.skip(ftype)
|
|
218
226
|
iprot.readFieldEnd()
|
|
@@ -228,6 +236,10 @@ class JoinMergeNode(object):
|
|
|
228
236
|
oprot.writeFieldBegin('join', TType.STRUCT, 2)
|
|
229
237
|
self.join.write(oprot)
|
|
230
238
|
oprot.writeFieldEnd()
|
|
239
|
+
if self.productionJoin is not None:
|
|
240
|
+
oprot.writeFieldBegin('productionJoin', TType.STRUCT, 3)
|
|
241
|
+
self.productionJoin.write(oprot)
|
|
242
|
+
oprot.writeFieldEnd()
|
|
231
243
|
oprot.writeFieldStop()
|
|
232
244
|
oprot.writeStructEnd()
|
|
233
245
|
|
|
@@ -269,7 +281,7 @@ class JoinDerivationNode(object):
|
|
|
269
281
|
break
|
|
270
282
|
if fid == 2:
|
|
271
283
|
if ftype == TType.STRUCT:
|
|
272
|
-
self.join =
|
|
284
|
+
self.join = gen_thrift.api.ttypes.Join()
|
|
273
285
|
self.join.read(iprot)
|
|
274
286
|
else:
|
|
275
287
|
iprot.skip(ftype)
|
|
@@ -313,16 +325,18 @@ class JoinPartNode(object):
|
|
|
313
325
|
- leftDataModel
|
|
314
326
|
- joinPart
|
|
315
327
|
- skewKeys
|
|
328
|
+
- productionJoinPart
|
|
316
329
|
|
|
317
330
|
"""
|
|
318
331
|
thrift_spec = None
|
|
319
332
|
|
|
320
333
|
|
|
321
|
-
def __init__(self, leftSourceTable = None, leftDataModel = None, joinPart = None, skewKeys = None,):
|
|
334
|
+
def __init__(self, leftSourceTable = None, leftDataModel = None, joinPart = None, skewKeys = None, productionJoinPart = None,):
|
|
322
335
|
self.leftSourceTable = leftSourceTable
|
|
323
336
|
self.leftDataModel = leftDataModel
|
|
324
337
|
self.joinPart = joinPart
|
|
325
338
|
self.skewKeys = skewKeys
|
|
339
|
+
self.productionJoinPart = productionJoinPart
|
|
326
340
|
|
|
327
341
|
def read(self, iprot):
|
|
328
342
|
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
@@ -345,7 +359,7 @@ class JoinPartNode(object):
|
|
|
345
359
|
iprot.skip(ftype)
|
|
346
360
|
elif fid == 4:
|
|
347
361
|
if ftype == TType.STRUCT:
|
|
348
|
-
self.joinPart =
|
|
362
|
+
self.joinPart = gen_thrift.api.ttypes.JoinPart()
|
|
349
363
|
self.joinPart.read(iprot)
|
|
350
364
|
else:
|
|
351
365
|
iprot.skip(ftype)
|
|
@@ -365,6 +379,12 @@ class JoinPartNode(object):
|
|
|
365
379
|
iprot.readMapEnd()
|
|
366
380
|
else:
|
|
367
381
|
iprot.skip(ftype)
|
|
382
|
+
elif fid == 6:
|
|
383
|
+
if ftype == TType.STRUCT:
|
|
384
|
+
self.productionJoinPart = gen_thrift.api.ttypes.JoinPart()
|
|
385
|
+
self.productionJoinPart.read(iprot)
|
|
386
|
+
else:
|
|
387
|
+
iprot.skip(ftype)
|
|
368
388
|
else:
|
|
369
389
|
iprot.skip(ftype)
|
|
370
390
|
iprot.readFieldEnd()
|
|
@@ -399,6 +419,10 @@ class JoinPartNode(object):
|
|
|
399
419
|
oprot.writeListEnd()
|
|
400
420
|
oprot.writeMapEnd()
|
|
401
421
|
oprot.writeFieldEnd()
|
|
422
|
+
if self.productionJoinPart is not None:
|
|
423
|
+
oprot.writeFieldBegin('productionJoinPart', TType.STRUCT, 6)
|
|
424
|
+
self.productionJoinPart.write(oprot)
|
|
425
|
+
oprot.writeFieldEnd()
|
|
402
426
|
oprot.writeFieldStop()
|
|
403
427
|
oprot.writeStructEnd()
|
|
404
428
|
|
|
@@ -440,7 +464,7 @@ class LabelJoinNode(object):
|
|
|
440
464
|
break
|
|
441
465
|
if fid == 2:
|
|
442
466
|
if ftype == TType.STRUCT:
|
|
443
|
-
self.join =
|
|
467
|
+
self.join = gen_thrift.api.ttypes.Join()
|
|
444
468
|
self.join.read(iprot)
|
|
445
469
|
else:
|
|
446
470
|
iprot.skip(ftype)
|
|
@@ -500,7 +524,7 @@ class MonolithJoinNode(object):
|
|
|
500
524
|
break
|
|
501
525
|
if fid == 1:
|
|
502
526
|
if ftype == TType.STRUCT:
|
|
503
|
-
self.join =
|
|
527
|
+
self.join = gen_thrift.api.ttypes.Join()
|
|
504
528
|
self.join.read(iprot)
|
|
505
529
|
else:
|
|
506
530
|
iprot.skip(ftype)
|
|
@@ -560,7 +584,7 @@ class StagingQueryNode(object):
|
|
|
560
584
|
break
|
|
561
585
|
if fid == 2:
|
|
562
586
|
if ftype == TType.STRUCT:
|
|
563
|
-
self.stagingQuery =
|
|
587
|
+
self.stagingQuery = gen_thrift.api.ttypes.StagingQuery()
|
|
564
588
|
self.stagingQuery.read(iprot)
|
|
565
589
|
else:
|
|
566
590
|
iprot.skip(ftype)
|
|
@@ -620,7 +644,7 @@ class GroupByBackfillNode(object):
|
|
|
620
644
|
break
|
|
621
645
|
if fid == 2:
|
|
622
646
|
if ftype == TType.STRUCT:
|
|
623
|
-
self.groupBy =
|
|
647
|
+
self.groupBy = gen_thrift.api.ttypes.GroupBy()
|
|
624
648
|
self.groupBy.read(iprot)
|
|
625
649
|
else:
|
|
626
650
|
iprot.skip(ftype)
|
|
@@ -680,7 +704,7 @@ class GroupByUploadNode(object):
|
|
|
680
704
|
break
|
|
681
705
|
if fid == 2:
|
|
682
706
|
if ftype == TType.STRUCT:
|
|
683
|
-
self.groupBy =
|
|
707
|
+
self.groupBy = gen_thrift.api.ttypes.GroupBy()
|
|
684
708
|
self.groupBy.read(iprot)
|
|
685
709
|
else:
|
|
686
710
|
iprot.skip(ftype)
|
|
@@ -740,7 +764,7 @@ class GroupByUploadToKVNode(object):
|
|
|
740
764
|
break
|
|
741
765
|
if fid == 2:
|
|
742
766
|
if ftype == TType.STRUCT:
|
|
743
|
-
self.groupBy =
|
|
767
|
+
self.groupBy = gen_thrift.api.ttypes.GroupBy()
|
|
744
768
|
self.groupBy.read(iprot)
|
|
745
769
|
else:
|
|
746
770
|
iprot.skip(ftype)
|
|
@@ -800,7 +824,7 @@ class GroupByStreamingNode(object):
|
|
|
800
824
|
break
|
|
801
825
|
if fid == 2:
|
|
802
826
|
if ftype == TType.STRUCT:
|
|
803
|
-
self.groupBy =
|
|
827
|
+
self.groupBy = gen_thrift.api.ttypes.GroupBy()
|
|
804
828
|
self.groupBy.read(iprot)
|
|
805
829
|
else:
|
|
806
830
|
iprot.skip(ftype)
|
|
@@ -860,7 +884,7 @@ class JoinMetadataUpload(object):
|
|
|
860
884
|
break
|
|
861
885
|
if fid == 2:
|
|
862
886
|
if ftype == TType.STRUCT:
|
|
863
|
-
self.join =
|
|
887
|
+
self.join = gen_thrift.api.ttypes.Join()
|
|
864
888
|
self.join.read(iprot)
|
|
865
889
|
else:
|
|
866
890
|
iprot.skip(ftype)
|
|
@@ -901,15 +925,19 @@ class ExternalSourceSensorNode(object):
|
|
|
901
925
|
"""
|
|
902
926
|
Attributes:
|
|
903
927
|
- metaData
|
|
904
|
-
-
|
|
928
|
+
- sourceTableDependency
|
|
929
|
+
- retryCount
|
|
930
|
+
- retryIntervalMin
|
|
905
931
|
|
|
906
932
|
"""
|
|
907
933
|
thrift_spec = None
|
|
908
934
|
|
|
909
935
|
|
|
910
|
-
def __init__(self, metaData = None,
|
|
936
|
+
def __init__(self, metaData = None, sourceTableDependency = None, retryCount = None, retryIntervalMin = None,):
|
|
911
937
|
self.metaData = metaData
|
|
912
|
-
self.
|
|
938
|
+
self.sourceTableDependency = sourceTableDependency
|
|
939
|
+
self.retryCount = retryCount
|
|
940
|
+
self.retryIntervalMin = retryIntervalMin
|
|
913
941
|
|
|
914
942
|
def read(self, iprot):
|
|
915
943
|
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
@@ -922,13 +950,24 @@ class ExternalSourceSensorNode(object):
|
|
|
922
950
|
break
|
|
923
951
|
if fid == 1:
|
|
924
952
|
if ftype == TType.STRUCT:
|
|
925
|
-
self.metaData =
|
|
953
|
+
self.metaData = gen_thrift.api.ttypes.MetaData()
|
|
926
954
|
self.metaData.read(iprot)
|
|
927
955
|
else:
|
|
928
956
|
iprot.skip(ftype)
|
|
929
957
|
elif fid == 2:
|
|
930
|
-
if ftype == TType.
|
|
931
|
-
self.
|
|
958
|
+
if ftype == TType.STRUCT:
|
|
959
|
+
self.sourceTableDependency = gen_thrift.common.ttypes.TableDependency()
|
|
960
|
+
self.sourceTableDependency.read(iprot)
|
|
961
|
+
else:
|
|
962
|
+
iprot.skip(ftype)
|
|
963
|
+
elif fid == 3:
|
|
964
|
+
if ftype == TType.I64:
|
|
965
|
+
self.retryCount = iprot.readI64()
|
|
966
|
+
else:
|
|
967
|
+
iprot.skip(ftype)
|
|
968
|
+
elif fid == 4:
|
|
969
|
+
if ftype == TType.I64:
|
|
970
|
+
self.retryIntervalMin = iprot.readI64()
|
|
932
971
|
else:
|
|
933
972
|
iprot.skip(ftype)
|
|
934
973
|
else:
|
|
@@ -946,9 +985,17 @@ class ExternalSourceSensorNode(object):
|
|
|
946
985
|
oprot.writeFieldBegin('metaData', TType.STRUCT, 1)
|
|
947
986
|
self.metaData.write(oprot)
|
|
948
987
|
oprot.writeFieldEnd()
|
|
949
|
-
if self.
|
|
950
|
-
oprot.writeFieldBegin('
|
|
951
|
-
|
|
988
|
+
if self.sourceTableDependency is not None:
|
|
989
|
+
oprot.writeFieldBegin('sourceTableDependency', TType.STRUCT, 2)
|
|
990
|
+
self.sourceTableDependency.write(oprot)
|
|
991
|
+
oprot.writeFieldEnd()
|
|
992
|
+
if self.retryCount is not None:
|
|
993
|
+
oprot.writeFieldBegin('retryCount', TType.I64, 3)
|
|
994
|
+
oprot.writeI64(self.retryCount)
|
|
995
|
+
oprot.writeFieldEnd()
|
|
996
|
+
if self.retryIntervalMin is not None:
|
|
997
|
+
oprot.writeFieldBegin('retryIntervalMin', TType.I64, 4)
|
|
998
|
+
oprot.writeI64(self.retryIntervalMin)
|
|
952
999
|
oprot.writeFieldEnd()
|
|
953
1000
|
oprot.writeFieldStop()
|
|
954
1001
|
oprot.writeStructEnd()
|
|
@@ -1103,7 +1150,7 @@ class NodeContent(object):
|
|
|
1103
1150
|
iprot.skip(ftype)
|
|
1104
1151
|
elif fid == 200:
|
|
1105
1152
|
if ftype == TType.STRUCT:
|
|
1106
|
-
self.stagingQueryBackfill =
|
|
1153
|
+
self.stagingQueryBackfill = gen_thrift.api.ttypes.StagingQuery()
|
|
1107
1154
|
self.stagingQueryBackfill.read(iprot)
|
|
1108
1155
|
else:
|
|
1109
1156
|
iprot.skip(ftype)
|
|
@@ -1225,7 +1272,7 @@ class Node(object):
|
|
|
1225
1272
|
break
|
|
1226
1273
|
if fid == 1:
|
|
1227
1274
|
if ftype == TType.STRUCT:
|
|
1228
|
-
self.metaData =
|
|
1275
|
+
self.metaData = gen_thrift.api.ttypes.MetaData()
|
|
1229
1276
|
self.metaData.read(iprot)
|
|
1230
1277
|
else:
|
|
1231
1278
|
iprot.skip(ftype)
|
|
@@ -1377,30 +1424,244 @@ class ConfPlan(object):
|
|
|
1377
1424
|
|
|
1378
1425
|
def __ne__(self, other):
|
|
1379
1426
|
return not (self == other)
|
|
1427
|
+
|
|
1428
|
+
|
|
1429
|
+
class NodeMeta(object):
|
|
1430
|
+
"""
|
|
1431
|
+
Attributes:
|
|
1432
|
+
- name
|
|
1433
|
+
- hash
|
|
1434
|
+
- deps
|
|
1435
|
+
- continuous
|
|
1436
|
+
- stepSize
|
|
1437
|
+
- outputTable
|
|
1438
|
+
- retries
|
|
1439
|
+
|
|
1440
|
+
"""
|
|
1441
|
+
thrift_spec = None
|
|
1442
|
+
|
|
1443
|
+
|
|
1444
|
+
def __init__(self, name = None, hash = None, deps = None, continuous = None, stepSize = None, outputTable = None, retries = None,):
|
|
1445
|
+
self.name = name
|
|
1446
|
+
self.hash = hash
|
|
1447
|
+
self.deps = deps
|
|
1448
|
+
self.continuous = continuous
|
|
1449
|
+
self.stepSize = stepSize
|
|
1450
|
+
self.outputTable = outputTable
|
|
1451
|
+
self.retries = retries
|
|
1452
|
+
|
|
1453
|
+
def read(self, iprot):
|
|
1454
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
1455
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
1456
|
+
return
|
|
1457
|
+
iprot.readStructBegin()
|
|
1458
|
+
while True:
|
|
1459
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
1460
|
+
if ftype == TType.STOP:
|
|
1461
|
+
break
|
|
1462
|
+
if fid == 1:
|
|
1463
|
+
if ftype == TType.STRING:
|
|
1464
|
+
self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1465
|
+
else:
|
|
1466
|
+
iprot.skip(ftype)
|
|
1467
|
+
elif fid == 2:
|
|
1468
|
+
if ftype == TType.STRING:
|
|
1469
|
+
self.hash = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1470
|
+
else:
|
|
1471
|
+
iprot.skip(ftype)
|
|
1472
|
+
elif fid == 3:
|
|
1473
|
+
if ftype == TType.LIST:
|
|
1474
|
+
self.deps = []
|
|
1475
|
+
(_etype51, _size48) = iprot.readListBegin()
|
|
1476
|
+
for _i52 in range(_size48):
|
|
1477
|
+
_elem53 = gen_thrift.common.ttypes.TableDependency()
|
|
1478
|
+
_elem53.read(iprot)
|
|
1479
|
+
self.deps.append(_elem53)
|
|
1480
|
+
iprot.readListEnd()
|
|
1481
|
+
else:
|
|
1482
|
+
iprot.skip(ftype)
|
|
1483
|
+
elif fid == 4:
|
|
1484
|
+
if ftype == TType.BOOL:
|
|
1485
|
+
self.continuous = iprot.readBool()
|
|
1486
|
+
else:
|
|
1487
|
+
iprot.skip(ftype)
|
|
1488
|
+
elif fid == 5:
|
|
1489
|
+
if ftype == TType.I32:
|
|
1490
|
+
self.stepSize = iprot.readI32()
|
|
1491
|
+
else:
|
|
1492
|
+
iprot.skip(ftype)
|
|
1493
|
+
elif fid == 6:
|
|
1494
|
+
if ftype == TType.STRING:
|
|
1495
|
+
self.outputTable = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1496
|
+
else:
|
|
1497
|
+
iprot.skip(ftype)
|
|
1498
|
+
elif fid == 7:
|
|
1499
|
+
if ftype == TType.I32:
|
|
1500
|
+
self.retries = iprot.readI32()
|
|
1501
|
+
else:
|
|
1502
|
+
iprot.skip(ftype)
|
|
1503
|
+
else:
|
|
1504
|
+
iprot.skip(ftype)
|
|
1505
|
+
iprot.readFieldEnd()
|
|
1506
|
+
iprot.readStructEnd()
|
|
1507
|
+
|
|
1508
|
+
def write(self, oprot):
|
|
1509
|
+
self.validate()
|
|
1510
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
1511
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
1512
|
+
return
|
|
1513
|
+
oprot.writeStructBegin('NodeMeta')
|
|
1514
|
+
if self.name is not None:
|
|
1515
|
+
oprot.writeFieldBegin('name', TType.STRING, 1)
|
|
1516
|
+
oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name)
|
|
1517
|
+
oprot.writeFieldEnd()
|
|
1518
|
+
if self.hash is not None:
|
|
1519
|
+
oprot.writeFieldBegin('hash', TType.STRING, 2)
|
|
1520
|
+
oprot.writeString(self.hash.encode('utf-8') if sys.version_info[0] == 2 else self.hash)
|
|
1521
|
+
oprot.writeFieldEnd()
|
|
1522
|
+
if self.deps is not None:
|
|
1523
|
+
oprot.writeFieldBegin('deps', TType.LIST, 3)
|
|
1524
|
+
oprot.writeListBegin(TType.STRUCT, len(self.deps))
|
|
1525
|
+
for iter54 in self.deps:
|
|
1526
|
+
iter54.write(oprot)
|
|
1527
|
+
oprot.writeListEnd()
|
|
1528
|
+
oprot.writeFieldEnd()
|
|
1529
|
+
if self.continuous is not None:
|
|
1530
|
+
oprot.writeFieldBegin('continuous', TType.BOOL, 4)
|
|
1531
|
+
oprot.writeBool(self.continuous)
|
|
1532
|
+
oprot.writeFieldEnd()
|
|
1533
|
+
if self.stepSize is not None:
|
|
1534
|
+
oprot.writeFieldBegin('stepSize', TType.I32, 5)
|
|
1535
|
+
oprot.writeI32(self.stepSize)
|
|
1536
|
+
oprot.writeFieldEnd()
|
|
1537
|
+
if self.outputTable is not None:
|
|
1538
|
+
oprot.writeFieldBegin('outputTable', TType.STRING, 6)
|
|
1539
|
+
oprot.writeString(self.outputTable.encode('utf-8') if sys.version_info[0] == 2 else self.outputTable)
|
|
1540
|
+
oprot.writeFieldEnd()
|
|
1541
|
+
if self.retries is not None:
|
|
1542
|
+
oprot.writeFieldBegin('retries', TType.I32, 7)
|
|
1543
|
+
oprot.writeI32(self.retries)
|
|
1544
|
+
oprot.writeFieldEnd()
|
|
1545
|
+
oprot.writeFieldStop()
|
|
1546
|
+
oprot.writeStructEnd()
|
|
1547
|
+
|
|
1548
|
+
def validate(self):
|
|
1549
|
+
return
|
|
1550
|
+
|
|
1551
|
+
def __repr__(self):
|
|
1552
|
+
L = ['%s=%r' % (key, value)
|
|
1553
|
+
for key, value in self.__dict__.items()]
|
|
1554
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
1555
|
+
|
|
1556
|
+
def __eq__(self, other):
|
|
1557
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
1558
|
+
|
|
1559
|
+
def __ne__(self, other):
|
|
1560
|
+
return not (self == other)
|
|
1561
|
+
|
|
1562
|
+
|
|
1563
|
+
class WorkflowPlan(object):
|
|
1564
|
+
"""
|
|
1565
|
+
Attributes:
|
|
1566
|
+
- nodes
|
|
1567
|
+
- terminalNode
|
|
1568
|
+
|
|
1569
|
+
"""
|
|
1570
|
+
thrift_spec = None
|
|
1571
|
+
|
|
1572
|
+
|
|
1573
|
+
def __init__(self, nodes = None, terminalNode = None,):
|
|
1574
|
+
self.nodes = nodes
|
|
1575
|
+
self.terminalNode = terminalNode
|
|
1576
|
+
|
|
1577
|
+
def read(self, iprot):
|
|
1578
|
+
if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None:
|
|
1579
|
+
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
|
|
1580
|
+
return
|
|
1581
|
+
iprot.readStructBegin()
|
|
1582
|
+
while True:
|
|
1583
|
+
(fname, ftype, fid) = iprot.readFieldBegin()
|
|
1584
|
+
if ftype == TType.STOP:
|
|
1585
|
+
break
|
|
1586
|
+
if fid == 1:
|
|
1587
|
+
if ftype == TType.LIST:
|
|
1588
|
+
self.nodes = []
|
|
1589
|
+
(_etype58, _size55) = iprot.readListBegin()
|
|
1590
|
+
for _i59 in range(_size55):
|
|
1591
|
+
_elem60 = NodeMeta()
|
|
1592
|
+
_elem60.read(iprot)
|
|
1593
|
+
self.nodes.append(_elem60)
|
|
1594
|
+
iprot.readListEnd()
|
|
1595
|
+
else:
|
|
1596
|
+
iprot.skip(ftype)
|
|
1597
|
+
elif fid == 2:
|
|
1598
|
+
if ftype == TType.STRING:
|
|
1599
|
+
self.terminalNode = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString()
|
|
1600
|
+
else:
|
|
1601
|
+
iprot.skip(ftype)
|
|
1602
|
+
else:
|
|
1603
|
+
iprot.skip(ftype)
|
|
1604
|
+
iprot.readFieldEnd()
|
|
1605
|
+
iprot.readStructEnd()
|
|
1606
|
+
|
|
1607
|
+
def write(self, oprot):
|
|
1608
|
+
self.validate()
|
|
1609
|
+
if oprot._fast_encode is not None and self.thrift_spec is not None:
|
|
1610
|
+
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
|
|
1611
|
+
return
|
|
1612
|
+
oprot.writeStructBegin('WorkflowPlan')
|
|
1613
|
+
if self.nodes is not None:
|
|
1614
|
+
oprot.writeFieldBegin('nodes', TType.LIST, 1)
|
|
1615
|
+
oprot.writeListBegin(TType.STRUCT, len(self.nodes))
|
|
1616
|
+
for iter61 in self.nodes:
|
|
1617
|
+
iter61.write(oprot)
|
|
1618
|
+
oprot.writeListEnd()
|
|
1619
|
+
oprot.writeFieldEnd()
|
|
1620
|
+
if self.terminalNode is not None:
|
|
1621
|
+
oprot.writeFieldBegin('terminalNode', TType.STRING, 2)
|
|
1622
|
+
oprot.writeString(self.terminalNode.encode('utf-8') if sys.version_info[0] == 2 else self.terminalNode)
|
|
1623
|
+
oprot.writeFieldEnd()
|
|
1624
|
+
oprot.writeFieldStop()
|
|
1625
|
+
oprot.writeStructEnd()
|
|
1626
|
+
|
|
1627
|
+
def validate(self):
|
|
1628
|
+
return
|
|
1629
|
+
|
|
1630
|
+
def __repr__(self):
|
|
1631
|
+
L = ['%s=%r' % (key, value)
|
|
1632
|
+
for key, value in self.__dict__.items()]
|
|
1633
|
+
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
|
|
1634
|
+
|
|
1635
|
+
def __eq__(self, other):
|
|
1636
|
+
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
|
|
1637
|
+
|
|
1638
|
+
def __ne__(self, other):
|
|
1639
|
+
return not (self == other)
|
|
1380
1640
|
all_structs.append(SourceWithFilterNode)
|
|
1381
1641
|
SourceWithFilterNode.thrift_spec = (
|
|
1382
1642
|
None, # 0
|
|
1383
1643
|
None, # 1
|
|
1384
|
-
(2, TType.STRUCT, 'source', [
|
|
1644
|
+
(2, TType.STRUCT, 'source', [gen_thrift.api.ttypes.Source, None], None, ), # 2
|
|
1385
1645
|
(3, TType.MAP, 'excludeKeys', (TType.STRING, 'UTF8', TType.LIST, (TType.STRING, 'UTF8', False), False), None, ), # 3
|
|
1386
1646
|
)
|
|
1387
1647
|
all_structs.append(JoinBootstrapNode)
|
|
1388
1648
|
JoinBootstrapNode.thrift_spec = (
|
|
1389
1649
|
None, # 0
|
|
1390
1650
|
None, # 1
|
|
1391
|
-
(2, TType.STRUCT, 'join', [
|
|
1651
|
+
(2, TType.STRUCT, 'join', [gen_thrift.api.ttypes.Join, None], None, ), # 2
|
|
1392
1652
|
)
|
|
1393
1653
|
all_structs.append(JoinMergeNode)
|
|
1394
1654
|
JoinMergeNode.thrift_spec = (
|
|
1395
1655
|
None, # 0
|
|
1396
1656
|
None, # 1
|
|
1397
|
-
(2, TType.STRUCT, 'join', [
|
|
1657
|
+
(2, TType.STRUCT, 'join', [gen_thrift.api.ttypes.Join, None], None, ), # 2
|
|
1658
|
+
(3, TType.STRUCT, 'productionJoin', [gen_thrift.api.ttypes.Join, None], None, ), # 3
|
|
1398
1659
|
)
|
|
1399
1660
|
all_structs.append(JoinDerivationNode)
|
|
1400
1661
|
JoinDerivationNode.thrift_spec = (
|
|
1401
1662
|
None, # 0
|
|
1402
1663
|
None, # 1
|
|
1403
|
-
(2, TType.STRUCT, 'join', [
|
|
1664
|
+
(2, TType.STRUCT, 'join', [gen_thrift.api.ttypes.Join, None], None, ), # 2
|
|
1404
1665
|
)
|
|
1405
1666
|
all_structs.append(JoinPartNode)
|
|
1406
1667
|
JoinPartNode.thrift_spec = (
|
|
@@ -1408,61 +1669,64 @@ JoinPartNode.thrift_spec = (
|
|
|
1408
1669
|
None, # 1
|
|
1409
1670
|
(2, TType.STRING, 'leftSourceTable', 'UTF8', None, ), # 2
|
|
1410
1671
|
(3, TType.I32, 'leftDataModel', None, None, ), # 3
|
|
1411
|
-
(4, TType.STRUCT, 'joinPart', [
|
|
1672
|
+
(4, TType.STRUCT, 'joinPart', [gen_thrift.api.ttypes.JoinPart, None], None, ), # 4
|
|
1412
1673
|
(5, TType.MAP, 'skewKeys', (TType.STRING, 'UTF8', TType.LIST, (TType.STRING, 'UTF8', False), False), None, ), # 5
|
|
1674
|
+
(6, TType.STRUCT, 'productionJoinPart', [gen_thrift.api.ttypes.JoinPart, None], None, ), # 6
|
|
1413
1675
|
)
|
|
1414
1676
|
all_structs.append(LabelJoinNode)
|
|
1415
1677
|
LabelJoinNode.thrift_spec = (
|
|
1416
1678
|
None, # 0
|
|
1417
1679
|
None, # 1
|
|
1418
|
-
(2, TType.STRUCT, 'join', [
|
|
1680
|
+
(2, TType.STRUCT, 'join', [gen_thrift.api.ttypes.Join, None], None, ), # 2
|
|
1419
1681
|
)
|
|
1420
1682
|
all_structs.append(MonolithJoinNode)
|
|
1421
1683
|
MonolithJoinNode.thrift_spec = (
|
|
1422
1684
|
None, # 0
|
|
1423
|
-
(1, TType.STRUCT, 'join', [
|
|
1685
|
+
(1, TType.STRUCT, 'join', [gen_thrift.api.ttypes.Join, None], None, ), # 1
|
|
1424
1686
|
)
|
|
1425
1687
|
all_structs.append(StagingQueryNode)
|
|
1426
1688
|
StagingQueryNode.thrift_spec = (
|
|
1427
1689
|
None, # 0
|
|
1428
1690
|
None, # 1
|
|
1429
|
-
(2, TType.STRUCT, 'stagingQuery', [
|
|
1691
|
+
(2, TType.STRUCT, 'stagingQuery', [gen_thrift.api.ttypes.StagingQuery, None], None, ), # 2
|
|
1430
1692
|
)
|
|
1431
1693
|
all_structs.append(GroupByBackfillNode)
|
|
1432
1694
|
GroupByBackfillNode.thrift_spec = (
|
|
1433
1695
|
None, # 0
|
|
1434
1696
|
None, # 1
|
|
1435
|
-
(2, TType.STRUCT, 'groupBy', [
|
|
1697
|
+
(2, TType.STRUCT, 'groupBy', [gen_thrift.api.ttypes.GroupBy, None], None, ), # 2
|
|
1436
1698
|
)
|
|
1437
1699
|
all_structs.append(GroupByUploadNode)
|
|
1438
1700
|
GroupByUploadNode.thrift_spec = (
|
|
1439
1701
|
None, # 0
|
|
1440
1702
|
None, # 1
|
|
1441
|
-
(2, TType.STRUCT, 'groupBy', [
|
|
1703
|
+
(2, TType.STRUCT, 'groupBy', [gen_thrift.api.ttypes.GroupBy, None], None, ), # 2
|
|
1442
1704
|
)
|
|
1443
1705
|
all_structs.append(GroupByUploadToKVNode)
|
|
1444
1706
|
GroupByUploadToKVNode.thrift_spec = (
|
|
1445
1707
|
None, # 0
|
|
1446
1708
|
None, # 1
|
|
1447
|
-
(2, TType.STRUCT, 'groupBy', [
|
|
1709
|
+
(2, TType.STRUCT, 'groupBy', [gen_thrift.api.ttypes.GroupBy, None], None, ), # 2
|
|
1448
1710
|
)
|
|
1449
1711
|
all_structs.append(GroupByStreamingNode)
|
|
1450
1712
|
GroupByStreamingNode.thrift_spec = (
|
|
1451
1713
|
None, # 0
|
|
1452
1714
|
None, # 1
|
|
1453
|
-
(2, TType.STRUCT, 'groupBy', [
|
|
1715
|
+
(2, TType.STRUCT, 'groupBy', [gen_thrift.api.ttypes.GroupBy, None], None, ), # 2
|
|
1454
1716
|
)
|
|
1455
1717
|
all_structs.append(JoinMetadataUpload)
|
|
1456
1718
|
JoinMetadataUpload.thrift_spec = (
|
|
1457
1719
|
None, # 0
|
|
1458
1720
|
None, # 1
|
|
1459
|
-
(2, TType.STRUCT, 'join', [
|
|
1721
|
+
(2, TType.STRUCT, 'join', [gen_thrift.api.ttypes.Join, None], None, ), # 2
|
|
1460
1722
|
)
|
|
1461
1723
|
all_structs.append(ExternalSourceSensorNode)
|
|
1462
1724
|
ExternalSourceSensorNode.thrift_spec = (
|
|
1463
1725
|
None, # 0
|
|
1464
|
-
(1, TType.STRUCT, 'metaData', [
|
|
1465
|
-
(2, TType.
|
|
1726
|
+
(1, TType.STRUCT, 'metaData', [gen_thrift.api.ttypes.MetaData, None], None, ), # 1
|
|
1727
|
+
(2, TType.STRUCT, 'sourceTableDependency', [gen_thrift.common.ttypes.TableDependency, None], None, ), # 2
|
|
1728
|
+
(3, TType.I64, 'retryCount', None, None, ), # 3
|
|
1729
|
+
(4, TType.I64, 'retryIntervalMin', None, None, ), # 4
|
|
1466
1730
|
)
|
|
1467
1731
|
all_structs.append(NodeContent)
|
|
1468
1732
|
NodeContent.thrift_spec = (
|
|
@@ -1666,12 +1930,12 @@ NodeContent.thrift_spec = (
|
|
|
1666
1930
|
None, # 197
|
|
1667
1931
|
None, # 198
|
|
1668
1932
|
None, # 199
|
|
1669
|
-
(200, TType.STRUCT, 'stagingQueryBackfill', [
|
|
1933
|
+
(200, TType.STRUCT, 'stagingQueryBackfill', [gen_thrift.api.ttypes.StagingQuery, None], None, ), # 200
|
|
1670
1934
|
)
|
|
1671
1935
|
all_structs.append(Node)
|
|
1672
1936
|
Node.thrift_spec = (
|
|
1673
1937
|
None, # 0
|
|
1674
|
-
(1, TType.STRUCT, 'metaData', [
|
|
1938
|
+
(1, TType.STRUCT, 'metaData', [gen_thrift.api.ttypes.MetaData, None], None, ), # 1
|
|
1675
1939
|
(2, TType.STRUCT, 'content', [NodeContent, None], None, ), # 2
|
|
1676
1940
|
(3, TType.STRING, 'semanticHash', 'UTF8', None, ), # 3
|
|
1677
1941
|
(4, TType.BOOL, 'isLongRunning', None, None, ), # 4
|
|
@@ -1682,5 +1946,22 @@ ConfPlan.thrift_spec = (
|
|
|
1682
1946
|
(1, TType.LIST, 'nodes', (TType.STRUCT, [Node, None], False), None, ), # 1
|
|
1683
1947
|
(2, TType.MAP, 'terminalNodeNames', (TType.I32, None, TType.STRING, 'UTF8', False), None, ), # 2
|
|
1684
1948
|
)
|
|
1949
|
+
all_structs.append(NodeMeta)
|
|
1950
|
+
NodeMeta.thrift_spec = (
|
|
1951
|
+
None, # 0
|
|
1952
|
+
(1, TType.STRING, 'name', 'UTF8', None, ), # 1
|
|
1953
|
+
(2, TType.STRING, 'hash', 'UTF8', None, ), # 2
|
|
1954
|
+
(3, TType.LIST, 'deps', (TType.STRUCT, [gen_thrift.common.ttypes.TableDependency, None], False), None, ), # 3
|
|
1955
|
+
(4, TType.BOOL, 'continuous', None, None, ), # 4
|
|
1956
|
+
(5, TType.I32, 'stepSize', None, None, ), # 5
|
|
1957
|
+
(6, TType.STRING, 'outputTable', 'UTF8', None, ), # 6
|
|
1958
|
+
(7, TType.I32, 'retries', None, None, ), # 7
|
|
1959
|
+
)
|
|
1960
|
+
all_structs.append(WorkflowPlan)
|
|
1961
|
+
WorkflowPlan.thrift_spec = (
|
|
1962
|
+
None, # 0
|
|
1963
|
+
(1, TType.LIST, 'nodes', (TType.STRUCT, [NodeMeta, None], False), None, ), # 1
|
|
1964
|
+
(2, TType.STRING, 'terminalNode', 'UTF8', None, ), # 2
|
|
1965
|
+
)
|
|
1685
1966
|
fix_spec(all_structs)
|
|
1686
1967
|
del all_structs
|