metaflow 2.17.1__py2.py3-none-any.whl → 2.17.2__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow/cli_components/run_cmds.py +15 -0
- metaflow/flowspec.py +91 -1
- metaflow/graph.py +152 -13
- metaflow/lint.py +66 -3
- metaflow/plugins/argo/argo_workflows.py +5 -0
- metaflow/plugins/aws/step_functions/step_functions.py +6 -0
- metaflow/plugins/cards/card_modules/basic.py +14 -2
- metaflow/plugins/cards/card_modules/main.css +1 -0
- metaflow/plugins/cards/card_modules/main.js +28 -28
- metaflow/runtime.py +57 -14
- metaflow/task.py +62 -34
- metaflow/user_decorators/user_step_decorator.py +24 -5
- metaflow/version.py +1 -1
- {metaflow-2.17.1.dist-info → metaflow-2.17.2.dist-info}/METADATA +2 -2
- {metaflow-2.17.1.dist-info → metaflow-2.17.2.dist-info}/RECORD +22 -21
- {metaflow-2.17.1.data → metaflow-2.17.2.data}/data/share/metaflow/devtools/Makefile +0 -0
- {metaflow-2.17.1.data → metaflow-2.17.2.data}/data/share/metaflow/devtools/Tiltfile +0 -0
- {metaflow-2.17.1.data → metaflow-2.17.2.data}/data/share/metaflow/devtools/pick_services.sh +0 -0
- {metaflow-2.17.1.dist-info → metaflow-2.17.2.dist-info}/WHEEL +0 -0
- {metaflow-2.17.1.dist-info → metaflow-2.17.2.dist-info}/entry_points.txt +0 -0
- {metaflow-2.17.1.dist-info → metaflow-2.17.2.dist-info}/licenses/LICENSE +0 -0
- {metaflow-2.17.1.dist-info → metaflow-2.17.2.dist-info}/top_level.txt +0 -0
@@ -13,6 +13,8 @@ from ..package import MetaflowPackage
|
|
13
13
|
from ..runtime import NativeRuntime
|
14
14
|
from ..system import _system_logger
|
15
15
|
|
16
|
+
# from ..client.core import Run
|
17
|
+
|
16
18
|
from ..tagging_util import validate_tags
|
17
19
|
from ..util import get_latest_run_id, write_latest_run_id
|
18
20
|
|
@@ -230,6 +232,19 @@ def resume(
|
|
230
232
|
step_to_rerun, ",".join(list(obj.graph.nodes.keys()))
|
231
233
|
)
|
232
234
|
)
|
235
|
+
|
236
|
+
## TODO: instead of checking execution path here, can add a warning later
|
237
|
+
## instead of throwing an error. This is for resuming a step which was not
|
238
|
+
## taken inside a branch i.e. not present in the execution path.
|
239
|
+
|
240
|
+
# origin_run = Run(f"{obj.flow.name}/{origin_run_id}", _namespace_check=False)
|
241
|
+
# executed_steps = {step.path_components[-1] for step in origin_run}
|
242
|
+
# if step_to_rerun not in executed_steps:
|
243
|
+
# raise CommandException(
|
244
|
+
# f"Cannot resume from step '{step_to_rerun}'. This step was not "
|
245
|
+
# f"part of the original execution path for run '{origin_run_id}'."
|
246
|
+
# )
|
247
|
+
|
233
248
|
steps_to_rerun = {step_to_rerun}
|
234
249
|
|
235
250
|
if run_id:
|
metaflow/flowspec.py
CHANGED
@@ -812,6 +812,15 @@ class FlowSpec(metaclass=FlowSpecMeta):
|
|
812
812
|
evaluates to an iterator. A task will be launched for each value in the iterator and
|
813
813
|
each task will execute the code specified by the step `foreach_step`.
|
814
814
|
|
815
|
+
- Switch statement:
|
816
|
+
```
|
817
|
+
self.next({"case1": self.step_a, "case2": self.step_b}, condition='condition_variable')
|
818
|
+
```
|
819
|
+
In this situation, `step_a` and `step_b` are methods in the current class decorated
|
820
|
+
with the `@step` decorator and `condition_variable` is a variable name in the current
|
821
|
+
class. The value of the condition variable determines which step to execute. If the
|
822
|
+
value doesn't match any of the dictionary keys, a RuntimeError is raised.
|
823
|
+
|
815
824
|
Parameters
|
816
825
|
----------
|
817
826
|
dsts : Callable[..., None]
|
@@ -827,6 +836,7 @@ class FlowSpec(metaclass=FlowSpecMeta):
|
|
827
836
|
|
828
837
|
foreach = kwargs.pop("foreach", None)
|
829
838
|
num_parallel = kwargs.pop("num_parallel", None)
|
839
|
+
condition = kwargs.pop("condition", None)
|
830
840
|
if kwargs:
|
831
841
|
kw = next(iter(kwargs))
|
832
842
|
msg = (
|
@@ -843,6 +853,86 @@ class FlowSpec(metaclass=FlowSpecMeta):
|
|
843
853
|
)
|
844
854
|
raise InvalidNextException(msg)
|
845
855
|
|
856
|
+
# check: switch case using condition
|
857
|
+
if condition is not None:
|
858
|
+
if len(dsts) != 1 or not isinstance(dsts[0], dict) or not dsts[0]:
|
859
|
+
msg = (
|
860
|
+
"Step *{step}* has an invalid self.next() transition. "
|
861
|
+
"When using 'condition', the transition must be to a single, "
|
862
|
+
"non-empty dictionary mapping condition values to step methods.".format(
|
863
|
+
step=step
|
864
|
+
)
|
865
|
+
)
|
866
|
+
raise InvalidNextException(msg)
|
867
|
+
|
868
|
+
if not isinstance(condition, basestring):
|
869
|
+
msg = (
|
870
|
+
"Step *{step}* has an invalid self.next() transition. "
|
871
|
+
"The argument to 'condition' must be a string.".format(step=step)
|
872
|
+
)
|
873
|
+
raise InvalidNextException(msg)
|
874
|
+
|
875
|
+
if foreach is not None or num_parallel is not None:
|
876
|
+
msg = (
|
877
|
+
"Step *{step}* has an invalid self.next() transition. "
|
878
|
+
"Switch statements cannot be combined with foreach or num_parallel.".format(
|
879
|
+
step=step
|
880
|
+
)
|
881
|
+
)
|
882
|
+
raise InvalidNextException(msg)
|
883
|
+
|
884
|
+
switch_cases = dsts[0]
|
885
|
+
|
886
|
+
# Validate that condition variable exists
|
887
|
+
try:
|
888
|
+
condition_value = getattr(self, condition)
|
889
|
+
except AttributeError:
|
890
|
+
msg = (
|
891
|
+
"Condition variable *self.{var}* in step *{step}* "
|
892
|
+
"does not exist. Make sure you set self.{var} in this step.".format(
|
893
|
+
step=step, var=condition
|
894
|
+
)
|
895
|
+
)
|
896
|
+
raise InvalidNextException(msg)
|
897
|
+
|
898
|
+
if condition_value not in switch_cases:
|
899
|
+
available_cases = list(switch_cases.keys())
|
900
|
+
raise RuntimeError(
|
901
|
+
f"Switch condition variable '{condition}' has value '{condition_value}' "
|
902
|
+
f"which is not in the available cases: {available_cases}"
|
903
|
+
)
|
904
|
+
|
905
|
+
# Get the chosen step and set transition directly
|
906
|
+
chosen_step_func = switch_cases[condition_value]
|
907
|
+
|
908
|
+
# Validate that the chosen step exists
|
909
|
+
try:
|
910
|
+
name = chosen_step_func.__func__.__name__
|
911
|
+
except:
|
912
|
+
msg = (
|
913
|
+
"Step *{step}* specifies a switch transition that is not a function. "
|
914
|
+
"Make sure the value in the dictionary is a method "
|
915
|
+
"of the Flow class.".format(step=step)
|
916
|
+
)
|
917
|
+
raise InvalidNextException(msg)
|
918
|
+
if not hasattr(self, name):
|
919
|
+
msg = (
|
920
|
+
"Step *{step}* specifies a switch transition to an "
|
921
|
+
"unknown step, *{name}*.".format(step=step, name=name)
|
922
|
+
)
|
923
|
+
raise InvalidNextException(msg)
|
924
|
+
|
925
|
+
self._transition = ([name], None)
|
926
|
+
return
|
927
|
+
|
928
|
+
# Check for an invalid transition: a dictionary used without a 'condition' parameter.
|
929
|
+
if len(dsts) == 1 and isinstance(dsts[0], dict):
|
930
|
+
msg = (
|
931
|
+
"Step *{step}* has an invalid self.next() transition. "
|
932
|
+
"Dictionary argument requires 'condition' parameter.".format(step=step)
|
933
|
+
)
|
934
|
+
raise InvalidNextException(msg)
|
935
|
+
|
846
936
|
# check: all destinations are methods of this object
|
847
937
|
funcs = []
|
848
938
|
for i, dst in enumerate(dsts):
|
@@ -933,7 +1023,7 @@ class FlowSpec(metaclass=FlowSpecMeta):
|
|
933
1023
|
self._foreach_var = foreach
|
934
1024
|
|
935
1025
|
# check: non-keyword transitions are valid
|
936
|
-
if foreach is None:
|
1026
|
+
if foreach is None and condition is None:
|
937
1027
|
if len(dsts) < 1:
|
938
1028
|
msg = (
|
939
1029
|
"Step *{step}* has an invalid self.next() transition. "
|
metaflow/graph.py
CHANGED
@@ -68,6 +68,8 @@ class DAGNode(object):
|
|
68
68
|
self.has_tail_next = False
|
69
69
|
self.invalid_tail_next = False
|
70
70
|
self.num_args = 0
|
71
|
+
self.switch_cases = {}
|
72
|
+
self.condition = None
|
71
73
|
self.foreach_param = None
|
72
74
|
self.num_parallel = 0
|
73
75
|
self.parallel_foreach = False
|
@@ -76,6 +78,7 @@ class DAGNode(object):
|
|
76
78
|
# these attributes are populated by _traverse_graph
|
77
79
|
self.in_funcs = set()
|
78
80
|
self.split_parents = []
|
81
|
+
self.split_branches = []
|
79
82
|
self.matching_join = None
|
80
83
|
# these attributes are populated by _postprocess
|
81
84
|
self.is_inside_foreach = False
|
@@ -83,6 +86,56 @@ class DAGNode(object):
|
|
83
86
|
def _expr_str(self, expr):
|
84
87
|
return "%s.%s" % (expr.value.id, expr.attr)
|
85
88
|
|
89
|
+
def _parse_switch_dict(self, dict_node):
|
90
|
+
switch_cases = {}
|
91
|
+
|
92
|
+
if isinstance(dict_node, ast.Dict):
|
93
|
+
for key, value in zip(dict_node.keys, dict_node.values):
|
94
|
+
case_key = None
|
95
|
+
|
96
|
+
# handle string literals
|
97
|
+
if isinstance(key, ast.Str):
|
98
|
+
case_key = key.s
|
99
|
+
elif isinstance(key, ast.Constant):
|
100
|
+
case_key = key.value
|
101
|
+
elif isinstance(key, ast.Attribute):
|
102
|
+
if isinstance(key.value, ast.Attribute) and isinstance(
|
103
|
+
key.value.value, ast.Name
|
104
|
+
):
|
105
|
+
# This handles self.config.some_key
|
106
|
+
if key.value.value.id == "self":
|
107
|
+
config_var = key.value.attr
|
108
|
+
config_key = key.attr
|
109
|
+
case_key = f"config:{config_var}.{config_key}"
|
110
|
+
else:
|
111
|
+
return None
|
112
|
+
else:
|
113
|
+
return None
|
114
|
+
|
115
|
+
# handle variables or other dynamic expressions - not allowed
|
116
|
+
elif isinstance(key, ast.Name):
|
117
|
+
return None
|
118
|
+
else:
|
119
|
+
# can't statically analyze this key
|
120
|
+
return None
|
121
|
+
|
122
|
+
if case_key is None:
|
123
|
+
return None
|
124
|
+
|
125
|
+
# extract the step name from the value
|
126
|
+
if isinstance(value, ast.Attribute) and isinstance(
|
127
|
+
value.value, ast.Name
|
128
|
+
):
|
129
|
+
if value.value.id == "self":
|
130
|
+
step_name = value.attr
|
131
|
+
switch_cases[case_key] = step_name
|
132
|
+
else:
|
133
|
+
return None
|
134
|
+
else:
|
135
|
+
return None
|
136
|
+
|
137
|
+
return switch_cases if switch_cases else None
|
138
|
+
|
86
139
|
def _parse(self, func_ast, lineno):
|
87
140
|
self.num_args = len(func_ast.args.args)
|
88
141
|
tail = func_ast.body[-1]
|
@@ -104,7 +157,38 @@ class DAGNode(object):
|
|
104
157
|
self.has_tail_next = True
|
105
158
|
self.invalid_tail_next = True
|
106
159
|
self.tail_next_lineno = lineno + tail.lineno - 1
|
107
|
-
|
160
|
+
|
161
|
+
# Check if first argument is a dictionary (switch case)
|
162
|
+
if (
|
163
|
+
len(tail.value.args) == 1
|
164
|
+
and isinstance(tail.value.args[0], ast.Dict)
|
165
|
+
and any(k.arg == "condition" for k in tail.value.keywords)
|
166
|
+
):
|
167
|
+
# This is a switch statement
|
168
|
+
switch_cases = self._parse_switch_dict(tail.value.args[0])
|
169
|
+
condition_name = None
|
170
|
+
|
171
|
+
# Get condition parameter
|
172
|
+
for keyword in tail.value.keywords:
|
173
|
+
if keyword.arg == "condition":
|
174
|
+
if isinstance(keyword.value, ast.Str):
|
175
|
+
condition_name = keyword.value.s
|
176
|
+
elif isinstance(keyword.value, ast.Constant) and isinstance(
|
177
|
+
keyword.value.value, str
|
178
|
+
):
|
179
|
+
condition_name = keyword.value.value
|
180
|
+
break
|
181
|
+
|
182
|
+
if switch_cases and condition_name:
|
183
|
+
self.type = "split-switch"
|
184
|
+
self.condition = condition_name
|
185
|
+
self.switch_cases = switch_cases
|
186
|
+
self.out_funcs = list(switch_cases.values())
|
187
|
+
self.invalid_tail_next = False
|
188
|
+
return
|
189
|
+
|
190
|
+
else:
|
191
|
+
self.out_funcs = [e.attr for e in tail.value.args]
|
108
192
|
|
109
193
|
keywords = dict(
|
110
194
|
(k.arg, getattr(k.value, "s", None)) for k in tail.value.keywords
|
@@ -144,6 +228,7 @@ class DAGNode(object):
|
|
144
228
|
in_funcs={in_funcs}
|
145
229
|
out_funcs={out_funcs}
|
146
230
|
split_parents={parents}
|
231
|
+
split_branches={branches}
|
147
232
|
matching_join={matching_join}
|
148
233
|
is_inside_foreach={is_inside_foreach}
|
149
234
|
decorators={decos}
|
@@ -151,6 +236,7 @@ class DAGNode(object):
|
|
151
236
|
has_tail_next={0.has_tail_next} (line {0.tail_next_lineno})
|
152
237
|
invalid_tail_next={0.invalid_tail_next}
|
153
238
|
foreach_param={0.foreach_param}
|
239
|
+
condition={0.condition}
|
154
240
|
parallel_step={0.parallel_step}
|
155
241
|
parallel_foreach={0.parallel_foreach}
|
156
242
|
-> {out}""".format(
|
@@ -160,6 +246,7 @@ class DAGNode(object):
|
|
160
246
|
out_funcs=", ".join("[%s]" % x for x in self.out_funcs),
|
161
247
|
in_funcs=", ".join("[%s]" % x for x in self.in_funcs),
|
162
248
|
parents=", ".join("[%s]" % x for x in self.split_parents),
|
249
|
+
branches=", ".join("[%s]" % x for x in self.split_branches),
|
163
250
|
decos=" | ".join(map(str, self.decorators)),
|
164
251
|
out=", ".join("[%s]" % x for x in self.out_funcs),
|
165
252
|
)
|
@@ -210,7 +297,8 @@ class FlowGraph(object):
|
|
210
297
|
node.is_inside_foreach = True
|
211
298
|
|
212
299
|
def _traverse_graph(self):
|
213
|
-
def traverse(node, seen, split_parents):
|
300
|
+
def traverse(node, seen, split_parents, split_branches):
|
301
|
+
add_split_branch = False
|
214
302
|
try:
|
215
303
|
self.sorted_nodes.remove(node.name)
|
216
304
|
except ValueError:
|
@@ -218,15 +306,23 @@ class FlowGraph(object):
|
|
218
306
|
self.sorted_nodes.append(node.name)
|
219
307
|
if node.type in ("split", "foreach"):
|
220
308
|
node.split_parents = split_parents
|
309
|
+
node.split_branches = split_branches
|
310
|
+
add_split_branch = True
|
221
311
|
split_parents = split_parents + [node.name]
|
312
|
+
elif node.type == "split-switch":
|
313
|
+
node.split_parents = split_parents
|
314
|
+
node.split_branches = split_branches
|
222
315
|
elif node.type == "join":
|
223
316
|
# ignore joins without splits
|
224
317
|
if split_parents:
|
225
318
|
self[split_parents[-1]].matching_join = node.name
|
226
319
|
node.split_parents = split_parents
|
320
|
+
node.split_branches = split_branches[:-1]
|
227
321
|
split_parents = split_parents[:-1]
|
322
|
+
split_branches = split_branches[:-1]
|
228
323
|
else:
|
229
324
|
node.split_parents = split_parents
|
325
|
+
node.split_branches = split_branches
|
230
326
|
|
231
327
|
for n in node.out_funcs:
|
232
328
|
# graph may contain loops - ignore them
|
@@ -235,10 +331,15 @@ class FlowGraph(object):
|
|
235
331
|
if n in self:
|
236
332
|
child = self[n]
|
237
333
|
child.in_funcs.add(node.name)
|
238
|
-
traverse(
|
334
|
+
traverse(
|
335
|
+
child,
|
336
|
+
seen + [n],
|
337
|
+
split_parents,
|
338
|
+
split_branches + ([n] if add_split_branch else []),
|
339
|
+
)
|
239
340
|
|
240
341
|
if "start" in self:
|
241
|
-
traverse(self["start"], [], [])
|
342
|
+
traverse(self["start"], [], [], [])
|
242
343
|
|
243
344
|
# fix the order of in_funcs
|
244
345
|
for node in self.nodes.values():
|
@@ -259,15 +360,37 @@ class FlowGraph(object):
|
|
259
360
|
def output_dot(self):
|
260
361
|
def edge_specs():
|
261
362
|
for node in self.nodes.values():
|
262
|
-
|
263
|
-
|
363
|
+
if node.type == "split-switch":
|
364
|
+
# Label edges for switch cases
|
365
|
+
for case_value, step_name in node.switch_cases.items():
|
366
|
+
yield (
|
367
|
+
'{0} -> {1} [label="{2}" color="blue" fontcolor="blue"];'.format(
|
368
|
+
node.name, step_name, case_value
|
369
|
+
)
|
370
|
+
)
|
371
|
+
else:
|
372
|
+
for edge in node.out_funcs:
|
373
|
+
yield "%s -> %s;" % (node.name, edge)
|
264
374
|
|
265
375
|
def node_specs():
|
266
376
|
for node in self.nodes.values():
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
377
|
+
if node.type == "split-switch":
|
378
|
+
# Hexagon shape for switch nodes
|
379
|
+
condition_label = (
|
380
|
+
f"switch: {node.condition}" if node.condition else "switch"
|
381
|
+
)
|
382
|
+
yield (
|
383
|
+
'"{0.name}" '
|
384
|
+
'[ label = <<b>{0.name}</b><br/><font point-size="9">{condition}</font>> '
|
385
|
+
' fontname = "Helvetica" '
|
386
|
+
' shape = "hexagon" '
|
387
|
+
' style = "filled" fillcolor = "lightgreen" ];'
|
388
|
+
).format(node, condition=condition_label)
|
389
|
+
else:
|
390
|
+
nodetype = "join" if node.num_args > 1 else node.type
|
391
|
+
yield '"{0.name}"' '[ label = <<b>{0.name}</b> | <font point-size="10">{type}</font>> ' ' fontname = "Helvetica" ' ' shape = "record" ];'.format(
|
392
|
+
node, type=nodetype
|
393
|
+
)
|
271
394
|
|
272
395
|
return (
|
273
396
|
"digraph {0.name} {{\n"
|
@@ -291,6 +414,8 @@ class FlowGraph(object):
|
|
291
414
|
if node.parallel_foreach:
|
292
415
|
return "split-parallel"
|
293
416
|
return "split-foreach"
|
417
|
+
elif node.type == "split-switch":
|
418
|
+
return "split-switch"
|
294
419
|
return "unknown" # Should never happen
|
295
420
|
|
296
421
|
def node_to_dict(name, node):
|
@@ -325,6 +450,9 @@ class FlowGraph(object):
|
|
325
450
|
d["foreach_artifact"] = node.foreach_param
|
326
451
|
elif d["type"] == "split-parallel":
|
327
452
|
d["num_parallel"] = node.num_parallel
|
453
|
+
elif d["type"] == "split-switch":
|
454
|
+
d["condition"] = node.condition
|
455
|
+
d["switch_cases"] = node.switch_cases
|
328
456
|
if node.matching_join:
|
329
457
|
d["matching_join"] = node.matching_join
|
330
458
|
return d
|
@@ -339,8 +467,8 @@ class FlowGraph(object):
|
|
339
467
|
steps_info[cur_name] = node_dict
|
340
468
|
resulting_list.append(cur_name)
|
341
469
|
|
342
|
-
|
343
|
-
|
470
|
+
node_type = node_to_type(cur_node)
|
471
|
+
if node_type in ("split-static", "split-foreach"):
|
344
472
|
resulting_list.append(
|
345
473
|
[
|
346
474
|
populate_block(s, cur_node.matching_join)
|
@@ -348,8 +476,19 @@ class FlowGraph(object):
|
|
348
476
|
]
|
349
477
|
)
|
350
478
|
cur_name = cur_node.matching_join
|
479
|
+
elif node_type == "split-switch":
|
480
|
+
all_paths = [
|
481
|
+
populate_block(s, end_name) for s in cur_node.out_funcs
|
482
|
+
]
|
483
|
+
resulting_list.append(all_paths)
|
484
|
+
cur_name = end_name
|
351
485
|
else:
|
352
|
-
|
486
|
+
# handles only linear, start, and join steps.
|
487
|
+
if cur_node.out_funcs:
|
488
|
+
cur_name = cur_node.out_funcs[0]
|
489
|
+
else:
|
490
|
+
# handles terminal nodes or when we jump to 'end_name'.
|
491
|
+
break
|
353
492
|
return resulting_list
|
354
493
|
|
355
494
|
graph_structure = populate_block("start", "end")
|
metaflow/lint.py
CHANGED
@@ -134,7 +134,13 @@ def check_valid_transitions(graph):
|
|
134
134
|
msg = (
|
135
135
|
"Step *{0.name}* specifies an invalid self.next() transition. "
|
136
136
|
"Make sure the self.next() expression matches with one of the "
|
137
|
-
"supported transition types
|
137
|
+
"supported transition types:\n"
|
138
|
+
" • Linear: self.next(self.step_name)\n"
|
139
|
+
" • Fan-out: self.next(self.step1, self.step2, ...)\n"
|
140
|
+
" • Foreach: self.next(self.step, foreach='variable')\n"
|
141
|
+
" • Switch: self.next({{\"key\": self.step, ...}}, condition='variable')\n\n"
|
142
|
+
"For switch statements, keys must be string literals, numbers or config expressions "
|
143
|
+
"(self.config.key_name), not variables."
|
138
144
|
)
|
139
145
|
for node in graph:
|
140
146
|
if node.type != "end" and node.has_tail_next and node.invalid_tail_next:
|
@@ -232,7 +238,13 @@ def check_split_join_balance(graph):
|
|
232
238
|
new_stack = split_stack
|
233
239
|
elif node.type in ("split", "foreach"):
|
234
240
|
new_stack = split_stack + [("split", node.out_funcs)]
|
241
|
+
elif node.type == "split-switch":
|
242
|
+
# For a switch, continue traversal down each path with the same stack
|
243
|
+
for n in node.out_funcs:
|
244
|
+
traverse(graph[n], split_stack)
|
245
|
+
return
|
235
246
|
elif node.type == "end":
|
247
|
+
new_stack = split_stack
|
236
248
|
if split_stack:
|
237
249
|
_, split_roots = split_stack.pop()
|
238
250
|
roots = ", ".join(split_roots)
|
@@ -240,11 +252,22 @@ def check_split_join_balance(graph):
|
|
240
252
|
msg0.format(roots=roots), node.func_lineno, node.source_file
|
241
253
|
)
|
242
254
|
elif node.type == "join":
|
255
|
+
new_stack = split_stack
|
243
256
|
if split_stack:
|
244
257
|
_, split_roots = split_stack[-1]
|
245
258
|
new_stack = split_stack[:-1]
|
246
|
-
|
247
|
-
|
259
|
+
|
260
|
+
# Resolve each incoming function to its root branch from the split.
|
261
|
+
resolved_branches = set(
|
262
|
+
graph[n].split_branches[-1] for n in node.in_funcs
|
263
|
+
)
|
264
|
+
|
265
|
+
# compares the set of resolved branches against the expected branches
|
266
|
+
# from the split.
|
267
|
+
if len(resolved_branches) != len(
|
268
|
+
split_roots
|
269
|
+
) or resolved_branches ^ set(split_roots):
|
270
|
+
paths = ", ".join(resolved_branches)
|
248
271
|
roots = ", ".join(split_roots)
|
249
272
|
raise LintWarn(
|
250
273
|
msg1.format(
|
@@ -266,6 +289,8 @@ def check_split_join_balance(graph):
|
|
266
289
|
|
267
290
|
if not all_equal(map(parents, node.in_funcs)):
|
268
291
|
raise LintWarn(msg3.format(node), node.func_lineno, node.source_file)
|
292
|
+
else:
|
293
|
+
new_stack = split_stack
|
269
294
|
|
270
295
|
for n in node.out_funcs:
|
271
296
|
traverse(graph[n], new_stack)
|
@@ -273,6 +298,44 @@ def check_split_join_balance(graph):
|
|
273
298
|
traverse(graph["start"], [])
|
274
299
|
|
275
300
|
|
301
|
+
@linter.ensure_static_graph
|
302
|
+
@linter.check
|
303
|
+
def check_switch_splits(graph):
|
304
|
+
"""Check conditional split constraints"""
|
305
|
+
msg0 = (
|
306
|
+
"Step *{0.name}* is a switch split but defines {num} transitions. "
|
307
|
+
"Switch splits must define at least 2 transitions."
|
308
|
+
)
|
309
|
+
msg1 = "Step *{0.name}* is a switch split but has no condition variable."
|
310
|
+
msg2 = "Step *{0.name}* is a switch split but has no switch cases defined."
|
311
|
+
|
312
|
+
for node in graph:
|
313
|
+
if node.type == "split-switch":
|
314
|
+
# Check at least 2 outputs
|
315
|
+
if len(node.out_funcs) < 2:
|
316
|
+
raise LintWarn(
|
317
|
+
msg0.format(node, num=len(node.out_funcs)),
|
318
|
+
node.func_lineno,
|
319
|
+
node.source_file,
|
320
|
+
)
|
321
|
+
|
322
|
+
# Check condition exists
|
323
|
+
if not node.condition:
|
324
|
+
raise LintWarn(
|
325
|
+
msg1.format(node),
|
326
|
+
node.func_lineno,
|
327
|
+
node.source_file,
|
328
|
+
)
|
329
|
+
|
330
|
+
# Check switch cases exist
|
331
|
+
if not node.switch_cases:
|
332
|
+
raise LintWarn(
|
333
|
+
msg2.format(node),
|
334
|
+
node.func_lineno,
|
335
|
+
node.source_file,
|
336
|
+
)
|
337
|
+
|
338
|
+
|
276
339
|
@linter.ensure_static_graph
|
277
340
|
@linter.check
|
278
341
|
def check_empty_foreaches(graph):
|
@@ -948,6 +948,11 @@ class ArgoWorkflows(object):
|
|
948
948
|
dag_task = DAGTask(self._sanitize(node.name)).template(
|
949
949
|
self._sanitize(node.name)
|
950
950
|
)
|
951
|
+
if node.type == "split-switch":
|
952
|
+
raise ArgoWorkflowsException(
|
953
|
+
"Deploying flows with switch statement "
|
954
|
+
"to Argo Workflows is not supported currently."
|
955
|
+
)
|
951
956
|
elif (
|
952
957
|
node.is_inside_foreach
|
953
958
|
and self.graph[node.in_funcs[0]].type == "foreach"
|
@@ -317,6 +317,12 @@ class StepFunctions(object):
|
|
317
317
|
"to AWS Step Functions is not supported currently."
|
318
318
|
)
|
319
319
|
|
320
|
+
if node.type == "split-switch":
|
321
|
+
raise StepFunctionsException(
|
322
|
+
"Deploying flows with switch statement "
|
323
|
+
"to AWS Step Functions is not supported currently."
|
324
|
+
)
|
325
|
+
|
320
326
|
# Assign an AWS Batch job to the AWS Step Functions state
|
321
327
|
# and pass the intermediate state by exposing `JobId` and
|
322
328
|
# `Parameters` to the child job(s) as outputs. `Index` and
|
@@ -20,12 +20,15 @@ def transform_flow_graph(step_info):
|
|
20
20
|
return "split"
|
21
21
|
elif node_type == "split-parallel" or node_type == "split-foreach":
|
22
22
|
return "foreach"
|
23
|
+
elif node_type == "split-switch":
|
24
|
+
return "switch"
|
23
25
|
return "unknown" # Should never happen
|
24
26
|
|
25
27
|
graph_dict = {}
|
26
28
|
for stepname in step_info:
|
27
|
-
|
28
|
-
|
29
|
+
node_type = node_to_type(step_info[stepname]["type"])
|
30
|
+
node_info = {
|
31
|
+
"type": node_type,
|
29
32
|
"box_next": step_info[stepname]["type"] not in ("linear", "join"),
|
30
33
|
"box_ends": (
|
31
34
|
None
|
@@ -35,6 +38,15 @@ def transform_flow_graph(step_info):
|
|
35
38
|
"next": step_info[stepname]["next"],
|
36
39
|
"doc": step_info[stepname]["doc"],
|
37
40
|
}
|
41
|
+
|
42
|
+
if node_type == "switch":
|
43
|
+
if "condition" in step_info[stepname]:
|
44
|
+
node_info["condition"] = step_info[stepname]["condition"]
|
45
|
+
if "switch_cases" in step_info[stepname]:
|
46
|
+
node_info["switch_cases"] = step_info[stepname]["switch_cases"]
|
47
|
+
|
48
|
+
graph_dict[stepname] = node_info
|
49
|
+
|
38
50
|
return graph_dict
|
39
51
|
|
40
52
|
|
@@ -0,0 +1 @@
|
|
1
|
+
@import"https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap";code[class*=language-],pre[class*=language-]{color:#000;background:0 0;text-shadow:0 1px #fff;font-family:Consolas,Monaco,Andale Mono,Ubuntu Mono,monospace;font-size:1em;text-align:left;white-space:pre;word-spacing:normal;word-break:normal;word-wrap:normal;line-height:1.5;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-hyphens:none;-moz-hyphens:none;-ms-hyphens:none;hyphens:none}code[class*=language-] ::-moz-selection,code[class*=language-]::-moz-selection,pre[class*=language-] ::-moz-selection,pre[class*=language-]::-moz-selection{text-shadow:none;background:#b3d4fc}code[class*=language-] ::selection,code[class*=language-]::selection,pre[class*=language-] ::selection,pre[class*=language-]::selection{text-shadow:none;background:#b3d4fc}@media print{code[class*=language-],pre[class*=language-]{text-shadow:none}}pre[class*=language-]{padding:1em;margin:.5em 0;overflow:auto}:not(pre)>code[class*=language-],pre[class*=language-]{background:#f5f2f0}:not(pre)>code[class*=language-]{padding:.1em;border-radius:.3em;white-space:normal}.token.cdata,.token.comment,.token.doctype,.token.prolog{color:#708090}.token.punctuation{color:#999}.token.namespace{opacity:.7}.token.boolean,.token.constant,.token.deleted,.token.number,.token.property,.token.symbol,.token.tag{color:#905}.token.attr-name,.token.builtin,.token.char,.token.inserted,.token.selector,.token.string{color:#690}.language-css .token.string,.style .token.string,.token.entity,.token.operator,.token.url{color:#9a6e3a;background:#ffffff80}.token.atrule,.token.attr-value,.token.keyword{color:#07a}.token.class-name,.token.function{color:#dd4a68}.token.important,.token.regex,.token.variable{color:#e90}.token.bold,.token.important{font-weight:700}.token.italic{font-style:italic}.token.entity{cursor:help}:root{--bg: #ffffff;--black: #333;--blue: #0c66de;--dk-grey: #767676;--dk-primary: #ef863b;--dk-secondary: #13172d;--dk-tertiary: #0f426e;--error: #cf483e;--grey: rgba(0, 0, 0, .125);--highlight: #f8d9d8;--lt-blue: #4fa7ff;--lt-grey: #f3f3f3;--lt-lt-grey: #f9f9f9;--lt-primary: #ffcb8b;--lt-secondary: #434d81;--lt-tertiary: #4189c9;--primary: #faab4a;--quadrary: #f8d9d8;--secondary: #2e3454;--tertiary: #2a679d;--white: #ffffff;--component-spacer: 3rem;--aside-width: 20rem;--embed-card-min-height: 12rem;--mono-font: ui-monospace, Menlo, Monaco, "Cascadia Mono", "Segoe UI Mono", "Roboto Mono", "Oxygen Mono", "Ubuntu Monospace", "Source Code Pro", "Fira Mono", "Droid Sans Mono", "Courier New", monospace}html,body{margin:0;min-height:100vh;overflow-y:visible;padding:0;width:100%}.card_app{width:100%;min-height:100vh}.embed .card_app{min-height:var(--embed-card-min-height)}.mf-card *{box-sizing:border-box}.mf-card{background:var(--bg);color:var(--black);font-family:Roboto,-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Oxygen,Ubuntu,Cantarell,Fira Sans,Droid Sans,Helvetica Neue,sans-serif;font-size:14px;font-weight:400;line-height:1.5;text-size-adjust:100%;margin:0;min-height:100vh;overflow-y:visible;padding:0;text-align:left;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;width:100%}.embed .mf-card{min-height:var(--embed-card-min-height)}.mf-card :is(.mono,code.mono,pre.mono){font-family:var(--mono-font);font-weight:lighter}.mf-card :is(table,th,td){border-spacing:1px;text-align:center;color:var(--black)}.mf-card table{position:relative;min-width:100%;table-layout:inherit!important}.mf-card td{padding:.66rem 1.25rem;background:var(--lt-lt-grey);border:none}.mf-card th{border:none;color:var(--dk-grey);font-weight:400;padding:.5rem}.mf-card :is(h1,h2,h3,h4,h5){font-weight:700;margin:.5rem 0}.mf-card ul{margin:0;padding:0}.mf-card p{margin:0 0 1rem}.mf-card p:last-of-type{margin:0}.mf-card button{font-size:1rem}.mf-card .textButton{cursor:pointer;text-align:left;background:none;border:1px solid transparent;outline:none;padding:0}.mf-card :is(button.textButton:focus,a:focus,button.textButton:active){border:1px dashed var(--grey);background:transparent}.mf-card button.textButton:hover{color:var(--blue);text-decoration:none}.mf-card :is(:not(pre)>code[class*=language-],pre[class*=language-]){background:transparent!important;text-shadow:none;-webkit-user-select:auto;user-select:auto}aside.svelte-1okdv0e{display:none;line-height:2;text-align:left}@media (min-width: 60rem){aside.svelte-1okdv0e{display:flex;flex-direction:column;height:100vh;justify-content:space-between;padding:2.5rem 0 1.5rem 1.5rem;position:fixed;width:var(--aside-width)}}.embed aside{display:none}aside ul{list-style-type:none}aside a,aside button,aside a:visited{text-decoration:none;cursor:pointer;font-weight:700;color:var(--black)}aside a:hover,aside button:hover{text-decoration:underline}.logoContainer svg{width:100%;max-width:140px;margin-bottom:3.75rem;height:auto}.idCell.svelte-pt8vzv{font-weight:700;text-align:right;background:var(--lt-grey);width:12%}.codeCell.svelte-pt8vzv{text-align:left;-webkit-user-select:all;user-select:all}.container.svelte-ubs992{width:100%;overflow:auto}table.svelte-ubs992{width:100%}:root{--dag-border: #282828;--dag-bg-static: var(--lt-grey);--dag-bg-success: #a5d46a;--dag-bg-running: #ffdf80;--dag-bg-error: #ffa080;--dag-connector: #cccccc;--dag-gap: 5rem;--dag-step-height: 6.25rem;--dag-step-width: 11.25rem;--dag-selected: #ffd700}.connectorwrapper.svelte-1hyaq5f{transform-origin:0 0;position:absolute;z-index:0;min-width:var(--strokeWidth)}.flip.svelte-1hyaq5f{transform:scaleX(-1)}.path.svelte-1hyaq5f{--strokeWidth: .5rem;--strokeColor: var(--dag-connector);--borderRadius: 1.25rem;box-sizing:border-box}.straightLine.svelte-1hyaq5f{position:absolute;inset:0;border-left:var(--strokeWidth) solid var(--strokeColor)}.topLeft.svelte-1hyaq5f{position:absolute;top:0;left:0;right:50%;bottom:calc(var(--dag-gap) / 2 - var(--strokeWidth) / 2);border-radius:0 0 0 var(--borderRadius);border-left:var(--strokeWidth) solid var(--strokeColor);border-bottom:var(--strokeWidth) solid var(--strokeColor)}.bottomRight.svelte-1hyaq5f{position:absolute;top:calc(100% - (var(--dag-gap) / 2 + var(--strokeWidth) / 2));left:50%;right:0;bottom:0;border-radius:0 var(--borderRadius) 0 0;border-top:var(--strokeWidth) solid var(--strokeColor);border-right:var(--strokeWidth) solid var(--strokeColor)}.wrapper.svelte-117ceti{position:relative;z-index:1}.step.svelte-117ceti{font-size:.75rem;padding:.5rem;color:var(--dk-grey)}.rectangle.svelte-117ceti{background-color:var(--dag-bg-static);border:1px solid var(--dag-border);box-sizing:border-box;position:relative;height:var(--dag-step-height);width:var(--dag-step-width)}.rectangle.error.svelte-117ceti{background-color:var(--dag-bg-error)}.rectangle.success.svelte-117ceti{background-color:var(--dag-bg-success)}.rectangle.running.svelte-117ceti{background-color:var(--dag-bg-running)}.level.svelte-117ceti{z-index:-1;filter:contrast(.5);position:absolute}.inner.svelte-117ceti{position:relative;height:100%;width:100%}.name.svelte-117ceti{font-weight:700;overflow:hidden;text-overflow:ellipsis;display:block}.description.svelte-117ceti{position:absolute;max-height:4rem;bottom:0;left:0;right:0;overflow:hidden;-webkit-line-clamp:4;line-clamp:4;display:-webkit-box;-webkit-box-orient:vertical}.overflown.description.svelte-117ceti{cursor:help}.current.svelte-117ceti .rectangle:where(.svelte-117ceti){box-shadow:0 0 10px var(--dag-selected)}.levelstoshow.svelte-117ceti{position:absolute;bottom:100%;right:0;font-size:.75rem;font-weight:100;text-align:right}.stepwrapper.svelte-18aex7a{display:flex;align-items:center;flex-direction:column;width:100%;position:relative;min-width:var(--dag-step-width)}.childwrapper.svelte-18aex7a{display:flex;width:100%}.gap.svelte-18aex7a{height:var(--dag-gap)}.title.svelte-117s0ws{text-align:left}.subtitle.svelte-lu9pnn{font-size:1rem;text-align:left}header.svelte-1ugmt5d{margin-bottom:var(--component-spacer)}figure.svelte-1x96yvr{background:var(--lt-grey);padding:1rem;border-radius:5px;text-align:center;margin:0 auto var(--component-spacer)}@media (min-width: 60rem){figure.svelte-1x96yvr{margin-bottom:0}}img.svelte-1x96yvr{max-width:100%;max-height:500px}.label.svelte-1x96yvr{font-weight:700;margin:.5rem 0}.description.svelte-1x96yvr{font-size:.9rem;font-style:italic;text-align:center;margin:.5rem 0}.log.svelte-1jhmsu{background:var(--lt-grey)!important;font-size:.9rem;padding:2rem}.page.svelte-v7ihqd:last-of-type{margin-bottom:var(--component-spacer)}.page:last-of-type section:last-of-type hr{display:none}progress.svelte-ljrmzp::-webkit-progress-bar{background-color:#fff!important;min-width:100%}progress.svelte-ljrmzp{background-color:#fff;color:#326cded9!important}progress.svelte-ljrmzp::-moz-progress-bar{background-color:#326cde!important}table .container{background:transparent!important;font-size:10px!important;padding:0!important}table progress{height:4px!important}.container.svelte-ljrmzp{display:flex;align-items:center;justify-content:center;font-size:12px;border-radius:3px;background:#edf5ff;padding:3rem}.inner.svelte-ljrmzp{max-width:410px;width:100%;text-align:center}.info.svelte-ljrmzp{display:flex;justify-content:space-between}table .info{text-align:left;flex-direction:column}label.svelte-ljrmzp{font-weight:700}.labelValue.svelte-ljrmzp{border-left:1px solid rgba(0,0,0,.1);margin-left:.25rem;padding-left:.5rem}.details.svelte-ljrmzp{font-family:var(--mono-font);font-size:8px;color:#333433;line-height:18px;overflow:hidden;white-space:nowrap}progress.svelte-ljrmzp{width:100%;border:none;border-radius:5px;height:8px;background:#fff}.heading.svelte-17n0qr8{margin-bottom:1.5rem}.sectionItems.svelte-17n0qr8{display:block}.sectionItems .imageContainer{max-height:500px}.container.svelte-17n0qr8{scroll-margin:var(--component-spacer)}hr.svelte-17n0qr8{background:var(--grey);border:none;height:1px;margin:var(--component-spacer) 0;padding:0}@media (min-width: 60rem){.sectionItems.svelte-17n0qr8{display:grid;grid-gap:2rem}}td.svelte-gl9h79{text-align:left}td.labelColumn.svelte-gl9h79{text-align:right;background-color:var(--lt-grey);font-weight:700;width:12%;white-space:nowrap}.tableContainer.svelte-q3hq57{overflow:auto}th.svelte-q3hq57{position:sticky;top:-1px;z-index:2;white-space:nowrap;background:var(--white)}.mainContainer.svelte-mqeomk{max-width:110rem}main.svelte-mqeomk{flex:0 1 auto;max-width:100rem;padding:1.5rem}@media (min-width: 60rem){main.svelte-mqeomk{margin-left:var(--aside-width)}}.embed main{margin:0 auto;min-width:80%}.modal.svelte-1hhf5ym{align-items:center;background:#00000080;cursor:pointer;display:flex;height:100%;justify-content:center;inset:0;overflow:hidden;position:fixed;width:100%;z-index:100}.modalContainer>*{background-color:#fff;border-radius:5px;cursor:default;flex:0 1 auto;padding:1rem;position:relative}.modal img{max-height:80vh!important}.cancelButton.svelte-1hhf5ym{color:#fff;cursor:pointer;font-size:2rem;position:absolute;right:1rem;top:1rem}.cancelButton.svelte-1hhf5ym:hover{color:var(--blue)}.nav.svelte-1kdpgko{border-radius:0 0 5px;display:none;margin:0;top:0}ul.navList.svelte-1kdpgko{list-style-type:none}ul.navList.svelte-1kdpgko ul:where(.svelte-1kdpgko){margin:.5rem 1rem 2rem}.navList.svelte-1kdpgko li:where(.svelte-1kdpgko){display:block;margin:0}.navItem.svelte-1kdpgko li:where(.svelte-1kdpgko):hover{color:var(--blue)}.pageId.svelte-1kdpgko{display:block;border-bottom:1px solid var(--grey);padding:0 .5rem;margin-bottom:1rem}@media (min-width: 60rem){.nav.svelte-1kdpgko{display:block}ul.navList.svelte-1kdpgko{text-align:left}.navList.svelte-1kdpgko li:where(.svelte-1kdpgko){display:block;margin:.5rem 0}}.container.svelte-teyund{width:100%;display:flex;flex-direction:column;position:relative}
|