ob-metaflow 2.17.0.1__py2.py3-none-any.whl → 2.17.3.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ob-metaflow might be problematic. Click here for more details.
- metaflow/cli_components/run_cmds.py +15 -0
- metaflow/cmd/make_wrapper.py +30 -0
- metaflow/flowspec.py +91 -1
- metaflow/graph.py +152 -13
- metaflow/lint.py +66 -3
- metaflow/metaflow_version.py +15 -0
- metaflow/plugins/argo/argo_workflows.py +233 -9
- metaflow/plugins/argo/argo_workflows_decorator.py +9 -0
- metaflow/plugins/argo/conditional_input_paths.py +21 -0
- metaflow/plugins/aws/step_functions/step_functions.py +6 -0
- metaflow/plugins/cards/card_modules/basic.py +14 -2
- metaflow/plugins/cards/card_modules/main.css +1 -0
- metaflow/plugins/cards/card_modules/main.js +28 -28
- metaflow/plugins/catch_decorator.py +9 -0
- metaflow/plugins/parallel_decorator.py +7 -0
- metaflow/runner/deployer_impl.py +15 -7
- metaflow/runtime.py +57 -14
- metaflow/task.py +62 -34
- metaflow/user_decorators/mutable_flow.py +5 -5
- metaflow/user_decorators/user_step_decorator.py +24 -5
- metaflow/version.py +1 -1
- {ob_metaflow-2.17.0.1.data → ob_metaflow-2.17.3.1.data}/data/share/metaflow/devtools/Makefile +3 -2
- {ob_metaflow-2.17.0.1.data → ob_metaflow-2.17.3.1.data}/data/share/metaflow/devtools/Tiltfile +2 -2
- {ob_metaflow-2.17.0.1.dist-info → ob_metaflow-2.17.3.1.dist-info}/METADATA +2 -2
- {ob_metaflow-2.17.0.1.dist-info → ob_metaflow-2.17.3.1.dist-info}/RECORD +30 -28
- {ob_metaflow-2.17.0.1.data → ob_metaflow-2.17.3.1.data}/data/share/metaflow/devtools/pick_services.sh +0 -0
- {ob_metaflow-2.17.0.1.dist-info → ob_metaflow-2.17.3.1.dist-info}/WHEEL +0 -0
- {ob_metaflow-2.17.0.1.dist-info → ob_metaflow-2.17.3.1.dist-info}/entry_points.txt +0 -0
- {ob_metaflow-2.17.0.1.dist-info → ob_metaflow-2.17.3.1.dist-info}/licenses/LICENSE +0 -0
- {ob_metaflow-2.17.0.1.dist-info → ob_metaflow-2.17.3.1.dist-info}/top_level.txt +0 -0
|
@@ -152,6 +152,7 @@ class ArgoWorkflows(object):
|
|
|
152
152
|
|
|
153
153
|
self.name = name
|
|
154
154
|
self.graph = graph
|
|
155
|
+
self._parse_conditional_branches()
|
|
155
156
|
self.flow = flow
|
|
156
157
|
self.code_package_metadata = code_package_metadata
|
|
157
158
|
self.code_package_sha = code_package_sha
|
|
@@ -929,6 +930,121 @@ class ArgoWorkflows(object):
|
|
|
929
930
|
)
|
|
930
931
|
)
|
|
931
932
|
|
|
933
|
+
# Visit every node and record information on conditional step structure
|
|
934
|
+
def _parse_conditional_branches(self):
|
|
935
|
+
self.conditional_nodes = set()
|
|
936
|
+
self.conditional_join_nodes = set()
|
|
937
|
+
self.matching_conditional_join_dict = {}
|
|
938
|
+
|
|
939
|
+
node_conditional_parents = {}
|
|
940
|
+
node_conditional_branches = {}
|
|
941
|
+
|
|
942
|
+
def _visit(node, seen, conditional_branch, conditional_parents=None):
|
|
943
|
+
if not node.type == "split-switch" and not (
|
|
944
|
+
conditional_branch and conditional_parents
|
|
945
|
+
):
|
|
946
|
+
# skip regular non-conditional nodes entirely
|
|
947
|
+
return
|
|
948
|
+
|
|
949
|
+
if node.type == "split-switch":
|
|
950
|
+
conditional_branch = conditional_branch + [node.name]
|
|
951
|
+
node_conditional_branches[node.name] = conditional_branch
|
|
952
|
+
|
|
953
|
+
conditional_parents = (
|
|
954
|
+
[node.name]
|
|
955
|
+
if not conditional_parents
|
|
956
|
+
else conditional_parents + [node.name]
|
|
957
|
+
)
|
|
958
|
+
node_conditional_parents[node.name] = conditional_parents
|
|
959
|
+
|
|
960
|
+
if conditional_parents and not node.type == "split-switch":
|
|
961
|
+
node_conditional_parents[node.name] = conditional_parents
|
|
962
|
+
conditional_branch = conditional_branch + [node.name]
|
|
963
|
+
node_conditional_branches[node.name] = conditional_branch
|
|
964
|
+
|
|
965
|
+
self.conditional_nodes.add(node.name)
|
|
966
|
+
|
|
967
|
+
if conditional_branch and conditional_parents:
|
|
968
|
+
for n in node.out_funcs:
|
|
969
|
+
child = self.graph[n]
|
|
970
|
+
if n not in seen:
|
|
971
|
+
_visit(
|
|
972
|
+
child, seen + [n], conditional_branch, conditional_parents
|
|
973
|
+
)
|
|
974
|
+
|
|
975
|
+
# First we visit all nodes to determine conditional parents and branches
|
|
976
|
+
for n in self.graph:
|
|
977
|
+
_visit(n, [], [])
|
|
978
|
+
|
|
979
|
+
# Then we traverse again in order to determine conditional join nodes, and matching conditional join info
|
|
980
|
+
for node in self.graph:
|
|
981
|
+
if node_conditional_parents.get(node.name, False):
|
|
982
|
+
# do the required postprocessing for anything requiring node.in_funcs
|
|
983
|
+
|
|
984
|
+
# check that in previous parsing we have not closed all conditional in_funcs.
|
|
985
|
+
# If so, this step can not be conditional either
|
|
986
|
+
is_conditional = any(
|
|
987
|
+
in_func in self.conditional_nodes
|
|
988
|
+
or self.graph[in_func].type == "split-switch"
|
|
989
|
+
for in_func in node.in_funcs
|
|
990
|
+
)
|
|
991
|
+
if is_conditional:
|
|
992
|
+
self.conditional_nodes.add(node.name)
|
|
993
|
+
else:
|
|
994
|
+
if node.name in self.conditional_nodes:
|
|
995
|
+
self.conditional_nodes.remove(node.name)
|
|
996
|
+
|
|
997
|
+
# does this node close the latest conditional parent branches?
|
|
998
|
+
conditional_in_funcs = [
|
|
999
|
+
in_func
|
|
1000
|
+
for in_func in node.in_funcs
|
|
1001
|
+
if node_conditional_branches.get(in_func, False)
|
|
1002
|
+
]
|
|
1003
|
+
closed_conditional_parents = []
|
|
1004
|
+
for last_split_switch in node_conditional_parents.get(node.name, [])[
|
|
1005
|
+
::-1
|
|
1006
|
+
]:
|
|
1007
|
+
last_conditional_split_nodes = self.graph[
|
|
1008
|
+
last_split_switch
|
|
1009
|
+
].out_funcs
|
|
1010
|
+
# p needs to be in at least one conditional_branch for it to be closed.
|
|
1011
|
+
if all(
|
|
1012
|
+
any(
|
|
1013
|
+
p in node_conditional_branches.get(in_func, [])
|
|
1014
|
+
for in_func in conditional_in_funcs
|
|
1015
|
+
)
|
|
1016
|
+
for p in last_conditional_split_nodes
|
|
1017
|
+
):
|
|
1018
|
+
closed_conditional_parents.append(last_split_switch)
|
|
1019
|
+
|
|
1020
|
+
self.conditional_join_nodes.add(node.name)
|
|
1021
|
+
self.matching_conditional_join_dict[last_split_switch] = (
|
|
1022
|
+
node.name
|
|
1023
|
+
)
|
|
1024
|
+
|
|
1025
|
+
# Did we close all conditionals? Then this branch and all its children are not conditional anymore (unless a new conditional branch is encountered).
|
|
1026
|
+
if not [
|
|
1027
|
+
p
|
|
1028
|
+
for p in node_conditional_parents.get(node.name, [])
|
|
1029
|
+
if p not in closed_conditional_parents
|
|
1030
|
+
]:
|
|
1031
|
+
if node.name in self.conditional_nodes:
|
|
1032
|
+
self.conditional_nodes.remove(node.name)
|
|
1033
|
+
node_conditional_parents[node.name] = []
|
|
1034
|
+
for p in node.out_funcs:
|
|
1035
|
+
if p in self.conditional_nodes:
|
|
1036
|
+
self.conditional_nodes.remove(p)
|
|
1037
|
+
node_conditional_parents[p] = []
|
|
1038
|
+
|
|
1039
|
+
def _is_conditional_node(self, node):
|
|
1040
|
+
return node.name in self.conditional_nodes
|
|
1041
|
+
|
|
1042
|
+
def _is_conditional_join_node(self, node):
|
|
1043
|
+
return node.name in self.conditional_join_nodes
|
|
1044
|
+
|
|
1045
|
+
def _matching_conditional_join(self, node):
|
|
1046
|
+
return self.matching_conditional_join_dict.get(node.name, None)
|
|
1047
|
+
|
|
932
1048
|
# Visit every node and yield the uber DAGTemplate(s).
|
|
933
1049
|
def _dag_templates(self):
|
|
934
1050
|
def _visit(
|
|
@@ -950,6 +1066,7 @@ class ArgoWorkflows(object):
|
|
|
950
1066
|
dag_tasks = []
|
|
951
1067
|
if templates is None:
|
|
952
1068
|
templates = []
|
|
1069
|
+
|
|
953
1070
|
if exit_node is not None and exit_node is node.name:
|
|
954
1071
|
return templates, dag_tasks
|
|
955
1072
|
if node.name == "start":
|
|
@@ -957,7 +1074,7 @@ class ArgoWorkflows(object):
|
|
|
957
1074
|
dag_task = DAGTask(self._sanitize(node.name)).template(
|
|
958
1075
|
self._sanitize(node.name)
|
|
959
1076
|
)
|
|
960
|
-
|
|
1077
|
+
if (
|
|
961
1078
|
node.is_inside_foreach
|
|
962
1079
|
and self.graph[node.in_funcs[0]].type == "foreach"
|
|
963
1080
|
and not self.graph[node.in_funcs[0]].parallel_foreach
|
|
@@ -1091,15 +1208,43 @@ class ArgoWorkflows(object):
|
|
|
1091
1208
|
]
|
|
1092
1209
|
)
|
|
1093
1210
|
|
|
1211
|
+
conditional_deps = [
|
|
1212
|
+
"%s.Succeeded" % self._sanitize(in_func)
|
|
1213
|
+
for in_func in node.in_funcs
|
|
1214
|
+
if self._is_conditional_node(self.graph[in_func])
|
|
1215
|
+
]
|
|
1216
|
+
required_deps = [
|
|
1217
|
+
"%s.Succeeded" % self._sanitize(in_func)
|
|
1218
|
+
for in_func in node.in_funcs
|
|
1219
|
+
if not self._is_conditional_node(self.graph[in_func])
|
|
1220
|
+
]
|
|
1221
|
+
both_conditions = required_deps and conditional_deps
|
|
1222
|
+
|
|
1223
|
+
depends_str = "{required}{_and}{conditional}".format(
|
|
1224
|
+
required=("(%s)" if both_conditions else "%s")
|
|
1225
|
+
% " && ".join(required_deps),
|
|
1226
|
+
_and=" && " if both_conditions else "",
|
|
1227
|
+
conditional=("(%s)" if both_conditions else "%s")
|
|
1228
|
+
% " || ".join(conditional_deps),
|
|
1229
|
+
)
|
|
1094
1230
|
dag_task = (
|
|
1095
1231
|
DAGTask(self._sanitize(node.name))
|
|
1096
|
-
.
|
|
1097
|
-
[self._sanitize(in_func) for in_func in node.in_funcs]
|
|
1098
|
-
)
|
|
1232
|
+
.depends(depends_str)
|
|
1099
1233
|
.template(self._sanitize(node.name))
|
|
1100
1234
|
.arguments(Arguments().parameters(parameters))
|
|
1101
1235
|
)
|
|
1102
1236
|
|
|
1237
|
+
# Add conditional if this is the first step in a conditional branch
|
|
1238
|
+
if (
|
|
1239
|
+
self._is_conditional_node(node)
|
|
1240
|
+
and self.graph[node.in_funcs[0]].type == "split-switch"
|
|
1241
|
+
):
|
|
1242
|
+
in_func = node.in_funcs[0]
|
|
1243
|
+
dag_task.when(
|
|
1244
|
+
"{{tasks.%s.outputs.parameters.switch-step}}==%s"
|
|
1245
|
+
% (self._sanitize(in_func), node.name)
|
|
1246
|
+
)
|
|
1247
|
+
|
|
1103
1248
|
dag_tasks.append(dag_task)
|
|
1104
1249
|
# End the workflow if we have reached the end of the flow
|
|
1105
1250
|
if node.type == "end":
|
|
@@ -1125,6 +1270,23 @@ class ArgoWorkflows(object):
|
|
|
1125
1270
|
dag_tasks,
|
|
1126
1271
|
parent_foreach,
|
|
1127
1272
|
)
|
|
1273
|
+
elif node.type == "split-switch":
|
|
1274
|
+
for n in node.out_funcs:
|
|
1275
|
+
_visit(
|
|
1276
|
+
self.graph[n],
|
|
1277
|
+
self._matching_conditional_join(node),
|
|
1278
|
+
templates,
|
|
1279
|
+
dag_tasks,
|
|
1280
|
+
parent_foreach,
|
|
1281
|
+
)
|
|
1282
|
+
|
|
1283
|
+
return _visit(
|
|
1284
|
+
self.graph[self._matching_conditional_join(node)],
|
|
1285
|
+
exit_node,
|
|
1286
|
+
templates,
|
|
1287
|
+
dag_tasks,
|
|
1288
|
+
parent_foreach,
|
|
1289
|
+
)
|
|
1128
1290
|
# For foreach nodes generate a new sub DAGTemplate
|
|
1129
1291
|
# We do this for "regular" foreaches (ie. `self.next(self.a, foreach=)`)
|
|
1130
1292
|
elif node.type == "foreach":
|
|
@@ -1152,7 +1314,7 @@ class ArgoWorkflows(object):
|
|
|
1152
1314
|
#
|
|
1153
1315
|
foreach_task = (
|
|
1154
1316
|
DAGTask(foreach_template_name)
|
|
1155
|
-
.
|
|
1317
|
+
.depends(f"{self._sanitize(node.name)}.Succeeded")
|
|
1156
1318
|
.template(foreach_template_name)
|
|
1157
1319
|
.arguments(
|
|
1158
1320
|
Arguments().parameters(
|
|
@@ -1197,6 +1359,16 @@ class ArgoWorkflows(object):
|
|
|
1197
1359
|
% self._sanitize(node.name)
|
|
1198
1360
|
)
|
|
1199
1361
|
)
|
|
1362
|
+
# Add conditional if this is the first step in a conditional branch
|
|
1363
|
+
if self._is_conditional_node(node) and not any(
|
|
1364
|
+
self._is_conditional_node(self.graph[in_func])
|
|
1365
|
+
for in_func in node.in_funcs
|
|
1366
|
+
):
|
|
1367
|
+
in_func = node.in_funcs[0]
|
|
1368
|
+
foreach_task.when(
|
|
1369
|
+
"{{tasks.%s.outputs.parameters.switch-step}}==%s"
|
|
1370
|
+
% (self._sanitize(in_func), node.name)
|
|
1371
|
+
)
|
|
1200
1372
|
dag_tasks.append(foreach_task)
|
|
1201
1373
|
templates, dag_tasks_1 = _visit(
|
|
1202
1374
|
self.graph[node.out_funcs[0]],
|
|
@@ -1240,7 +1412,22 @@ class ArgoWorkflows(object):
|
|
|
1240
1412
|
self.graph[node.matching_join].in_funcs[0]
|
|
1241
1413
|
)
|
|
1242
1414
|
}
|
|
1243
|
-
|
|
1415
|
+
if not self._is_conditional_join_node(
|
|
1416
|
+
self.graph[node.matching_join]
|
|
1417
|
+
)
|
|
1418
|
+
else
|
|
1419
|
+
# Note: If the nodes leading to the join are conditional, then we need to use an expression to pick the outputs from the task that executed.
|
|
1420
|
+
# ref for operators: https://github.com/expr-lang/expr/blob/master/docs/language-definition.md
|
|
1421
|
+
{
|
|
1422
|
+
"expression": "get((%s)?.parameters, 'task-id')"
|
|
1423
|
+
% " ?? ".join(
|
|
1424
|
+
f"tasks['{self._sanitize(func)}']?.outputs"
|
|
1425
|
+
for func in self.graph[
|
|
1426
|
+
node.matching_join
|
|
1427
|
+
].in_funcs
|
|
1428
|
+
)
|
|
1429
|
+
}
|
|
1430
|
+
),
|
|
1244
1431
|
]
|
|
1245
1432
|
if not node.parallel_foreach
|
|
1246
1433
|
else [
|
|
@@ -1273,7 +1460,7 @@ class ArgoWorkflows(object):
|
|
|
1273
1460
|
join_foreach_task = (
|
|
1274
1461
|
DAGTask(self._sanitize(self.graph[node.matching_join].name))
|
|
1275
1462
|
.template(self._sanitize(self.graph[node.matching_join].name))
|
|
1276
|
-
.
|
|
1463
|
+
.depends(f"{foreach_template_name}.Succeeded")
|
|
1277
1464
|
.arguments(
|
|
1278
1465
|
Arguments().parameters(
|
|
1279
1466
|
(
|
|
@@ -1400,6 +1587,14 @@ class ArgoWorkflows(object):
|
|
|
1400
1587
|
input_paths_expr = (
|
|
1401
1588
|
"export INPUT_PATHS={{inputs.parameters.input-paths}}"
|
|
1402
1589
|
)
|
|
1590
|
+
if self._is_conditional_join_node(node):
|
|
1591
|
+
# NOTE: Argo template expressions that fail to resolve, output the expression itself as a value.
|
|
1592
|
+
# With conditional steps, some of the input-paths are therefore 'broken' due to containing a nil expression
|
|
1593
|
+
# e.g. "{{ tasks['A'].outputs.parameters.task-id }}" when task A never executed.
|
|
1594
|
+
# We base64 encode the input-paths in order to not pollute the execution environment with templating expressions.
|
|
1595
|
+
# NOTE: Adding conditionals that check if a key exists or not does not work either, due to an issue with how Argo
|
|
1596
|
+
# handles tasks in a nested foreach (withParam template) leading to all such expressions getting evaluated as false.
|
|
1597
|
+
input_paths_expr = "export INPUT_PATHS={{=toBase64(inputs.parameters['input-paths'])}}"
|
|
1403
1598
|
input_paths = "$(echo $INPUT_PATHS)"
|
|
1404
1599
|
if any(self.graph[n].type == "foreach" for n in node.in_funcs):
|
|
1405
1600
|
task_idx = "{{inputs.parameters.split-index}}"
|
|
@@ -1415,7 +1610,6 @@ class ArgoWorkflows(object):
|
|
|
1415
1610
|
# foreaches
|
|
1416
1611
|
task_idx = "{{inputs.parameters.split-index}}"
|
|
1417
1612
|
root_input = "{{inputs.parameters.root-input-path}}"
|
|
1418
|
-
|
|
1419
1613
|
# Task string to be hashed into an ID
|
|
1420
1614
|
task_str = "-".join(
|
|
1421
1615
|
[
|
|
@@ -1572,10 +1766,25 @@ class ArgoWorkflows(object):
|
|
|
1572
1766
|
]
|
|
1573
1767
|
)
|
|
1574
1768
|
input_paths = "%s/_parameters/%s" % (run_id, task_id_params)
|
|
1769
|
+
# Only for static joins and conditional_joins
|
|
1770
|
+
elif self._is_conditional_join_node(node) and not (
|
|
1771
|
+
node.type == "join"
|
|
1772
|
+
and self.graph[node.split_parents[-1]].type == "foreach"
|
|
1773
|
+
):
|
|
1774
|
+
input_paths = (
|
|
1775
|
+
"$(python -m metaflow.plugins.argo.conditional_input_paths %s)"
|
|
1776
|
+
% input_paths
|
|
1777
|
+
)
|
|
1575
1778
|
elif (
|
|
1576
1779
|
node.type == "join"
|
|
1577
1780
|
and self.graph[node.split_parents[-1]].type == "foreach"
|
|
1578
1781
|
):
|
|
1782
|
+
# foreach-joins straight out of conditional branches are not yet supported
|
|
1783
|
+
if self._is_conditional_join_node(node):
|
|
1784
|
+
raise ArgoWorkflowsException(
|
|
1785
|
+
"Conditionals steps that transition directly into a join step are not currently supported. "
|
|
1786
|
+
"As a workaround, you can add a normal step after the conditional steps that transitions to a join step."
|
|
1787
|
+
)
|
|
1579
1788
|
# Set aggregated input-paths for a for-each join
|
|
1580
1789
|
foreach_step = next(
|
|
1581
1790
|
n for n in node.in_funcs if self.graph[n].is_inside_foreach
|
|
@@ -1818,7 +2027,7 @@ class ArgoWorkflows(object):
|
|
|
1818
2027
|
[Parameter("num-parallel"), Parameter("task-id-entropy")]
|
|
1819
2028
|
)
|
|
1820
2029
|
else:
|
|
1821
|
-
# append
|
|
2030
|
+
# append these only for joins of foreaches, not static splits
|
|
1822
2031
|
inputs.append(Parameter("split-cardinality"))
|
|
1823
2032
|
# check if the node is a @parallel node.
|
|
1824
2033
|
elif node.parallel_step:
|
|
@@ -1853,6 +2062,13 @@ class ArgoWorkflows(object):
|
|
|
1853
2062
|
# are derived at runtime.
|
|
1854
2063
|
if not (node.name == "end" or node.parallel_step):
|
|
1855
2064
|
outputs = [Parameter("task-id").valueFrom({"path": "/mnt/out/task_id"})]
|
|
2065
|
+
|
|
2066
|
+
# If this step is a split-switch one, we need to output the switch step name
|
|
2067
|
+
if node.type == "split-switch":
|
|
2068
|
+
outputs.append(
|
|
2069
|
+
Parameter("switch-step").valueFrom({"path": "/mnt/out/switch_step"})
|
|
2070
|
+
)
|
|
2071
|
+
|
|
1856
2072
|
if node.type == "foreach":
|
|
1857
2073
|
# Emit split cardinality from foreach task
|
|
1858
2074
|
outputs.append(
|
|
@@ -4027,6 +4243,10 @@ class DAGTask(object):
|
|
|
4027
4243
|
self.payload["dependencies"] = dependencies
|
|
4028
4244
|
return self
|
|
4029
4245
|
|
|
4246
|
+
def depends(self, depends: str):
|
|
4247
|
+
self.payload["depends"] = depends
|
|
4248
|
+
return self
|
|
4249
|
+
|
|
4030
4250
|
def template(self, template):
|
|
4031
4251
|
# Template reference
|
|
4032
4252
|
self.payload["template"] = template
|
|
@@ -4038,6 +4258,10 @@ class DAGTask(object):
|
|
|
4038
4258
|
self.payload["inline"] = template.to_json()
|
|
4039
4259
|
return self
|
|
4040
4260
|
|
|
4261
|
+
def when(self, when: str):
|
|
4262
|
+
self.payload["when"] = when
|
|
4263
|
+
return self
|
|
4264
|
+
|
|
4041
4265
|
def with_param(self, with_param):
|
|
4042
4266
|
self.payload["withParam"] = with_param
|
|
4043
4267
|
return self
|
|
@@ -123,6 +123,15 @@ class ArgoWorkflowsInternalDecorator(StepDecorator):
|
|
|
123
123
|
with open("/mnt/out/split_cardinality", "w") as file:
|
|
124
124
|
json.dump(flow._foreach_num_splits, file)
|
|
125
125
|
|
|
126
|
+
# For conditional branches we need to record the value of the switch to disk, in order to pass it as an
|
|
127
|
+
# output from the switching step to be used further down the DAG
|
|
128
|
+
if graph[step_name].type == "split-switch":
|
|
129
|
+
# TODO: A nicer way to access the chosen step?
|
|
130
|
+
_out_funcs, _ = flow._transition
|
|
131
|
+
chosen_step = _out_funcs[0]
|
|
132
|
+
with open("/mnt/out/switch_step", "w") as file:
|
|
133
|
+
file.write(chosen_step)
|
|
134
|
+
|
|
126
135
|
# For steps that have a `@parallel` decorator set to them, we will be relying on Jobsets
|
|
127
136
|
# to run the task. In this case, we cannot set anything in the
|
|
128
137
|
# `/mnt/out` directory, since such form of output mounts are not available to Jobset executions.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
from math import inf
|
|
2
|
+
import sys
|
|
3
|
+
from metaflow.util import decompress_list, compress_list
|
|
4
|
+
import base64
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def generate_input_paths(input_paths):
|
|
8
|
+
# => run_id/step/:foo,bar
|
|
9
|
+
# input_paths are base64 encoded due to Argo shenanigans
|
|
10
|
+
decoded = base64.b64decode(input_paths).decode("utf-8")
|
|
11
|
+
paths = decompress_list(decoded)
|
|
12
|
+
|
|
13
|
+
# some of the paths are going to be malformed due to never having executed per conditional.
|
|
14
|
+
# strip these out of the list.
|
|
15
|
+
|
|
16
|
+
trimmed = [path for path in paths if not "{{" in path]
|
|
17
|
+
return compress_list(trimmed, zlibmin=inf)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
if __name__ == "__main__":
|
|
21
|
+
print(generate_input_paths(sys.argv[1]))
|
|
@@ -317,6 +317,12 @@ class StepFunctions(object):
|
|
|
317
317
|
"to AWS Step Functions is not supported currently."
|
|
318
318
|
)
|
|
319
319
|
|
|
320
|
+
if node.type == "split-switch":
|
|
321
|
+
raise StepFunctionsException(
|
|
322
|
+
"Deploying flows with switch statement "
|
|
323
|
+
"to AWS Step Functions is not supported currently."
|
|
324
|
+
)
|
|
325
|
+
|
|
320
326
|
# Assign an AWS Batch job to the AWS Step Functions state
|
|
321
327
|
# and pass the intermediate state by exposing `JobId` and
|
|
322
328
|
# `Parameters` to the child job(s) as outputs. `Index` and
|
|
@@ -20,12 +20,15 @@ def transform_flow_graph(step_info):
|
|
|
20
20
|
return "split"
|
|
21
21
|
elif node_type == "split-parallel" or node_type == "split-foreach":
|
|
22
22
|
return "foreach"
|
|
23
|
+
elif node_type == "split-switch":
|
|
24
|
+
return "switch"
|
|
23
25
|
return "unknown" # Should never happen
|
|
24
26
|
|
|
25
27
|
graph_dict = {}
|
|
26
28
|
for stepname in step_info:
|
|
27
|
-
|
|
28
|
-
|
|
29
|
+
node_type = node_to_type(step_info[stepname]["type"])
|
|
30
|
+
node_info = {
|
|
31
|
+
"type": node_type,
|
|
29
32
|
"box_next": step_info[stepname]["type"] not in ("linear", "join"),
|
|
30
33
|
"box_ends": (
|
|
31
34
|
None
|
|
@@ -35,6 +38,15 @@ def transform_flow_graph(step_info):
|
|
|
35
38
|
"next": step_info[stepname]["next"],
|
|
36
39
|
"doc": step_info[stepname]["doc"],
|
|
37
40
|
}
|
|
41
|
+
|
|
42
|
+
if node_type == "switch":
|
|
43
|
+
if "condition" in step_info[stepname]:
|
|
44
|
+
node_info["condition"] = step_info[stepname]["condition"]
|
|
45
|
+
if "switch_cases" in step_info[stepname]:
|
|
46
|
+
node_info["switch_cases"] = step_info[stepname]["switch_cases"]
|
|
47
|
+
|
|
48
|
+
graph_dict[stepname] = node_info
|
|
49
|
+
|
|
38
50
|
return graph_dict
|
|
39
51
|
|
|
40
52
|
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
@import"https://fonts.googleapis.com/css2?family=Roboto:wght@300;400;500;700&display=swap";code[class*=language-],pre[class*=language-]{color:#000;background:0 0;text-shadow:0 1px #fff;font-family:Consolas,Monaco,Andale Mono,Ubuntu Mono,monospace;font-size:1em;text-align:left;white-space:pre;word-spacing:normal;word-break:normal;word-wrap:normal;line-height:1.5;-moz-tab-size:4;-o-tab-size:4;tab-size:4;-webkit-hyphens:none;-moz-hyphens:none;-ms-hyphens:none;hyphens:none}code[class*=language-] ::-moz-selection,code[class*=language-]::-moz-selection,pre[class*=language-] ::-moz-selection,pre[class*=language-]::-moz-selection{text-shadow:none;background:#b3d4fc}code[class*=language-] ::selection,code[class*=language-]::selection,pre[class*=language-] ::selection,pre[class*=language-]::selection{text-shadow:none;background:#b3d4fc}@media print{code[class*=language-],pre[class*=language-]{text-shadow:none}}pre[class*=language-]{padding:1em;margin:.5em 0;overflow:auto}:not(pre)>code[class*=language-],pre[class*=language-]{background:#f5f2f0}:not(pre)>code[class*=language-]{padding:.1em;border-radius:.3em;white-space:normal}.token.cdata,.token.comment,.token.doctype,.token.prolog{color:#708090}.token.punctuation{color:#999}.token.namespace{opacity:.7}.token.boolean,.token.constant,.token.deleted,.token.number,.token.property,.token.symbol,.token.tag{color:#905}.token.attr-name,.token.builtin,.token.char,.token.inserted,.token.selector,.token.string{color:#690}.language-css .token.string,.style .token.string,.token.entity,.token.operator,.token.url{color:#9a6e3a;background:#ffffff80}.token.atrule,.token.attr-value,.token.keyword{color:#07a}.token.class-name,.token.function{color:#dd4a68}.token.important,.token.regex,.token.variable{color:#e90}.token.bold,.token.important{font-weight:700}.token.italic{font-style:italic}.token.entity{cursor:help}:root{--bg: #ffffff;--black: #333;--blue: #0c66de;--dk-grey: #767676;--dk-primary: #ef863b;--dk-secondary: #13172d;--dk-tertiary: #0f426e;--error: #cf483e;--grey: rgba(0, 0, 0, .125);--highlight: #f8d9d8;--lt-blue: #4fa7ff;--lt-grey: #f3f3f3;--lt-lt-grey: #f9f9f9;--lt-primary: #ffcb8b;--lt-secondary: #434d81;--lt-tertiary: #4189c9;--primary: #faab4a;--quadrary: #f8d9d8;--secondary: #2e3454;--tertiary: #2a679d;--white: #ffffff;--component-spacer: 3rem;--aside-width: 20rem;--embed-card-min-height: 12rem;--mono-font: ui-monospace, Menlo, Monaco, "Cascadia Mono", "Segoe UI Mono", "Roboto Mono", "Oxygen Mono", "Ubuntu Monospace", "Source Code Pro", "Fira Mono", "Droid Sans Mono", "Courier New", monospace}html,body{margin:0;min-height:100vh;overflow-y:visible;padding:0;width:100%}.card_app{width:100%;min-height:100vh}.embed .card_app{min-height:var(--embed-card-min-height)}.mf-card *{box-sizing:border-box}.mf-card{background:var(--bg);color:var(--black);font-family:Roboto,-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Oxygen,Ubuntu,Cantarell,Fira Sans,Droid Sans,Helvetica Neue,sans-serif;font-size:14px;font-weight:400;line-height:1.5;text-size-adjust:100%;margin:0;min-height:100vh;overflow-y:visible;padding:0;text-align:left;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;width:100%}.embed .mf-card{min-height:var(--embed-card-min-height)}.mf-card :is(.mono,code.mono,pre.mono){font-family:var(--mono-font);font-weight:lighter}.mf-card :is(table,th,td){border-spacing:1px;text-align:center;color:var(--black)}.mf-card table{position:relative;min-width:100%;table-layout:inherit!important}.mf-card td{padding:.66rem 1.25rem;background:var(--lt-lt-grey);border:none}.mf-card th{border:none;color:var(--dk-grey);font-weight:400;padding:.5rem}.mf-card :is(h1,h2,h3,h4,h5){font-weight:700;margin:.5rem 0}.mf-card ul{margin:0;padding:0}.mf-card p{margin:0 0 1rem}.mf-card p:last-of-type{margin:0}.mf-card button{font-size:1rem}.mf-card .textButton{cursor:pointer;text-align:left;background:none;border:1px solid transparent;outline:none;padding:0}.mf-card :is(button.textButton:focus,a:focus,button.textButton:active){border:1px dashed var(--grey);background:transparent}.mf-card button.textButton:hover{color:var(--blue);text-decoration:none}.mf-card :is(:not(pre)>code[class*=language-],pre[class*=language-]){background:transparent!important;text-shadow:none;-webkit-user-select:auto;user-select:auto}aside.svelte-1okdv0e{display:none;line-height:2;text-align:left}@media (min-width: 60rem){aside.svelte-1okdv0e{display:flex;flex-direction:column;height:100vh;justify-content:space-between;padding:2.5rem 0 1.5rem 1.5rem;position:fixed;width:var(--aside-width)}}.embed aside{display:none}aside ul{list-style-type:none}aside a,aside button,aside a:visited{text-decoration:none;cursor:pointer;font-weight:700;color:var(--black)}aside a:hover,aside button:hover{text-decoration:underline}.logoContainer svg{width:100%;max-width:140px;margin-bottom:3.75rem;height:auto}.idCell.svelte-pt8vzv{font-weight:700;text-align:right;background:var(--lt-grey);width:12%}.codeCell.svelte-pt8vzv{text-align:left;-webkit-user-select:all;user-select:all}.container.svelte-ubs992{width:100%;overflow:auto}table.svelte-ubs992{width:100%}:root{--dag-border: #282828;--dag-bg-static: var(--lt-grey);--dag-bg-success: #a5d46a;--dag-bg-running: #ffdf80;--dag-bg-error: #ffa080;--dag-connector: #cccccc;--dag-gap: 5rem;--dag-step-height: 6.25rem;--dag-step-width: 11.25rem;--dag-selected: #ffd700}.connectorwrapper.svelte-1hyaq5f{transform-origin:0 0;position:absolute;z-index:0;min-width:var(--strokeWidth)}.flip.svelte-1hyaq5f{transform:scaleX(-1)}.path.svelte-1hyaq5f{--strokeWidth: .5rem;--strokeColor: var(--dag-connector);--borderRadius: 1.25rem;box-sizing:border-box}.straightLine.svelte-1hyaq5f{position:absolute;inset:0;border-left:var(--strokeWidth) solid var(--strokeColor)}.topLeft.svelte-1hyaq5f{position:absolute;top:0;left:0;right:50%;bottom:calc(var(--dag-gap) / 2 - var(--strokeWidth) / 2);border-radius:0 0 0 var(--borderRadius);border-left:var(--strokeWidth) solid var(--strokeColor);border-bottom:var(--strokeWidth) solid var(--strokeColor)}.bottomRight.svelte-1hyaq5f{position:absolute;top:calc(100% - (var(--dag-gap) / 2 + var(--strokeWidth) / 2));left:50%;right:0;bottom:0;border-radius:0 var(--borderRadius) 0 0;border-top:var(--strokeWidth) solid var(--strokeColor);border-right:var(--strokeWidth) solid var(--strokeColor)}.wrapper.svelte-117ceti{position:relative;z-index:1}.step.svelte-117ceti{font-size:.75rem;padding:.5rem;color:var(--dk-grey)}.rectangle.svelte-117ceti{background-color:var(--dag-bg-static);border:1px solid var(--dag-border);box-sizing:border-box;position:relative;height:var(--dag-step-height);width:var(--dag-step-width)}.rectangle.error.svelte-117ceti{background-color:var(--dag-bg-error)}.rectangle.success.svelte-117ceti{background-color:var(--dag-bg-success)}.rectangle.running.svelte-117ceti{background-color:var(--dag-bg-running)}.level.svelte-117ceti{z-index:-1;filter:contrast(.5);position:absolute}.inner.svelte-117ceti{position:relative;height:100%;width:100%}.name.svelte-117ceti{font-weight:700;overflow:hidden;text-overflow:ellipsis;display:block}.description.svelte-117ceti{position:absolute;max-height:4rem;bottom:0;left:0;right:0;overflow:hidden;-webkit-line-clamp:4;line-clamp:4;display:-webkit-box;-webkit-box-orient:vertical}.overflown.description.svelte-117ceti{cursor:help}.current.svelte-117ceti .rectangle:where(.svelte-117ceti){box-shadow:0 0 10px var(--dag-selected)}.levelstoshow.svelte-117ceti{position:absolute;bottom:100%;right:0;font-size:.75rem;font-weight:100;text-align:right}.stepwrapper.svelte-18aex7a{display:flex;align-items:center;flex-direction:column;width:100%;position:relative;min-width:var(--dag-step-width)}.childwrapper.svelte-18aex7a{display:flex;width:100%}.gap.svelte-18aex7a{height:var(--dag-gap)}.title.svelte-117s0ws{text-align:left}.subtitle.svelte-lu9pnn{font-size:1rem;text-align:left}header.svelte-1ugmt5d{margin-bottom:var(--component-spacer)}figure.svelte-1x96yvr{background:var(--lt-grey);padding:1rem;border-radius:5px;text-align:center;margin:0 auto var(--component-spacer)}@media (min-width: 60rem){figure.svelte-1x96yvr{margin-bottom:0}}img.svelte-1x96yvr{max-width:100%;max-height:500px}.label.svelte-1x96yvr{font-weight:700;margin:.5rem 0}.description.svelte-1x96yvr{font-size:.9rem;font-style:italic;text-align:center;margin:.5rem 0}.log.svelte-1jhmsu{background:var(--lt-grey)!important;font-size:.9rem;padding:2rem}.page.svelte-v7ihqd:last-of-type{margin-bottom:var(--component-spacer)}.page:last-of-type section:last-of-type hr{display:none}progress.svelte-ljrmzp::-webkit-progress-bar{background-color:#fff!important;min-width:100%}progress.svelte-ljrmzp{background-color:#fff;color:#326cded9!important}progress.svelte-ljrmzp::-moz-progress-bar{background-color:#326cde!important}table .container{background:transparent!important;font-size:10px!important;padding:0!important}table progress{height:4px!important}.container.svelte-ljrmzp{display:flex;align-items:center;justify-content:center;font-size:12px;border-radius:3px;background:#edf5ff;padding:3rem}.inner.svelte-ljrmzp{max-width:410px;width:100%;text-align:center}.info.svelte-ljrmzp{display:flex;justify-content:space-between}table .info{text-align:left;flex-direction:column}label.svelte-ljrmzp{font-weight:700}.labelValue.svelte-ljrmzp{border-left:1px solid rgba(0,0,0,.1);margin-left:.25rem;padding-left:.5rem}.details.svelte-ljrmzp{font-family:var(--mono-font);font-size:8px;color:#333433;line-height:18px;overflow:hidden;white-space:nowrap}progress.svelte-ljrmzp{width:100%;border:none;border-radius:5px;height:8px;background:#fff}.heading.svelte-17n0qr8{margin-bottom:1.5rem}.sectionItems.svelte-17n0qr8{display:block}.sectionItems .imageContainer{max-height:500px}.container.svelte-17n0qr8{scroll-margin:var(--component-spacer)}hr.svelte-17n0qr8{background:var(--grey);border:none;height:1px;margin:var(--component-spacer) 0;padding:0}@media (min-width: 60rem){.sectionItems.svelte-17n0qr8{display:grid;grid-gap:2rem}}td.svelte-gl9h79{text-align:left}td.labelColumn.svelte-gl9h79{text-align:right;background-color:var(--lt-grey);font-weight:700;width:12%;white-space:nowrap}.tableContainer.svelte-q3hq57{overflow:auto}th.svelte-q3hq57{position:sticky;top:-1px;z-index:2;white-space:nowrap;background:var(--white)}.mainContainer.svelte-mqeomk{max-width:110rem}main.svelte-mqeomk{flex:0 1 auto;max-width:100rem;padding:1.5rem}@media (min-width: 60rem){main.svelte-mqeomk{margin-left:var(--aside-width)}}.embed main{margin:0 auto;min-width:80%}.modal.svelte-1hhf5ym{align-items:center;background:#00000080;cursor:pointer;display:flex;height:100%;justify-content:center;inset:0;overflow:hidden;position:fixed;width:100%;z-index:100}.modalContainer>*{background-color:#fff;border-radius:5px;cursor:default;flex:0 1 auto;padding:1rem;position:relative}.modal img{max-height:80vh!important}.cancelButton.svelte-1hhf5ym{color:#fff;cursor:pointer;font-size:2rem;position:absolute;right:1rem;top:1rem}.cancelButton.svelte-1hhf5ym:hover{color:var(--blue)}.nav.svelte-1kdpgko{border-radius:0 0 5px;display:none;margin:0;top:0}ul.navList.svelte-1kdpgko{list-style-type:none}ul.navList.svelte-1kdpgko ul:where(.svelte-1kdpgko){margin:.5rem 1rem 2rem}.navList.svelte-1kdpgko li:where(.svelte-1kdpgko){display:block;margin:0}.navItem.svelte-1kdpgko li:where(.svelte-1kdpgko):hover{color:var(--blue)}.pageId.svelte-1kdpgko{display:block;border-bottom:1px solid var(--grey);padding:0 .5rem;margin-bottom:1rem}@media (min-width: 60rem){.nav.svelte-1kdpgko{display:block}ul.navList.svelte-1kdpgko{text-align:left}.navList.svelte-1kdpgko li:where(.svelte-1kdpgko){display:block;margin:.5rem 0}}.container.svelte-teyund{width:100%;display:flex;flex-direction:column;position:relative}
|