metaflow 2.12.29__py2.py3-none-any.whl → 2.12.31__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow/cli.py +13 -23
- metaflow/parameters.py +8 -2
- metaflow/plugins/argo/argo_workflows.py +8 -4
- metaflow/plugins/events_decorator.py +253 -72
- metaflow/plugins/kubernetes/kubernetes_jobsets.py +43 -28
- metaflow/version.py +1 -1
- {metaflow-2.12.29.dist-info → metaflow-2.12.31.dist-info}/METADATA +2 -2
- {metaflow-2.12.29.dist-info → metaflow-2.12.31.dist-info}/RECORD +12 -12
- {metaflow-2.12.29.dist-info → metaflow-2.12.31.dist-info}/WHEEL +1 -1
- {metaflow-2.12.29.dist-info → metaflow-2.12.31.dist-info}/LICENSE +0 -0
- {metaflow-2.12.29.dist-info → metaflow-2.12.31.dist-info}/entry_points.txt +0 -0
- {metaflow-2.12.29.dist-info → metaflow-2.12.31.dist-info}/top_level.txt +0 -0
metaflow/cli.py
CHANGED
@@ -282,31 +282,21 @@ def dump(obj, input_path, private=None, max_value_size=None, include=None, file=
|
|
282
282
|
else:
|
283
283
|
ds_list = list(datastore_set) # get all tasks
|
284
284
|
|
285
|
-
tasks_processed = False
|
286
285
|
for ds in ds_list:
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
fg="magenta",
|
295
|
-
)
|
296
|
-
|
297
|
-
if file is None:
|
298
|
-
echo_always(
|
299
|
-
ds.format(**kwargs),
|
300
|
-
highlight="green",
|
301
|
-
highlight_bold=False,
|
302
|
-
err=False,
|
303
|
-
)
|
304
|
-
else:
|
305
|
-
output[ds.pathspec] = ds.to_dict(**kwargs)
|
286
|
+
echo(
|
287
|
+
"Dumping output of run_id=*{run_id}* "
|
288
|
+
"step=*{step}* task_id=*{task_id}*".format(
|
289
|
+
run_id=ds.run_id, step=ds.step_name, task_id=ds.task_id
|
290
|
+
),
|
291
|
+
fg="magenta",
|
292
|
+
)
|
306
293
|
|
307
|
-
|
308
|
-
|
309
|
-
|
294
|
+
if file is None:
|
295
|
+
echo_always(
|
296
|
+
ds.format(**kwargs), highlight="green", highlight_bold=False, err=False
|
297
|
+
)
|
298
|
+
else:
|
299
|
+
output[ds.pathspec] = ds.to_dict(**kwargs)
|
310
300
|
|
311
301
|
if file is not None:
|
312
302
|
with open(file, "wb") as f:
|
metaflow/parameters.py
CHANGED
@@ -151,6 +151,7 @@ class DeployTimeField(object):
|
|
151
151
|
return self._check_type(val, deploy_time)
|
152
152
|
|
153
153
|
def _check_type(self, val, deploy_time):
|
154
|
+
|
154
155
|
# it is easy to introduce a deploy-time function that accidentally
|
155
156
|
# returns a value whose type is not compatible with what is defined
|
156
157
|
# in Parameter. Let's catch those mistakes early here, instead of
|
@@ -158,7 +159,7 @@ class DeployTimeField(object):
|
|
158
159
|
|
159
160
|
# note: this doesn't work with long in Python2 or types defined as
|
160
161
|
# click types, e.g. click.INT
|
161
|
-
TYPES = {bool: "bool", int: "int", float: "float", list: "list"}
|
162
|
+
TYPES = {bool: "bool", int: "int", float: "float", list: "list", dict: "dict"}
|
162
163
|
|
163
164
|
msg = (
|
164
165
|
"The value returned by the deploy-time function for "
|
@@ -166,7 +167,12 @@ class DeployTimeField(object):
|
|
166
167
|
% (self.parameter_name, self.field)
|
167
168
|
)
|
168
169
|
|
169
|
-
if self.parameter_type
|
170
|
+
if isinstance(self.parameter_type, list):
|
171
|
+
if not any(isinstance(val, x) for x in self.parameter_type):
|
172
|
+
msg += "Expected one of the following %s." % TYPES[self.parameter_type]
|
173
|
+
raise ParameterFieldTypeMismatch(msg)
|
174
|
+
return str(val) if self.return_str else val
|
175
|
+
elif self.parameter_type in TYPES:
|
170
176
|
if type(val) != self.parameter_type:
|
171
177
|
msg += "Expected a %s." % TYPES[self.parameter_type]
|
172
178
|
raise ParameterFieldTypeMismatch(msg)
|
@@ -522,7 +522,9 @@ class ArgoWorkflows(object):
|
|
522
522
|
params = set(
|
523
523
|
[param.name.lower() for var, param in self.flow._get_parameters()]
|
524
524
|
)
|
525
|
-
|
525
|
+
trigger_deco = self.flow._flow_decorators.get("trigger")[0]
|
526
|
+
trigger_deco.format_deploytime_value()
|
527
|
+
for event in trigger_deco.triggers:
|
526
528
|
parameters = {}
|
527
529
|
# TODO: Add a check to guard against names starting with numerals(?)
|
528
530
|
if not re.match(r"^[A-Za-z0-9_.-]+$", event["name"]):
|
@@ -562,9 +564,11 @@ class ArgoWorkflows(object):
|
|
562
564
|
|
563
565
|
# @trigger_on_finish decorator
|
564
566
|
if self.flow._flow_decorators.get("trigger_on_finish"):
|
565
|
-
|
566
|
-
|
567
|
-
]
|
567
|
+
trigger_on_finish_deco = self.flow._flow_decorators.get(
|
568
|
+
"trigger_on_finish"
|
569
|
+
)[0]
|
570
|
+
trigger_on_finish_deco.format_deploytime_value()
|
571
|
+
for event in trigger_on_finish_deco.triggers:
|
568
572
|
# Actual filters are deduced here since we don't have access to
|
569
573
|
# the current object in the @trigger_on_finish decorator.
|
570
574
|
triggers.append(
|
@@ -1,9 +1,11 @@
|
|
1
1
|
import re
|
2
|
+
import json
|
2
3
|
|
3
4
|
from metaflow import current
|
4
5
|
from metaflow.decorators import FlowDecorator
|
5
6
|
from metaflow.exception import MetaflowException
|
6
7
|
from metaflow.util import is_stringish
|
8
|
+
from metaflow.parameters import DeployTimeField, deploy_time_eval
|
7
9
|
|
8
10
|
# TODO: Support dynamic parameter mapping through a context object that exposes
|
9
11
|
# flow name and user name similar to parameter context
|
@@ -68,6 +70,75 @@ class TriggerDecorator(FlowDecorator):
|
|
68
70
|
"options": {},
|
69
71
|
}
|
70
72
|
|
73
|
+
def process_event_name(self, event):
|
74
|
+
if is_stringish(event):
|
75
|
+
return {"name": str(event)}
|
76
|
+
elif isinstance(event, dict):
|
77
|
+
if "name" not in event:
|
78
|
+
raise MetaflowException(
|
79
|
+
"The *event* attribute for *@trigger* is missing the *name* key."
|
80
|
+
)
|
81
|
+
if callable(event["name"]) and not isinstance(
|
82
|
+
event["name"], DeployTimeField
|
83
|
+
):
|
84
|
+
event["name"] = DeployTimeField(
|
85
|
+
"event_name", str, None, event["name"], False
|
86
|
+
)
|
87
|
+
event["parameters"] = self.process_parameters(event.get("parameters", {}))
|
88
|
+
return event
|
89
|
+
elif callable(event) and not isinstance(event, DeployTimeField):
|
90
|
+
return DeployTimeField("event", [str, dict], None, event, False)
|
91
|
+
else:
|
92
|
+
raise MetaflowException(
|
93
|
+
"Incorrect format for *event* attribute in *@trigger* decorator. "
|
94
|
+
"Supported formats are string and dictionary - \n"
|
95
|
+
"@trigger(event='foo') or @trigger(event={'name': 'foo', "
|
96
|
+
"'parameters': {'alpha': 'beta'}})"
|
97
|
+
)
|
98
|
+
|
99
|
+
def process_parameters(self, parameters):
|
100
|
+
new_param_values = {}
|
101
|
+
if isinstance(parameters, (list, tuple)):
|
102
|
+
for mapping in parameters:
|
103
|
+
if is_stringish(mapping):
|
104
|
+
new_param_values[mapping] = mapping
|
105
|
+
elif callable(mapping) and not isinstance(mapping, DeployTimeField):
|
106
|
+
mapping = DeployTimeField(
|
107
|
+
"parameter_val", str, None, mapping, False
|
108
|
+
)
|
109
|
+
new_param_values[mapping] = mapping
|
110
|
+
elif isinstance(mapping, (list, tuple)) and len(mapping) == 2:
|
111
|
+
if callable(mapping[0]) and not isinstance(
|
112
|
+
mapping[0], DeployTimeField
|
113
|
+
):
|
114
|
+
mapping[0] = DeployTimeField(
|
115
|
+
"parameter_val", str, None, mapping[0], False
|
116
|
+
)
|
117
|
+
if callable(mapping[1]) and not isinstance(
|
118
|
+
mapping[1], DeployTimeField
|
119
|
+
):
|
120
|
+
mapping[1] = DeployTimeField(
|
121
|
+
"parameter_val", str, None, mapping[1], False
|
122
|
+
)
|
123
|
+
new_param_values[mapping[0]] = mapping[1]
|
124
|
+
else:
|
125
|
+
raise MetaflowException(
|
126
|
+
"The *parameters* attribute for event is invalid. "
|
127
|
+
"It should be a list/tuple of strings and lists/tuples of size 2"
|
128
|
+
)
|
129
|
+
elif callable(parameters) and not isinstance(parameters, DeployTimeField):
|
130
|
+
return DeployTimeField(
|
131
|
+
"parameters", [list, dict, tuple], None, parameters, False
|
132
|
+
)
|
133
|
+
elif isinstance(parameters, dict):
|
134
|
+
for key, value in parameters.items():
|
135
|
+
if callable(key) and not isinstance(key, DeployTimeField):
|
136
|
+
key = DeployTimeField("flow_parameter", str, None, key, False)
|
137
|
+
if callable(value) and not isinstance(value, DeployTimeField):
|
138
|
+
value = DeployTimeField("signal_parameter", str, None, value, False)
|
139
|
+
new_param_values[key] = value
|
140
|
+
return new_param_values
|
141
|
+
|
71
142
|
def flow_init(
|
72
143
|
self,
|
73
144
|
flow_name,
|
@@ -86,41 +157,9 @@ class TriggerDecorator(FlowDecorator):
|
|
86
157
|
"attributes in *@trigger* decorator."
|
87
158
|
)
|
88
159
|
elif self.attributes["event"]:
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
# 'parameters': {'alpha': 'member_weight'}}
|
93
|
-
if is_stringish(self.attributes["event"]):
|
94
|
-
self.triggers.append({"name": str(self.attributes["event"])})
|
95
|
-
elif isinstance(self.attributes["event"], dict):
|
96
|
-
if "name" not in self.attributes["event"]:
|
97
|
-
raise MetaflowException(
|
98
|
-
"The *event* attribute for *@trigger* is missing the "
|
99
|
-
"*name* key."
|
100
|
-
)
|
101
|
-
param_value = self.attributes["event"].get("parameters", {})
|
102
|
-
if isinstance(param_value, (list, tuple)):
|
103
|
-
new_param_value = {}
|
104
|
-
for mapping in param_value:
|
105
|
-
if is_stringish(mapping):
|
106
|
-
new_param_value[mapping] = mapping
|
107
|
-
elif isinstance(mapping, (list, tuple)) and len(mapping) == 2:
|
108
|
-
new_param_value[mapping[0]] = mapping[1]
|
109
|
-
else:
|
110
|
-
raise MetaflowException(
|
111
|
-
"The *parameters* attribute for event '%s' is invalid. "
|
112
|
-
"It should be a list/tuple of strings and lists/tuples "
|
113
|
-
"of size 2" % self.attributes["event"]["name"]
|
114
|
-
)
|
115
|
-
self.attributes["event"]["parameters"] = new_param_value
|
116
|
-
self.triggers.append(self.attributes["event"])
|
117
|
-
else:
|
118
|
-
raise MetaflowException(
|
119
|
-
"Incorrect format for *event* attribute in *@trigger* decorator. "
|
120
|
-
"Supported formats are string and dictionary - \n"
|
121
|
-
"@trigger(event='foo') or @trigger(event={'name': 'foo', "
|
122
|
-
"'parameters': {'alpha': 'beta'}})"
|
123
|
-
)
|
160
|
+
event = self.attributes["event"]
|
161
|
+
processed_event = self.process_event_name(event)
|
162
|
+
self.triggers.append(processed_event)
|
124
163
|
elif self.attributes["events"]:
|
125
164
|
# events attribute supports the following formats -
|
126
165
|
# 1. events=[{'name': 'table.prod_db.members',
|
@@ -128,43 +167,17 @@ class TriggerDecorator(FlowDecorator):
|
|
128
167
|
# {'name': 'table.prod_db.metadata',
|
129
168
|
# 'parameters': {'beta': 'grade'}}]
|
130
169
|
if isinstance(self.attributes["events"], list):
|
170
|
+
# process every event in events
|
131
171
|
for event in self.attributes["events"]:
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
if isinstance(param_value, (list, tuple)):
|
142
|
-
new_param_value = {}
|
143
|
-
for mapping in param_value:
|
144
|
-
if is_stringish(mapping):
|
145
|
-
new_param_value[mapping] = mapping
|
146
|
-
elif (
|
147
|
-
isinstance(mapping, (list, tuple))
|
148
|
-
and len(mapping) == 2
|
149
|
-
):
|
150
|
-
new_param_value[mapping[0]] = mapping[1]
|
151
|
-
else:
|
152
|
-
raise MetaflowException(
|
153
|
-
"The *parameters* attribute for event '%s' is "
|
154
|
-
"invalid. It should be a list/tuple of strings "
|
155
|
-
"and lists/tuples of size 2" % event["name"]
|
156
|
-
)
|
157
|
-
event["parameters"] = new_param_value
|
158
|
-
self.triggers.append(event)
|
159
|
-
else:
|
160
|
-
raise MetaflowException(
|
161
|
-
"One or more events in *events* attribute in *@trigger* "
|
162
|
-
"decorator have an incorrect format. Supported format "
|
163
|
-
"is dictionary - \n"
|
164
|
-
"@trigger(events=[{'name': 'foo', 'parameters': {'alpha': "
|
165
|
-
"'beta'}}, {'name': 'bar', 'parameters': "
|
166
|
-
"{'gamma': 'kappa'}}])"
|
167
|
-
)
|
172
|
+
processed_event = self.process_event_name(event)
|
173
|
+
self.triggers.append("processed event", processed_event)
|
174
|
+
elif callable(self.attributes["events"]) and not isinstance(
|
175
|
+
self.attributes["events"], DeployTimeField
|
176
|
+
):
|
177
|
+
trig = DeployTimeField(
|
178
|
+
"events", list, None, self.attributes["events"], False
|
179
|
+
)
|
180
|
+
self.triggers.append(trig)
|
168
181
|
else:
|
169
182
|
raise MetaflowException(
|
170
183
|
"Incorrect format for *events* attribute in *@trigger* decorator. "
|
@@ -178,7 +191,12 @@ class TriggerDecorator(FlowDecorator):
|
|
178
191
|
raise MetaflowException("No event(s) specified in *@trigger* decorator.")
|
179
192
|
|
180
193
|
# same event shouldn't occur more than once
|
181
|
-
names = [
|
194
|
+
names = [
|
195
|
+
x["name"]
|
196
|
+
for x in self.triggers
|
197
|
+
if not isinstance(x, DeployTimeField)
|
198
|
+
and not isinstance(x["name"], DeployTimeField)
|
199
|
+
]
|
182
200
|
if len(names) != len(set(names)):
|
183
201
|
raise MetaflowException(
|
184
202
|
"Duplicate event names defined in *@trigger* decorator."
|
@@ -188,6 +206,104 @@ class TriggerDecorator(FlowDecorator):
|
|
188
206
|
|
189
207
|
# TODO: Handle scenario for local testing using --trigger.
|
190
208
|
|
209
|
+
def format_deploytime_value(self):
|
210
|
+
new_triggers = []
|
211
|
+
for trigger in self.triggers:
|
212
|
+
# Case where trigger is a function that returns a list of events
|
213
|
+
# Need to do this bc we need to iterate over list later
|
214
|
+
if isinstance(trigger, DeployTimeField):
|
215
|
+
evaluated_trigger = deploy_time_eval(trigger)
|
216
|
+
if isinstance(evaluated_trigger, dict):
|
217
|
+
trigger = evaluated_trigger
|
218
|
+
elif isinstance(evaluated_trigger, str):
|
219
|
+
trigger = {"name": evaluated_trigger}
|
220
|
+
if isinstance(evaluated_trigger, list):
|
221
|
+
for trig in evaluated_trigger:
|
222
|
+
if is_stringish(trig):
|
223
|
+
new_triggers.append({"name": trig})
|
224
|
+
else: # dict or another deploytimefield
|
225
|
+
new_triggers.append(trig)
|
226
|
+
else:
|
227
|
+
new_triggers.append(trigger)
|
228
|
+
else:
|
229
|
+
new_triggers.append(trigger)
|
230
|
+
|
231
|
+
self.triggers = new_triggers
|
232
|
+
for trigger in self.triggers:
|
233
|
+
old_trigger = trigger
|
234
|
+
trigger_params = trigger.get("parameters", {})
|
235
|
+
# Case where param is a function (can return list or dict)
|
236
|
+
if isinstance(trigger_params, DeployTimeField):
|
237
|
+
trigger_params = deploy_time_eval(trigger_params)
|
238
|
+
# If params is a list of strings, convert to dict with same key and value
|
239
|
+
if isinstance(trigger_params, (list, tuple)):
|
240
|
+
new_trigger_params = {}
|
241
|
+
for mapping in trigger_params:
|
242
|
+
if is_stringish(mapping) or callable(mapping):
|
243
|
+
new_trigger_params[mapping] = mapping
|
244
|
+
elif callable(mapping) and not isinstance(mapping, DeployTimeField):
|
245
|
+
mapping = DeployTimeField(
|
246
|
+
"parameter_val", str, None, mapping, False
|
247
|
+
)
|
248
|
+
new_trigger_params[mapping] = mapping
|
249
|
+
elif isinstance(mapping, (list, tuple)) and len(mapping) == 2:
|
250
|
+
if callable(mapping[0]) and not isinstance(
|
251
|
+
mapping[0], DeployTimeField
|
252
|
+
):
|
253
|
+
mapping[0] = DeployTimeField(
|
254
|
+
"parameter_val",
|
255
|
+
str,
|
256
|
+
None,
|
257
|
+
mapping[1],
|
258
|
+
False,
|
259
|
+
)
|
260
|
+
if callable(mapping[1]) and not isinstance(
|
261
|
+
mapping[1], DeployTimeField
|
262
|
+
):
|
263
|
+
mapping[1] = DeployTimeField(
|
264
|
+
"parameter_val",
|
265
|
+
str,
|
266
|
+
None,
|
267
|
+
mapping[1],
|
268
|
+
False,
|
269
|
+
)
|
270
|
+
|
271
|
+
new_trigger_params[mapping[0]] = mapping[1]
|
272
|
+
else:
|
273
|
+
raise MetaflowException(
|
274
|
+
"The *parameters* attribute for event '%s' is invalid. "
|
275
|
+
"It should be a list/tuple of strings and lists/tuples "
|
276
|
+
"of size 2" % self.attributes["event"]["name"]
|
277
|
+
)
|
278
|
+
trigger_params = new_trigger_params
|
279
|
+
trigger["parameters"] = trigger_params
|
280
|
+
|
281
|
+
trigger_name = trigger.get("name")
|
282
|
+
# Case where just the name is a function (always a str)
|
283
|
+
if isinstance(trigger_name, DeployTimeField):
|
284
|
+
trigger_name = deploy_time_eval(trigger_name)
|
285
|
+
trigger["name"] = trigger_name
|
286
|
+
|
287
|
+
# Third layer
|
288
|
+
# {name:, parameters:[func, ..., ...]}
|
289
|
+
# {name:, parameters:{func : func2}}
|
290
|
+
for trigger in self.triggers:
|
291
|
+
old_trigger = trigger
|
292
|
+
trigger_params = trigger.get("parameters", {})
|
293
|
+
new_trigger_params = {}
|
294
|
+
for key, value in trigger_params.items():
|
295
|
+
if isinstance(value, DeployTimeField) and key is value:
|
296
|
+
evaluated_param = deploy_time_eval(value)
|
297
|
+
new_trigger_params[evaluated_param] = evaluated_param
|
298
|
+
elif isinstance(value, DeployTimeField):
|
299
|
+
new_trigger_params[key] = deploy_time_eval(value)
|
300
|
+
elif isinstance(key, DeployTimeField):
|
301
|
+
new_trigger_params[deploy_time_eval(key)] = value
|
302
|
+
else:
|
303
|
+
new_trigger_params[key] = value
|
304
|
+
trigger["parameters"] = new_trigger_params
|
305
|
+
self.triggers[self.triggers.index(old_trigger)] = trigger
|
306
|
+
|
191
307
|
|
192
308
|
class TriggerOnFinishDecorator(FlowDecorator):
|
193
309
|
"""
|
@@ -312,6 +428,13 @@ class TriggerOnFinishDecorator(FlowDecorator):
|
|
312
428
|
"The *project_branch* attribute of the *flow* is not a string"
|
313
429
|
)
|
314
430
|
self.triggers.append(result)
|
431
|
+
elif callable(self.attributes["flow"]) and not isinstance(
|
432
|
+
self.attributes["flow"], DeployTimeField
|
433
|
+
):
|
434
|
+
trig = DeployTimeField(
|
435
|
+
"fq_name", [str, dict], None, self.attributes["flow"], False
|
436
|
+
)
|
437
|
+
self.triggers.append(trig)
|
315
438
|
else:
|
316
439
|
raise MetaflowException(
|
317
440
|
"Incorrect type for *flow* attribute in *@trigger_on_finish* "
|
@@ -369,6 +492,13 @@ class TriggerOnFinishDecorator(FlowDecorator):
|
|
369
492
|
"Supported type is string or Dict[str, str]- \n"
|
370
493
|
"@trigger_on_finish(flows=['FooFlow', 'BarFlow']"
|
371
494
|
)
|
495
|
+
elif callable(self.attributes["flows"]) and not isinstance(
|
496
|
+
self.attributes["flows"], DeployTimeField
|
497
|
+
):
|
498
|
+
trig = DeployTimeField(
|
499
|
+
"flows", list, None, self.attributes["flows"], False
|
500
|
+
)
|
501
|
+
self.triggers.append(trig)
|
372
502
|
else:
|
373
503
|
raise MetaflowException(
|
374
504
|
"Incorrect type for *flows* attribute in *@trigger_on_finish* "
|
@@ -383,6 +513,8 @@ class TriggerOnFinishDecorator(FlowDecorator):
|
|
383
513
|
|
384
514
|
# Make triggers @project aware
|
385
515
|
for trigger in self.triggers:
|
516
|
+
if isinstance(trigger, DeployTimeField):
|
517
|
+
continue
|
386
518
|
if trigger["fq_name"].count(".") == 0:
|
387
519
|
# fully qualified name is just the flow name
|
388
520
|
trigger["flow"] = trigger["fq_name"]
|
@@ -427,5 +559,54 @@ class TriggerOnFinishDecorator(FlowDecorator):
|
|
427
559
|
run_objs.append(run_obj)
|
428
560
|
current._update_env({"trigger": Trigger.from_runs(run_objs)})
|
429
561
|
|
562
|
+
def _parse_fq_name(self, trigger):
|
563
|
+
if isinstance(trigger, DeployTimeField):
|
564
|
+
trigger["fq_name"] = deploy_time_eval(trigger["fq_name"])
|
565
|
+
if trigger["fq_name"].count(".") == 0:
|
566
|
+
# fully qualified name is just the flow name
|
567
|
+
trigger["flow"] = trigger["fq_name"]
|
568
|
+
elif trigger["fq_name"].count(".") >= 2:
|
569
|
+
# fully qualified name is of the format - project.branch.flow_name
|
570
|
+
trigger["project"], tail = trigger["fq_name"].split(".", maxsplit=1)
|
571
|
+
trigger["branch"], trigger["flow"] = tail.rsplit(".", maxsplit=1)
|
572
|
+
else:
|
573
|
+
raise MetaflowException(
|
574
|
+
"Incorrect format for *flow* in *@trigger_on_finish* "
|
575
|
+
"decorator. Specify either just the *flow_name* or a fully "
|
576
|
+
"qualified name like *project_name.branch_name.flow_name*."
|
577
|
+
)
|
578
|
+
if not re.match(r"^[A-Za-z0-9_]+$", trigger["flow"]):
|
579
|
+
raise MetaflowException(
|
580
|
+
"Invalid flow name *%s* in *@trigger_on_finish* "
|
581
|
+
"decorator. Only alphanumeric characters and "
|
582
|
+
"underscores(_) are allowed." % trigger["flow"]
|
583
|
+
)
|
584
|
+
return trigger
|
585
|
+
|
586
|
+
def format_deploytime_value(self):
|
587
|
+
for trigger in self.triggers:
|
588
|
+
# Case were trigger is a function that returns a list
|
589
|
+
# Need to do this bc we need to iterate over list and process
|
590
|
+
if isinstance(trigger, DeployTimeField):
|
591
|
+
deploy_value = deploy_time_eval(trigger)
|
592
|
+
if isinstance(deploy_value, list):
|
593
|
+
self.triggers = deploy_value
|
594
|
+
else:
|
595
|
+
break
|
596
|
+
for trigger in self.triggers:
|
597
|
+
# Entire trigger is a function (returns either string or dict)
|
598
|
+
old_trig = trigger
|
599
|
+
if isinstance(trigger, DeployTimeField):
|
600
|
+
trigger = deploy_time_eval(trigger)
|
601
|
+
if isinstance(trigger, dict):
|
602
|
+
trigger["fq_name"] = trigger.get("name")
|
603
|
+
trigger["project"] = trigger.get("project")
|
604
|
+
trigger["branch"] = trigger.get("project_branch")
|
605
|
+
# We also added this bc it won't be formatted yet
|
606
|
+
if isinstance(trigger, str):
|
607
|
+
trigger = {"fq_name": trigger}
|
608
|
+
trigger = self._parse_fq_name(trigger)
|
609
|
+
self.triggers[self.triggers.index(old_trig)] = trigger
|
610
|
+
|
430
611
|
def get_top_level_options(self):
|
431
612
|
return list(self._option_values.items())
|
@@ -4,7 +4,6 @@ import math
|
|
4
4
|
import random
|
5
5
|
import time
|
6
6
|
from collections import namedtuple
|
7
|
-
|
8
7
|
from metaflow.exception import MetaflowException
|
9
8
|
from metaflow.metaflow_config import KUBERNETES_JOBSET_GROUP, KUBERNETES_JOBSET_VERSION
|
10
9
|
from metaflow.tracing import inject_tracing_vars
|
@@ -320,33 +319,49 @@ class RunningJobSet(object):
|
|
320
319
|
def kill(self):
|
321
320
|
plural = "jobsets"
|
322
321
|
client = self._client.get()
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
322
|
+
try:
|
323
|
+
# Killing the control pod will trigger the jobset to mark everything as failed.
|
324
|
+
# Since jobsets have a successPolicy set to `All` which ensures that everything has
|
325
|
+
# to succeed for the jobset to succeed.
|
326
|
+
from kubernetes.stream import stream
|
327
|
+
|
328
|
+
control_pod = self._fetch_pod()
|
329
|
+
stream(
|
330
|
+
client.CoreV1Api().connect_get_namespaced_pod_exec,
|
331
|
+
name=control_pod["metadata"]["name"],
|
332
|
+
namespace=control_pod["metadata"]["namespace"],
|
333
|
+
command=[
|
334
|
+
"/bin/sh",
|
335
|
+
"-c",
|
336
|
+
"/sbin/killall5",
|
337
|
+
],
|
338
|
+
stderr=True,
|
339
|
+
stdin=False,
|
340
|
+
stdout=True,
|
341
|
+
tty=False,
|
342
|
+
)
|
343
|
+
except Exception as e:
|
344
|
+
with client.ApiClient() as api_client:
|
345
|
+
# If we are unable to kill the control pod then
|
346
|
+
# Delete the jobset to kill the subsequent pods.
|
347
|
+
# There are a few reasons for deleting a jobset to kill it :
|
348
|
+
# 1. Jobset has a `suspend` attribute to suspend it's execution, but this
|
349
|
+
# doesn't play nicely when jobsets are deployed with other components like kueue.
|
350
|
+
# 2. Jobset doesn't play nicely when we mutate status
|
351
|
+
# 3. Deletion is a gaurenteed way of removing any pods.
|
352
|
+
api_instance = client.CustomObjectsApi(api_client)
|
353
|
+
try:
|
354
|
+
api_instance.delete_namespaced_custom_object(
|
355
|
+
group=self._group,
|
356
|
+
version=self._version,
|
357
|
+
namespace=self._namespace,
|
358
|
+
plural=plural,
|
359
|
+
name=self._name,
|
360
|
+
)
|
361
|
+
except Exception as e:
|
362
|
+
raise KubernetesJobsetException(
|
363
|
+
"Exception when deleting existing jobset: %s\n" % e
|
364
|
+
)
|
350
365
|
|
351
366
|
@property
|
352
367
|
def id(self):
|
metaflow/version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
metaflow_version = "2.12.
|
1
|
+
metaflow_version = "2.12.31"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: metaflow
|
3
|
-
Version: 2.12.
|
3
|
+
Version: 2.12.31
|
4
4
|
Summary: Metaflow: More Data Science, Less Engineering
|
5
5
|
Author: Metaflow Developers
|
6
6
|
Author-email: help@metaflow.org
|
@@ -26,7 +26,7 @@ License-File: LICENSE
|
|
26
26
|
Requires-Dist: requests
|
27
27
|
Requires-Dist: boto3
|
28
28
|
Provides-Extra: stubs
|
29
|
-
Requires-Dist: metaflow-stubs==2.12.
|
29
|
+
Requires-Dist: metaflow-stubs==2.12.31; extra == "stubs"
|
30
30
|
|
31
31
|

|
32
32
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
metaflow/R.py,sha256=CqVfIatvmjciuICNnoyyNGrwE7Va9iXfLdFbQa52hwA,3958
|
2
2
|
metaflow/__init__.py,sha256=S_RsIec8DhFP9kEpEo28oexTUqgmW7XG3snqjg9BFxo,5669
|
3
3
|
metaflow/cards.py,sha256=tP1_RrtmqdFh741pqE4t98S7SA0MtGRlGvRICRZF1Mg,426
|
4
|
-
metaflow/cli.py,sha256=
|
4
|
+
metaflow/cli.py,sha256=pBA0Nah6jo_oClIJ7miY-DIh3gJTTwlUFzD8NDA3kfM,35088
|
5
5
|
metaflow/cli_args.py,sha256=lcgBGNTvfaiPxiUnejAe60Upt9swG6lRy1_3OqbU6MY,2616
|
6
6
|
metaflow/clone_util.py,sha256=LSuVbFpPUh92UW32DBcnZbL0FFw-4w3CLa0tpEbCkzk,2066
|
7
7
|
metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
|
@@ -25,7 +25,7 @@ metaflow/metaflow_version.py,sha256=duhIzfKZtcxMVMs2uiBqBvUarSHJqyWDwMhaBOQd_g0,
|
|
25
25
|
metaflow/monitor.py,sha256=T0NMaBPvXynlJAO_avKtk8OIIRMyEuMAyF8bIp79aZU,5323
|
26
26
|
metaflow/multicore_utils.py,sha256=vdTNgczVLODifscUbbveJbuSDOl3Y9pAxhr7sqYiNf4,4760
|
27
27
|
metaflow/package.py,sha256=QutDP6WzjwGk1UCKXqBfXa9F10Q--FlRr0J7fwlple0,7399
|
28
|
-
metaflow/parameters.py,sha256=
|
28
|
+
metaflow/parameters.py,sha256=pzjG0ssuVHPyYQqWE86dS3yYChEqbT90rOUcRD4wNog,16079
|
29
29
|
metaflow/procpoll.py,sha256=U2tE4iK_Mwj2WDyVTx_Uglh6xZ-jixQOo4wrM9OOhxg,2859
|
30
30
|
metaflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
31
31
|
metaflow/pylint_wrapper.py,sha256=zzBY9YaSUZOGH-ypDKAv2B_7XcoyMZj-zCoCrmYqNRc,2865
|
@@ -36,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
|
|
36
36
|
metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
|
37
37
|
metaflow/util.py,sha256=olAvJK3y1it_k99MhLulTaAJo7OFVt5rnrD-ulIFLCU,13616
|
38
38
|
metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
|
39
|
-
metaflow/version.py,sha256=
|
39
|
+
metaflow/version.py,sha256=mjs6ElvxMu3mW-SxAf4zoxycHosTUA5ZCr3a93h0duk,29
|
40
40
|
metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
|
41
41
|
metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
|
42
42
|
metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
|
@@ -148,7 +148,7 @@ metaflow/plugins/catch_decorator.py,sha256=UOM2taN_OL2RPpuJhwEOA9ZALm0-hHD0XS2Hn
|
|
148
148
|
metaflow/plugins/debug_logger.py,sha256=mcF5HYzJ0NQmqCMjyVUk3iAP-heroHRIiVWQC6Ha2-I,879
|
149
149
|
metaflow/plugins/debug_monitor.py,sha256=Md5X_sDOSssN9pt2D8YcaIjTK5JaQD55UAYTcF6xYF0,1099
|
150
150
|
metaflow/plugins/environment_decorator.py,sha256=6m9j2B77d-Ja_l_9CTJ__0O6aB2a8Qt_lAZu6UjAcUA,587
|
151
|
-
metaflow/plugins/events_decorator.py,sha256=
|
151
|
+
metaflow/plugins/events_decorator.py,sha256=oULWTJw426MA9bdfMS-7q72LOQ1rI4oc_fp8JJd0P9o,26475
|
152
152
|
metaflow/plugins/logs_cli.py,sha256=77W5UNagU2mOKSMMvrQxQmBLRzvmjK-c8dWxd-Ygbqs,11410
|
153
153
|
metaflow/plugins/package_cli.py,sha256=-J6D4cupHfWSZ4GEFo2yy9Je9oL3owRWm5pEJwaiqd4,1649
|
154
154
|
metaflow/plugins/parallel_decorator.py,sha256=GIjZZVTqkvtnMuGE8RNtObX6CAJavZTxttqRujGmnGs,8973
|
@@ -175,7 +175,7 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqN
|
|
175
175
|
metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
176
176
|
metaflow/plugins/argo/argo_client.py,sha256=Z_A1TO9yw4Y-a8VAlwrFS0BwunWzXpbtik-j_xjcuHE,16303
|
177
177
|
metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
|
178
|
-
metaflow/plugins/argo/argo_workflows.py,sha256=
|
178
|
+
metaflow/plugins/argo/argo_workflows.py,sha256=d-d_xvjUeb5t11kaHZ98yIrGAqYg__FhxNcQTY8LVoA,173881
|
179
179
|
metaflow/plugins/argo/argo_workflows_cli.py,sha256=NdLwzfBcTsR72qLycZBesR4Pwv48o3Z_v6OfYrZuVEY,36721
|
180
180
|
metaflow/plugins/argo/argo_workflows_decorator.py,sha256=QdM1rK9gM-lDhyZldK8WqvFqJDvfJ7i3JPR5Uzaq2as,7887
|
181
181
|
metaflow/plugins/argo/argo_workflows_deployer.py,sha256=6kHxEnYXJwzNCM9swI8-0AckxtPWqwhZLerYkX8fxUM,4444
|
@@ -287,7 +287,7 @@ metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=TAYOKTQegYxex5piasLc53kNQPL
|
|
287
287
|
metaflow/plugins/kubernetes/kubernetes_client.py,sha256=tuvXP-QKpdeSmzVolB2R_TaacOr5DIb0j642eKcjsiM,6491
|
288
288
|
metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=Gq3lGKA8SPh3pHDbP_FCkUQPMRrIxvbcmw6Jly5PhEY,27846
|
289
289
|
metaflow/plugins/kubernetes/kubernetes_job.py,sha256=Cfkee8LbXC17jSXWoeNdomQRvF_8YSeXNg1gvxm6E_M,31806
|
290
|
-
metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=
|
290
|
+
metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=iehUEKv2KogyJKnp5jejdGP8R-TtF2aX9Wx1WpjKLvM,42030
|
291
291
|
metaflow/plugins/metadata_providers/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
292
292
|
metaflow/plugins/metadata_providers/local.py,sha256=9UAxe9caN6kU1lkSlIoJbRGgTqsMa62cBTnyMwhqiaA,22446
|
293
293
|
metaflow/plugins/metadata_providers/service.py,sha256=NKZfFMamx6upP6aFRJfXlfYIhySgFNzz6kbp1yPD7LA,20222
|
@@ -348,9 +348,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
|
|
348
348
|
metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
|
349
349
|
metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
|
350
350
|
metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
|
351
|
-
metaflow-2.12.
|
352
|
-
metaflow-2.12.
|
353
|
-
metaflow-2.12.
|
354
|
-
metaflow-2.12.
|
355
|
-
metaflow-2.12.
|
356
|
-
metaflow-2.12.
|
351
|
+
metaflow-2.12.31.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
|
352
|
+
metaflow-2.12.31.dist-info/METADATA,sha256=BAmAJa4ZmLnSxXvYTcjq7AwqfKu29nZBKa6iLU8DDHg,5907
|
353
|
+
metaflow-2.12.31.dist-info/WHEEL,sha256=pxeNX5JdtCe58PUSYP9upmc7jdRPgvT0Gm9kb1SHlVw,109
|
354
|
+
metaflow-2.12.31.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
|
355
|
+
metaflow-2.12.31.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
|
356
|
+
metaflow-2.12.31.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|