metaflow 2.12.26__py2.py3-none-any.whl → 2.12.28__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow/cli.py +31 -4
- metaflow/client/core.py +77 -39
- metaflow/metadata/metadata.py +6 -0
- metaflow/plugins/argo/argo_workflows_cli.py +3 -3
- metaflow/plugins/aws/step_functions/step_functions_cli.py +2 -2
- metaflow/plugins/kubernetes/kubernetes_jobsets.py +1 -4
- metaflow/plugins/resources_decorator.py +2 -2
- metaflow/plugins/tag_cli.py +1 -4
- metaflow/runner/metaflow_runner.py +4 -3
- metaflow/system/system_logger.py +1 -19
- metaflow/system/system_monitor.py +0 -24
- metaflow/task.py +31 -25
- metaflow/version.py +1 -1
- {metaflow-2.12.26.dist-info → metaflow-2.12.28.dist-info}/METADATA +2 -2
- {metaflow-2.12.26.dist-info → metaflow-2.12.28.dist-info}/RECORD +19 -19
- {metaflow-2.12.26.dist-info → metaflow-2.12.28.dist-info}/WHEEL +1 -1
- {metaflow-2.12.26.dist-info → metaflow-2.12.28.dist-info}/LICENSE +0 -0
- {metaflow-2.12.26.dist-info → metaflow-2.12.28.dist-info}/entry_points.txt +0 -0
- {metaflow-2.12.26.dist-info → metaflow-2.12.28.dist-info}/top_level.txt +0 -0
metaflow/cli.py
CHANGED
@@ -701,12 +701,12 @@ def resume(
|
|
701
701
|
runtime.persist_constants()
|
702
702
|
|
703
703
|
if runner_attribute_file:
|
704
|
-
with open(runner_attribute_file, "w") as f:
|
704
|
+
with open(runner_attribute_file, "w", encoding="utf-8") as f:
|
705
705
|
json.dump(
|
706
706
|
{
|
707
707
|
"run_id": runtime.run_id,
|
708
708
|
"flow_name": obj.flow.name,
|
709
|
-
"metadata":
|
709
|
+
"metadata": obj.metadata.metadata_str(),
|
710
710
|
},
|
711
711
|
f,
|
712
712
|
)
|
@@ -716,6 +716,20 @@ def resume(
|
|
716
716
|
if runtime.should_skip_clone_only_execution():
|
717
717
|
return
|
718
718
|
|
719
|
+
current._update_env(
|
720
|
+
{
|
721
|
+
"run_id": runtime.run_id,
|
722
|
+
}
|
723
|
+
)
|
724
|
+
_system_logger.log_event(
|
725
|
+
level="info",
|
726
|
+
module="metaflow.resume",
|
727
|
+
name="start",
|
728
|
+
payload={
|
729
|
+
"msg": "Resuming run",
|
730
|
+
},
|
731
|
+
)
|
732
|
+
|
719
733
|
with runtime.run_heartbeat():
|
720
734
|
if clone_only:
|
721
735
|
runtime.clone_original_run()
|
@@ -775,16 +789,29 @@ def run(
|
|
775
789
|
write_file(run_id_file, runtime.run_id)
|
776
790
|
|
777
791
|
obj.flow._set_constants(obj.graph, kwargs)
|
792
|
+
current._update_env(
|
793
|
+
{
|
794
|
+
"run_id": runtime.run_id,
|
795
|
+
}
|
796
|
+
)
|
797
|
+
_system_logger.log_event(
|
798
|
+
level="info",
|
799
|
+
module="metaflow.run",
|
800
|
+
name="start",
|
801
|
+
payload={
|
802
|
+
"msg": "Starting run",
|
803
|
+
},
|
804
|
+
)
|
778
805
|
runtime.print_workflow_info()
|
779
806
|
runtime.persist_constants()
|
780
807
|
|
781
808
|
if runner_attribute_file:
|
782
|
-
with open(runner_attribute_file, "w") as f:
|
809
|
+
with open(runner_attribute_file, "w", encoding="utf-8") as f:
|
783
810
|
json.dump(
|
784
811
|
{
|
785
812
|
"run_id": runtime.run_id,
|
786
813
|
"flow_name": obj.flow.name,
|
787
|
-
"metadata":
|
814
|
+
"metadata": obj.metadata.metadata_str(),
|
788
815
|
},
|
789
816
|
f,
|
790
817
|
)
|
metaflow/client/core.py
CHANGED
@@ -16,6 +16,7 @@ from typing import (
|
|
16
16
|
List,
|
17
17
|
NamedTuple,
|
18
18
|
Optional,
|
19
|
+
TYPE_CHECKING,
|
19
20
|
Tuple,
|
20
21
|
)
|
21
22
|
|
@@ -37,6 +38,9 @@ from metaflow.util import cached_property, is_stringish, resolve_identity, to_un
|
|
37
38
|
from ..info_file import INFO_FILE
|
38
39
|
from .filecache import FileCache
|
39
40
|
|
41
|
+
if TYPE_CHECKING:
|
42
|
+
from metaflow.metadata import MetadataProvider
|
43
|
+
|
40
44
|
try:
|
41
45
|
# python2
|
42
46
|
import cPickle as pickle
|
@@ -82,28 +86,16 @@ def metadata(ms: str) -> str:
|
|
82
86
|
get_metadata()).
|
83
87
|
"""
|
84
88
|
global current_metadata
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
else:
|
96
|
-
metadata_type = "local"
|
97
|
-
res = [m for m in METADATA_PROVIDERS if m.TYPE == metadata_type]
|
98
|
-
if not res:
|
99
|
-
print(
|
100
|
-
"Cannot find a '%s' metadata provider -- "
|
101
|
-
"try specifying one explicitly using <type>@<info>",
|
102
|
-
metadata_type,
|
103
|
-
)
|
104
|
-
return get_metadata()
|
105
|
-
current_metadata = res[0]
|
106
|
-
current_metadata.INFO = ms
|
89
|
+
provider, info = _metadata(ms)
|
90
|
+
if provider is None:
|
91
|
+
print(
|
92
|
+
"Cannot find a metadata provider -- "
|
93
|
+
"try specifying one explicitly using <type>@<info>",
|
94
|
+
)
|
95
|
+
return get_metadata()
|
96
|
+
current_metadata = provider
|
97
|
+
if info:
|
98
|
+
current_metadata.INFO = info
|
107
99
|
return get_metadata()
|
108
100
|
|
109
101
|
|
@@ -127,7 +119,7 @@ def get_metadata() -> str:
|
|
127
119
|
"""
|
128
120
|
if current_metadata is False:
|
129
121
|
default_metadata()
|
130
|
-
return
|
122
|
+
return current_metadata.metadata_str()
|
131
123
|
|
132
124
|
|
133
125
|
def default_metadata() -> str:
|
@@ -268,9 +260,16 @@ class MetaflowObject(object):
|
|
268
260
|
_object: Optional["MetaflowObject"] = None,
|
269
261
|
_parent: Optional["MetaflowObject"] = None,
|
270
262
|
_namespace_check: bool = True,
|
263
|
+
_metaflow: Optional["Metaflow"] = None,
|
271
264
|
_current_namespace: Optional[str] = None,
|
265
|
+
_current_metadata: Optional[str] = None,
|
272
266
|
):
|
273
|
-
|
267
|
+
# the default namespace is activated lazily at the first
|
268
|
+
# get_namespace(). The other option of activating
|
269
|
+
# the namespace at the import time is problematic, since there
|
270
|
+
# may be other modules that alter environment variables etc.
|
271
|
+
# which may affect the namespace setting.
|
272
|
+
self._metaflow = Metaflow(_current_metadata) or _metaflow
|
274
273
|
self._parent = _parent
|
275
274
|
self._path_components = None
|
276
275
|
self._attempt = attempt
|
@@ -390,6 +389,7 @@ class MetaflowObject(object):
|
|
390
389
|
attempt=self._attempt,
|
391
390
|
_object=obj,
|
392
391
|
_parent=self,
|
392
|
+
_metaflow=self._metaflow,
|
393
393
|
_namespace_check=self._namespace_check,
|
394
394
|
_current_namespace=(
|
395
395
|
self._current_namespace if self._namespace_check else None
|
@@ -505,6 +505,7 @@ class MetaflowObject(object):
|
|
505
505
|
attempt=self._attempt,
|
506
506
|
_object=obj,
|
507
507
|
_parent=self,
|
508
|
+
_metaflow=self._metaflow,
|
508
509
|
_namespace_check=self._namespace_check,
|
509
510
|
_current_namespace=(
|
510
511
|
self._current_namespace if self._namespace_check else None
|
@@ -552,7 +553,25 @@ class MetaflowObject(object):
|
|
552
553
|
_current_namespace=ns,
|
553
554
|
)
|
554
555
|
|
555
|
-
|
556
|
+
def _unpickle_21227(self, data):
|
557
|
+
if len(data) != 5:
|
558
|
+
raise MetaflowInternalError(
|
559
|
+
"Unexpected size of array: {}".format(len(data))
|
560
|
+
)
|
561
|
+
pathspec, attempt, md, ns, namespace_check = data
|
562
|
+
self.__init__(
|
563
|
+
pathspec=pathspec,
|
564
|
+
attempt=attempt,
|
565
|
+
_namespace_check=namespace_check,
|
566
|
+
_current_metadata=md,
|
567
|
+
_current_namespace=ns,
|
568
|
+
)
|
569
|
+
|
570
|
+
_UNPICKLE_FUNC = {
|
571
|
+
"2.8.4": _unpickle_284,
|
572
|
+
"2.12.4": _unpickle_2124,
|
573
|
+
"2.12.27": _unpickle_21227,
|
574
|
+
}
|
556
575
|
|
557
576
|
def __setstate__(self, state):
|
558
577
|
"""
|
@@ -595,10 +614,11 @@ class MetaflowObject(object):
|
|
595
614
|
# checking for the namespace even after unpickling since we will know which
|
596
615
|
# namespace to check.
|
597
616
|
return {
|
598
|
-
"version": "2.12.
|
617
|
+
"version": "2.12.27",
|
599
618
|
"data": [
|
600
619
|
self.pathspec,
|
601
620
|
self._attempt,
|
621
|
+
self._metaflow.metadata.metadata_str(),
|
602
622
|
self._current_namespace,
|
603
623
|
self._namespace_check,
|
604
624
|
],
|
@@ -2288,17 +2308,16 @@ class Metaflow(object):
|
|
2288
2308
|
if it has at least one run in the namespace.
|
2289
2309
|
"""
|
2290
2310
|
|
2291
|
-
def __init__(self):
|
2292
|
-
|
2293
|
-
|
2294
|
-
|
2295
|
-
|
2296
|
-
|
2297
|
-
|
2298
|
-
|
2299
|
-
|
2300
|
-
|
2301
|
-
self.metadata = current_metadata
|
2311
|
+
def __init__(self, _current_metadata: Optional[str] = None):
|
2312
|
+
if _current_metadata:
|
2313
|
+
provider, info = _metadata(_current_metadata)
|
2314
|
+
self.metadata = provider
|
2315
|
+
if info:
|
2316
|
+
self.metadata.INFO = info
|
2317
|
+
else:
|
2318
|
+
if current_metadata is False:
|
2319
|
+
default_metadata()
|
2320
|
+
self.metadata = current_metadata
|
2302
2321
|
|
2303
2322
|
@property
|
2304
2323
|
def flows(self) -> List[Flow]:
|
@@ -2335,7 +2354,7 @@ class Metaflow(object):
|
|
2335
2354
|
all_flows = all_flows if all_flows else []
|
2336
2355
|
for flow in all_flows:
|
2337
2356
|
try:
|
2338
|
-
v = Flow(_object=flow)
|
2357
|
+
v = Flow(_object=flow, _metaflow=self)
|
2339
2358
|
yield v
|
2340
2359
|
except MetaflowNamespaceMismatch:
|
2341
2360
|
continue
|
@@ -2359,7 +2378,26 @@ class Metaflow(object):
|
|
2359
2378
|
Flow
|
2360
2379
|
Flow with the given name.
|
2361
2380
|
"""
|
2362
|
-
return Flow(name)
|
2381
|
+
return Flow(name, _metaflow=self)
|
2382
|
+
|
2383
|
+
|
2384
|
+
def _metadata(ms: str) -> Tuple[Optional["MetadataProvider"], Optional[str]]:
|
2385
|
+
infos = ms.split("@", 1)
|
2386
|
+
types = [m.TYPE for m in METADATA_PROVIDERS]
|
2387
|
+
if infos[0] in types:
|
2388
|
+
provider = [m for m in METADATA_PROVIDERS if m.TYPE == infos[0]][0]
|
2389
|
+
if len(infos) > 1:
|
2390
|
+
return provider, infos[1]
|
2391
|
+
return provider, None
|
2392
|
+
# Deduce from ms; if starts with http, use service or else use local
|
2393
|
+
if ms.startswith("http"):
|
2394
|
+
metadata_type = "service"
|
2395
|
+
else:
|
2396
|
+
metadata_type = "local"
|
2397
|
+
res = [m for m in METADATA_PROVIDERS if m.TYPE == metadata_type]
|
2398
|
+
if not res:
|
2399
|
+
return None, None
|
2400
|
+
return res[0], ms
|
2363
2401
|
|
2364
2402
|
|
2365
2403
|
_CLASSES["flow"] = Flow
|
metaflow/metadata/metadata.py
CHANGED
@@ -76,6 +76,12 @@ class ObjectOrder:
|
|
76
76
|
|
77
77
|
@with_metaclass(MetadataProviderMeta)
|
78
78
|
class MetadataProvider(object):
|
79
|
+
TYPE = None
|
80
|
+
|
81
|
+
@classmethod
|
82
|
+
def metadata_str(cls):
|
83
|
+
return "%s@%s" % (cls.TYPE, cls.INFO)
|
84
|
+
|
79
85
|
@classmethod
|
80
86
|
def compute_info(cls, val):
|
81
87
|
"""
|
@@ -226,12 +226,12 @@ def create(
|
|
226
226
|
validate_tags(tags)
|
227
227
|
|
228
228
|
if deployer_attribute_file:
|
229
|
-
with open(deployer_attribute_file, "w") as f:
|
229
|
+
with open(deployer_attribute_file, "w", encoding="utf-8") as f:
|
230
230
|
json.dump(
|
231
231
|
{
|
232
232
|
"name": obj.workflow_name,
|
233
233
|
"flow_name": obj.flow.name,
|
234
|
-
"metadata":
|
234
|
+
"metadata": obj.metadata.metadata_str(),
|
235
235
|
},
|
236
236
|
f,
|
237
237
|
)
|
@@ -657,7 +657,7 @@ def trigger(obj, run_id_file=None, deployer_attribute_file=None, **kwargs):
|
|
657
657
|
json.dump(
|
658
658
|
{
|
659
659
|
"name": obj.workflow_name,
|
660
|
-
"metadata":
|
660
|
+
"metadata": obj.metadata.metadata_str(),
|
661
661
|
"pathspec": "/".join((obj.flow.name, run_id)),
|
662
662
|
},
|
663
663
|
f,
|
@@ -162,7 +162,7 @@ def create(
|
|
162
162
|
{
|
163
163
|
"name": obj.state_machine_name,
|
164
164
|
"flow_name": obj.flow.name,
|
165
|
-
"metadata":
|
165
|
+
"metadata": obj.metadata.metadata_str(),
|
166
166
|
},
|
167
167
|
f,
|
168
168
|
)
|
@@ -502,7 +502,7 @@ def trigger(obj, run_id_file=None, deployer_attribute_file=None, **kwargs):
|
|
502
502
|
json.dump(
|
503
503
|
{
|
504
504
|
"name": obj.state_machine_name,
|
505
|
-
"metadata":
|
505
|
+
"metadata": obj.metadata.metadata_str(),
|
506
506
|
"pathspec": "/".join((obj.flow.name, run_id)),
|
507
507
|
},
|
508
508
|
f,
|
@@ -332,11 +332,8 @@ class RunningJobSet(object):
|
|
332
332
|
name=self._name,
|
333
333
|
)
|
334
334
|
|
335
|
-
# Suspend the jobset
|
336
|
-
#
|
335
|
+
# Suspend the jobset
|
337
336
|
obj["spec"]["suspend"] = True
|
338
|
-
for replicated_job in obj["spec"]["replicatedJobs"]:
|
339
|
-
replicated_job["replicas"] = 0
|
340
337
|
|
341
338
|
api_instance.replace_namespaced_custom_object(
|
342
339
|
group=self._group,
|
@@ -23,7 +23,7 @@ class ResourcesDecorator(StepDecorator):
|
|
23
23
|
----------
|
24
24
|
cpu : int, default 1
|
25
25
|
Number of CPUs required for this step.
|
26
|
-
gpu : int, default
|
26
|
+
gpu : int, optional, default None
|
27
27
|
Number of GPUs required for this step.
|
28
28
|
disk : int, optional, default None
|
29
29
|
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
@@ -37,7 +37,7 @@ class ResourcesDecorator(StepDecorator):
|
|
37
37
|
name = "resources"
|
38
38
|
defaults = {
|
39
39
|
"cpu": "1",
|
40
|
-
"gpu":
|
40
|
+
"gpu": None,
|
41
41
|
"disk": None,
|
42
42
|
"memory": "4096",
|
43
43
|
"shared_memory": None,
|
metaflow/plugins/tag_cli.py
CHANGED
@@ -225,10 +225,7 @@ def _get_client_run_obj(obj, run_id, user_namespace):
|
|
225
225
|
|
226
226
|
|
227
227
|
def _set_current(obj):
|
228
|
-
current._set_env(
|
229
|
-
metadata_str="%s@%s"
|
230
|
-
% (obj.metadata.__class__.TYPE, obj.metadata.__class__.INFO)
|
231
|
-
)
|
228
|
+
current._set_env(metadata_str=obj.metadata.metadata_str())
|
232
229
|
|
233
230
|
|
234
231
|
@click.group()
|
@@ -6,7 +6,7 @@ import tempfile
|
|
6
6
|
|
7
7
|
from typing import Dict, Iterator, Optional, Tuple
|
8
8
|
|
9
|
-
from metaflow import Run
|
9
|
+
from metaflow import Run
|
10
10
|
|
11
11
|
from .utils import handle_timeout
|
12
12
|
from .subprocess_manager import CommandManager, SubprocessManager
|
@@ -275,9 +275,10 @@ class Runner(object):
|
|
275
275
|
|
276
276
|
# Set the correct metadata from the runner_attribute file corresponding to this run.
|
277
277
|
metadata_for_flow = content.get("metadata")
|
278
|
-
metadata(metadata_for_flow)
|
279
278
|
|
280
|
-
run_object = Run(
|
279
|
+
run_object = Run(
|
280
|
+
pathspec, _namespace_check=False, _current_metadata=metadata_for_flow
|
281
|
+
)
|
281
282
|
return ExecutingRun(self, command_obj, run_object)
|
282
283
|
|
283
284
|
def run(self, **kwargs) -> ExecutingRun:
|
metaflow/system/system_logger.py
CHANGED
@@ -7,26 +7,11 @@ class SystemLogger(object):
|
|
7
7
|
def __init__(self):
|
8
8
|
self._logger = None
|
9
9
|
self._flow_name = None
|
10
|
-
self._context = {}
|
11
|
-
self._is_context_updated = False
|
12
10
|
|
13
11
|
def __del__(self):
|
14
12
|
if self._flow_name == "not_a_real_flow":
|
15
13
|
self.logger.terminate()
|
16
14
|
|
17
|
-
def update_context(self, context: Dict[str, Any]):
|
18
|
-
"""
|
19
|
-
Update the global context maintained by the system logger.
|
20
|
-
|
21
|
-
Parameters
|
22
|
-
----------
|
23
|
-
context : Dict[str, Any]
|
24
|
-
A dictionary containing the context to update.
|
25
|
-
|
26
|
-
"""
|
27
|
-
self._is_context_updated = True
|
28
|
-
self._context.update(context)
|
29
|
-
|
30
15
|
def init_system_logger(
|
31
16
|
self, flow_name: str, logger: "metaflow.event_logger.NullEventLogger"
|
32
17
|
):
|
@@ -71,7 +56,7 @@ class SystemLogger(object):
|
|
71
56
|
"false",
|
72
57
|
"",
|
73
58
|
):
|
74
|
-
print("system
|
59
|
+
print("system logger: %s" % msg, file=sys.stderr)
|
75
60
|
|
76
61
|
def log_event(
|
77
62
|
self, level: str, module: str, name: str, payload: Optional[Any] = None
|
@@ -96,8 +81,5 @@ class SystemLogger(object):
|
|
96
81
|
"module": module,
|
97
82
|
"name": name,
|
98
83
|
"payload": payload if payload is not None else {},
|
99
|
-
"context": self._context,
|
100
|
-
"is_context_updated": self._is_context_updated,
|
101
84
|
}
|
102
85
|
)
|
103
|
-
self._is_context_updated = False
|
@@ -9,35 +9,11 @@ class SystemMonitor(object):
|
|
9
9
|
def __init__(self):
|
10
10
|
self._monitor = None
|
11
11
|
self._flow_name = None
|
12
|
-
self._context = {}
|
13
12
|
|
14
13
|
def __del__(self):
|
15
14
|
if self._flow_name == "not_a_real_flow":
|
16
15
|
self.monitor.terminate()
|
17
16
|
|
18
|
-
def update_context(self, context: Dict[str, Any]):
|
19
|
-
"""
|
20
|
-
Update the global context maintained by the system monitor.
|
21
|
-
|
22
|
-
Parameters
|
23
|
-
----------
|
24
|
-
context : Dict[str, Any]
|
25
|
-
A dictionary containing the context to update.
|
26
|
-
|
27
|
-
"""
|
28
|
-
from metaflow.sidecar import Message, MessageTypes
|
29
|
-
|
30
|
-
self._context.update(context)
|
31
|
-
self.monitor.send(
|
32
|
-
Message(
|
33
|
-
MessageTypes.MUST_SEND,
|
34
|
-
{
|
35
|
-
"is_context_updated": True,
|
36
|
-
**self._context,
|
37
|
-
},
|
38
|
-
)
|
39
|
-
)
|
40
|
-
|
41
17
|
def init_system_monitor(
|
42
18
|
self, flow_name: str, monitor: "metaflow.monitor.NullMonitor"
|
43
19
|
):
|
metaflow/task.py
CHANGED
@@ -306,8 +306,6 @@ class MetaflowTask(object):
|
|
306
306
|
"origin_run_id": origin_run_id,
|
307
307
|
"origin_task_id": origin_task_id,
|
308
308
|
}
|
309
|
-
_system_logger.update_context(task_payload)
|
310
|
-
_system_monitor.update_context(task_payload)
|
311
309
|
|
312
310
|
msg = "Cloning task from {}/{}/{}/{} to {}/{}/{}/{}".format(
|
313
311
|
self.flow.name,
|
@@ -512,8 +510,7 @@ class MetaflowTask(object):
|
|
512
510
|
origin_run_id=origin_run_id,
|
513
511
|
namespace=resolve_identity(),
|
514
512
|
username=get_username(),
|
515
|
-
metadata_str=
|
516
|
-
% (self.metadata.__class__.TYPE, self.metadata.__class__.INFO),
|
513
|
+
metadata_str=self.metadata.metadata_str(),
|
517
514
|
is_running=True,
|
518
515
|
tags=self.metadata.sticky_tags,
|
519
516
|
)
|
@@ -546,9 +543,6 @@ class MetaflowTask(object):
|
|
546
543
|
"project_flow_name": current.get("project_flow_name"),
|
547
544
|
"trace_id": trace_id or None,
|
548
545
|
}
|
549
|
-
|
550
|
-
_system_logger.update_context(task_payload)
|
551
|
-
_system_monitor.update_context(task_payload)
|
552
546
|
start = time.time()
|
553
547
|
self.metadata.start_task_heartbeat(self.flow.name, run_id, step_name, task_id)
|
554
548
|
with self.monitor.measure("metaflow.task.duration"):
|
@@ -593,7 +587,8 @@ class MetaflowTask(object):
|
|
593
587
|
{
|
594
588
|
"parameter_names": self._init_parameters(
|
595
589
|
inputs[0], passdown=True
|
596
|
-
)
|
590
|
+
),
|
591
|
+
"graph_info": self.flow._graph_info,
|
597
592
|
}
|
598
593
|
)
|
599
594
|
else:
|
@@ -617,7 +612,8 @@ class MetaflowTask(object):
|
|
617
612
|
{
|
618
613
|
"parameter_names": self._init_parameters(
|
619
614
|
inputs[0], passdown=False
|
620
|
-
)
|
615
|
+
),
|
616
|
+
"graph_info": self.flow._graph_info,
|
621
617
|
}
|
622
618
|
)
|
623
619
|
|
@@ -709,24 +705,34 @@ class MetaflowTask(object):
|
|
709
705
|
name="end",
|
710
706
|
payload={**task_payload, "msg": "Task ended"},
|
711
707
|
)
|
712
|
-
|
713
|
-
|
714
|
-
|
715
|
-
|
716
|
-
|
717
|
-
|
718
|
-
|
719
|
-
|
720
|
-
|
721
|
-
|
722
|
-
|
723
|
-
|
724
|
-
|
725
|
-
|
726
|
-
|
708
|
+
try:
|
709
|
+
# persisting might fail due to unpicklable artifacts.
|
710
|
+
output.persist(self.flow)
|
711
|
+
except Exception as ex:
|
712
|
+
self.flow._task_ok = False
|
713
|
+
raise ex
|
714
|
+
finally:
|
715
|
+
# The attempt_ok metadata is used to determine task status so it is important
|
716
|
+
# we ensure that it is written even in case of preceding failures.
|
717
|
+
# f.ex. failing to serialize artifacts leads to a non-zero exit code for the process,
|
718
|
+
# even if user code finishes successfully. Flow execution will not continue due to the exit,
|
719
|
+
# so arguably we should mark the task as failed.
|
720
|
+
attempt_ok = str(bool(self.flow._task_ok))
|
721
|
+
self.metadata.register_metadata(
|
722
|
+
run_id,
|
723
|
+
step_name,
|
724
|
+
task_id,
|
725
|
+
[
|
726
|
+
MetaDatum(
|
727
|
+
field="attempt_ok",
|
728
|
+
value=attempt_ok,
|
729
|
+
type="internal_attempt_status",
|
730
|
+
tags=["attempt_id:{0}".format(retry_count)],
|
731
|
+
)
|
732
|
+
],
|
733
|
+
)
|
727
734
|
|
728
735
|
output.save_metadata({"task_end": {}})
|
729
|
-
output.persist(self.flow)
|
730
736
|
|
731
737
|
# this writes a success marker indicating that the
|
732
738
|
# "transaction" is done
|
metaflow/version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
metaflow_version = "2.12.
|
1
|
+
metaflow_version = "2.12.28"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: metaflow
|
3
|
-
Version: 2.12.
|
3
|
+
Version: 2.12.28
|
4
4
|
Summary: Metaflow: More Data Science, Less Engineering
|
5
5
|
Author: Metaflow Developers
|
6
6
|
Author-email: help@metaflow.org
|
@@ -26,7 +26,7 @@ License-File: LICENSE
|
|
26
26
|
Requires-Dist: requests
|
27
27
|
Requires-Dist: boto3
|
28
28
|
Provides-Extra: stubs
|
29
|
-
Requires-Dist: metaflow-stubs==2.12.
|
29
|
+
Requires-Dist: metaflow-stubs==2.12.28; extra == "stubs"
|
30
30
|
|
31
31
|

|
32
32
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
metaflow/R.py,sha256=CqVfIatvmjciuICNnoyyNGrwE7Va9iXfLdFbQa52hwA,3958
|
2
2
|
metaflow/__init__.py,sha256=9pnkRH00bbtzaQDSxEspANaB3r6YDC37_EjX7GxtJug,5641
|
3
3
|
metaflow/cards.py,sha256=tP1_RrtmqdFh741pqE4t98S7SA0MtGRlGvRICRZF1Mg,426
|
4
|
-
metaflow/cli.py,sha256=
|
4
|
+
metaflow/cli.py,sha256=pBA0Nah6jo_oClIJ7miY-DIh3gJTTwlUFzD8NDA3kfM,35088
|
5
5
|
metaflow/cli_args.py,sha256=lcgBGNTvfaiPxiUnejAe60Upt9swG6lRy1_3OqbU6MY,2616
|
6
6
|
metaflow/clone_util.py,sha256=XfUX0vssu_hPlyZfhFl1AOnKkLqvt33Qp8xNrmdocGg,2057
|
7
7
|
metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
|
@@ -31,12 +31,12 @@ metaflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
31
31
|
metaflow/pylint_wrapper.py,sha256=zzBY9YaSUZOGH-ypDKAv2B_7XcoyMZj-zCoCrmYqNRc,2865
|
32
32
|
metaflow/runtime.py,sha256=fbBObJJciagHWPzR3T7x9e_jez_RBnLZIHsXMvYnW_M,68875
|
33
33
|
metaflow/tagging_util.py,sha256=ctyf0Q1gBi0RyZX6J0e9DQGNkNHblV_CITfy66axXB4,2346
|
34
|
-
metaflow/task.py,sha256
|
34
|
+
metaflow/task.py,sha256=-QKLVY6Uky3EKNq3g14i0VG2uaj4DHb8b05JneUyKNM,29682
|
35
35
|
metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
|
36
36
|
metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
|
37
37
|
metaflow/util.py,sha256=olAvJK3y1it_k99MhLulTaAJo7OFVt5rnrD-ulIFLCU,13616
|
38
38
|
metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
|
39
|
-
metaflow/version.py,sha256=
|
39
|
+
metaflow/version.py,sha256=7kZIaI8ejbLYDBD1Sw2PdpQdQVXaz0v5TrphyTixJ-0,29
|
40
40
|
metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
|
41
41
|
metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
|
42
42
|
metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
|
@@ -111,7 +111,7 @@ metaflow/_vendor/v3_6/importlib_metadata/_meta.py,sha256=_F48Hu_jFxkfKWz5wcYS8vO
|
|
111
111
|
metaflow/_vendor/v3_6/importlib_metadata/_text.py,sha256=HCsFksZpJLeTP3NEk_ngrAeXVRRtTrtyh9eOABoRP4A,2166
|
112
112
|
metaflow/_vendor/v3_6/importlib_metadata/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
113
113
|
metaflow/client/__init__.py,sha256=1GtQB4Y_CBkzaxg32L1syNQSlfj762wmLrfrDxGi1b8,226
|
114
|
-
metaflow/client/core.py,sha256=
|
114
|
+
metaflow/client/core.py,sha256=CI-34oZDZhsC0RZxO-NldeSdsK_1x7Kn_t-F6exDDEs,75418
|
115
115
|
metaflow/client/filecache.py,sha256=Wy0yhhCqC1JZgebqi7z52GCwXYnkAqMZHTtxThvwBgM,15229
|
116
116
|
metaflow/cmd/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
117
117
|
metaflow/cmd/configure_cmd.py,sha256=o-DKnUf2FBo_HiMVyoyzQaGBSMtpbEPEdFTQZ0hkU-k,33396
|
@@ -136,7 +136,7 @@ metaflow/extension_support/integrations.py,sha256=AWAh-AZ-vo9IxuAVEjGw3s8p_NMm2D
|
|
136
136
|
metaflow/extension_support/plugins.py,sha256=vz6q-6Eiy2QQ6Lo4Rv3sFTHKJZ9M3eorzD1tfS_L0SY,10831
|
137
137
|
metaflow/metadata/__init__.py,sha256=FZNSnz26VB_m18DQG8mup6-Gfl7r1U6lRMljJBp3VAM,64
|
138
138
|
metaflow/metadata/heartbeat.py,sha256=9XJs4SV4R4taxUSMOKmRChnJCQ5a5o0dzRZ1iICWr9Y,2423
|
139
|
-
metaflow/metadata/metadata.py,sha256=
|
139
|
+
metaflow/metadata/metadata.py,sha256=4tmySlgQaoV-p9bHig7BjsEsqFRZWEOuwSLpODuKxjA,26169
|
140
140
|
metaflow/metadata/util.py,sha256=lYoQKbqoTM1iZChgyVWN-gX-HyM9tt9bXEMJexY9XmM,1723
|
141
141
|
metaflow/mflog/__init__.py,sha256=9iMMn2xYB0oaDXXcInxa9AdDqeVBeiJeB3klnqGkyL0,5983
|
142
142
|
metaflow/mflog/mflog.py,sha256=VebXxqitOtNAs7VJixnNfziO_i_urG7bsJ5JiB5IXgY,4370
|
@@ -153,10 +153,10 @@ metaflow/plugins/logs_cli.py,sha256=77W5UNagU2mOKSMMvrQxQmBLRzvmjK-c8dWxd-Ygbqs,
|
|
153
153
|
metaflow/plugins/package_cli.py,sha256=-J6D4cupHfWSZ4GEFo2yy9Je9oL3owRWm5pEJwaiqd4,1649
|
154
154
|
metaflow/plugins/parallel_decorator.py,sha256=aAM5jHQguADrTdnFu878FxHynOdc3zdye2G98X65HIw,8964
|
155
155
|
metaflow/plugins/project_decorator.py,sha256=eJOe0Ea7CbUCReEhR_XQvRkhV6jyRqDxM72oZI7EMCk,5336
|
156
|
-
metaflow/plugins/resources_decorator.py,sha256=
|
156
|
+
metaflow/plugins/resources_decorator.py,sha256=AtoOwg4mHYHYthg-CAfbfam-QiT0ViuDLDoukoDvF6Q,1347
|
157
157
|
metaflow/plugins/retry_decorator.py,sha256=tz_2Tq6GLg3vjDBZp0KKVTk3ADlCvqaWTSf7blmFdUw,1548
|
158
158
|
metaflow/plugins/storage_executor.py,sha256=FqAgR0-L9MuqN8fRtTe4jjUfJL9lqt6fQkYaglAjRbk,6137
|
159
|
-
metaflow/plugins/tag_cli.py,sha256=
|
159
|
+
metaflow/plugins/tag_cli.py,sha256=10039-0DUF0cmhudoDNrRGLWq8tCGQJ7tBsQAGAmkBQ,17549
|
160
160
|
metaflow/plugins/test_unbounded_foreach_decorator.py,sha256=-3o_VhmZW7R9i-0RgqEradmPqx9rS6jJxcIKV6-WQMg,5948
|
161
161
|
metaflow/plugins/timeout_decorator.py,sha256=R-X8rKeMqd-xhfJFqskWb6ZpmZt2JB14U1BZJSRriwM,3648
|
162
162
|
metaflow/plugins/airflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -176,7 +176,7 @@ metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
|
|
176
176
|
metaflow/plugins/argo/argo_client.py,sha256=Z_A1TO9yw4Y-a8VAlwrFS0BwunWzXpbtik-j_xjcuHE,16303
|
177
177
|
metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
|
178
178
|
metaflow/plugins/argo/argo_workflows.py,sha256=9ygK_yp5YMCmcNo3TGwi562pRKXJtCw0OyRvKl-B9JQ,173669
|
179
|
-
metaflow/plugins/argo/argo_workflows_cli.py,sha256=
|
179
|
+
metaflow/plugins/argo/argo_workflows_cli.py,sha256=NdLwzfBcTsR72qLycZBesR4Pwv48o3Z_v6OfYrZuVEY,36721
|
180
180
|
metaflow/plugins/argo/argo_workflows_decorator.py,sha256=yprszMdbE3rBTcEA9VR0IEnPjTprUauZBc4SBb-Q7sA,7878
|
181
181
|
metaflow/plugins/argo/argo_workflows_deployer.py,sha256=wSSZtThn_VPvE_Wu6NB1L0Q86LmBJh9g009v_lpvBPM,8125
|
182
182
|
metaflow/plugins/argo/capture_error.py,sha256=Ys9dscGrTpW-ZCirLBU0gD9qBM0BjxyxGlUMKcwewQc,1852
|
@@ -199,7 +199,7 @@ metaflow/plugins/aws/step_functions/production_token.py,sha256=_o4emv3rozYZoWpaj
|
|
199
199
|
metaflow/plugins/aws/step_functions/schedule_decorator.py,sha256=Ab1rW8O_no4HNZm4__iBmFDCDW0Z8-TgK4lnxHHA6HI,1940
|
200
200
|
metaflow/plugins/aws/step_functions/set_batch_environment.py,sha256=ibiGWFHDjKcLfprH3OsX-g2M9lUsh6J-bp7v2cdLhD4,1294
|
201
201
|
metaflow/plugins/aws/step_functions/step_functions.py,sha256=8tBs4pvdIVgNoZRm0Dzh4HK4HW3JT1OvtsB4cmUo8no,51855
|
202
|
-
metaflow/plugins/aws/step_functions/step_functions_cli.py,sha256=
|
202
|
+
metaflow/plugins/aws/step_functions/step_functions_cli.py,sha256=riTMPZBiQ0mhBEN-t1ApR_wIyz_q5lmHZta68oIoQWc,25789
|
203
203
|
metaflow/plugins/aws/step_functions/step_functions_client.py,sha256=DKpNwAIWElvWjFANs5Ku3rgzjxFoqAD6k-EF8Xhkg3Q,4754
|
204
204
|
metaflow/plugins/aws/step_functions/step_functions_decorator.py,sha256=9hw_MX36RyFp6IowuAYaJzJg9UC5KCe1FNt1PcG7_J0,3791
|
205
205
|
metaflow/plugins/aws/step_functions/step_functions_deployer.py,sha256=WrfQjvXnnInXwSePwoLUMb2EjqFG4RK1krO_8bW0qGI,7218
|
@@ -285,7 +285,7 @@ metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=sFZ9Zrjef85vCO0MGpUF-em8Pw3
|
|
285
285
|
metaflow/plugins/kubernetes/kubernetes_client.py,sha256=tuvXP-QKpdeSmzVolB2R_TaacOr5DIb0j642eKcjsiM,6491
|
286
286
|
metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=p7-6dyzxoMWKkuCqW6JvvCs_dZUgnBTOchZGIRS_yjo,27820
|
287
287
|
metaflow/plugins/kubernetes/kubernetes_job.py,sha256=Cfkee8LbXC17jSXWoeNdomQRvF_8YSeXNg1gvxm6E_M,31806
|
288
|
-
metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=
|
288
|
+
metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=TNj1qLAL_yjmMmj5Xh0lReOLgMn730Hy6W4pS1S-dSs,41027
|
289
289
|
metaflow/plugins/metadata/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
290
290
|
metaflow/plugins/metadata/local.py,sha256=YhLJC5zjVJrvQFIyQ92ZBByiUmhCC762RUX7ITX12O8,22428
|
291
291
|
metaflow/plugins/metadata/service.py,sha256=ihq5F7KQZlxvYwzH_-jyP2aWN_I96i2vp92j_d697s8,20204
|
@@ -304,7 +304,7 @@ metaflow/plugins/secrets/secrets_decorator.py,sha256=s-sFzPWOjahhpr5fMj-ZEaHkDYA
|
|
304
304
|
metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
305
305
|
metaflow/runner/click_api.py,sha256=Qfg4BOz5K2LaXTYBsi1y4zTfNIsGGHBVF3UkorX_-o8,13878
|
306
306
|
metaflow/runner/deployer.py,sha256=xNqxFIezWz3AcVqR4jeL-JnInGtefwEYMbXttxgB_I8,12276
|
307
|
-
metaflow/runner/metaflow_runner.py,sha256=
|
307
|
+
metaflow/runner/metaflow_runner.py,sha256=dHPBefSv9Ev4TB_P32T-7uPalb9R25mHLK8wz2-U8UI,14514
|
308
308
|
metaflow/runner/nbdeploy.py,sha256=lCCeSh_qKVA-LNMwmh3DtEPEfGRYNaLNcwldCF2JzXg,4130
|
309
309
|
metaflow/runner/nbrun.py,sha256=yGOSux8rs97Cix9LsjfPrSgV4rDmiyopjquV8GoCT7s,7276
|
310
310
|
metaflow/runner/subprocess_manager.py,sha256=jC_PIYIeAp_G__lf6WHZF3Lxzpp-WAQleMrRZq9j7nc,20467
|
@@ -315,8 +315,8 @@ metaflow/sidecar/sidecar_messages.py,sha256=zPsCoYgDIcDkkvdC9MEpJTJ3y6TSGm2JWkRc
|
|
315
315
|
metaflow/sidecar/sidecar_subprocess.py,sha256=f72n5iJJAYfCIbz4D94-RxR37VvM7kVvE3c8E9dYHe8,9708
|
316
316
|
metaflow/sidecar/sidecar_worker.py,sha256=4DfpxtnuphngOnIehKjNR_Knhu1hY7DYBcHl4Svpe3Y,2050
|
317
317
|
metaflow/system/__init__.py,sha256=SB9Py7Acecqi76MY9MonSHXFuDD1yIJEGJtEQH8cNq4,149
|
318
|
-
metaflow/system/system_logger.py,sha256=
|
319
|
-
metaflow/system/system_monitor.py,sha256=
|
318
|
+
metaflow/system/system_logger.py,sha256=NwL40-cHnUud58WRW3v7hke5IWIpwDdTnM8USiu-uio,2726
|
319
|
+
metaflow/system/system_monitor.py,sha256=6h3EAisD762NzL7fMuc7RQGygnhp0qqD97kDHrTacU8,2939
|
320
320
|
metaflow/system/system_utils.py,sha256=E5C66_oeAxEE4-2okKlA-X6rgeYndPFP7KmWKHvvFZ8,657
|
321
321
|
metaflow/tracing/__init__.py,sha256=xYTOT5BS5jbwhjk6hskxqNSU9st2LYtfeLN2Hknm3EI,1551
|
322
322
|
metaflow/tracing/propagator.py,sha256=AdPeAqoeRauH82pTw01hLFNPRAzm29nlwM7C2iqKFFk,2502
|
@@ -345,9 +345,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
|
|
345
345
|
metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
|
346
346
|
metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
|
347
347
|
metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
|
348
|
-
metaflow-2.12.
|
349
|
-
metaflow-2.12.
|
350
|
-
metaflow-2.12.
|
351
|
-
metaflow-2.12.
|
352
|
-
metaflow-2.12.
|
353
|
-
metaflow-2.12.
|
348
|
+
metaflow-2.12.28.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
|
349
|
+
metaflow-2.12.28.dist-info/METADATA,sha256=SjP7clYCqBNcP7yvau5BOwsQBgPLQKgcnQI9rNIyHFI,5907
|
350
|
+
metaflow-2.12.28.dist-info/WHEEL,sha256=OpXWERl2xLPRHTvd2ZXo_iluPEQd8uSbYkJ53NAER_Y,109
|
351
|
+
metaflow-2.12.28.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
|
352
|
+
metaflow-2.12.28.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
|
353
|
+
metaflow-2.12.28.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|