metaflow 2.12.12__py2.py3-none-any.whl → 2.12.13__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow/cli.py +3 -3
- metaflow/client/core.py +2 -0
- metaflow/flowspec.py +2 -1
- metaflow/graph.py +7 -3
- metaflow/metaflow_config.py +11 -2
- metaflow/metaflow_config_funcs.py +51 -3
- metaflow/metaflow_environment.py +36 -21
- metaflow/plugins/argo/argo_workflows_cli.py +2 -2
- metaflow/plugins/aws/aws_utils.py +16 -0
- metaflow/plugins/kubernetes/kubernetes_decorator.py +14 -0
- metaflow/runtime.py +26 -13
- metaflow/util.py +19 -0
- metaflow/version.py +1 -1
- {metaflow-2.12.12.dist-info → metaflow-2.12.13.dist-info}/METADATA +2 -2
- {metaflow-2.12.12.dist-info → metaflow-2.12.13.dist-info}/RECORD +19 -19
- {metaflow-2.12.12.dist-info → metaflow-2.12.13.dist-info}/WHEEL +1 -1
- {metaflow-2.12.12.dist-info → metaflow-2.12.13.dist-info}/LICENSE +0 -0
- {metaflow-2.12.12.dist-info → metaflow-2.12.13.dist-info}/entry_points.txt +0 -0
- {metaflow-2.12.12.dist-info → metaflow-2.12.13.dist-info}/top_level.txt +0 -0
metaflow/cli.py
CHANGED
@@ -650,7 +650,7 @@ def resume(
|
|
650
650
|
)
|
651
651
|
|
652
652
|
if step_to_rerun is None:
|
653
|
-
|
653
|
+
steps_to_rerun = set()
|
654
654
|
else:
|
655
655
|
# validate step name
|
656
656
|
if step_to_rerun not in obj.graph.nodes:
|
@@ -660,7 +660,7 @@ def resume(
|
|
660
660
|
step_to_rerun, ",".join(list(obj.graph.nodes.keys()))
|
661
661
|
)
|
662
662
|
)
|
663
|
-
|
663
|
+
steps_to_rerun = {step_to_rerun}
|
664
664
|
|
665
665
|
if run_id:
|
666
666
|
# Run-ids that are provided by the metadata service are always integers.
|
@@ -688,7 +688,7 @@ def resume(
|
|
688
688
|
clone_run_id=origin_run_id,
|
689
689
|
clone_only=clone_only,
|
690
690
|
reentrant=reentrant,
|
691
|
-
|
691
|
+
steps_to_rerun=steps_to_rerun,
|
692
692
|
max_workers=max_workers,
|
693
693
|
max_num_splits=max_num_splits,
|
694
694
|
max_log_size=max_log_size * 1024 * 1024,
|
metaflow/client/core.py
CHANGED
@@ -1027,6 +1027,8 @@ class MetaflowData(object):
|
|
1027
1027
|
self._artifacts = dict((art.id, art) for art in artifacts)
|
1028
1028
|
|
1029
1029
|
def __getattr__(self, name: str):
|
1030
|
+
if name not in self._artifacts:
|
1031
|
+
raise AttributeError(name)
|
1030
1032
|
return self._artifacts[name].data
|
1031
1033
|
|
1032
1034
|
def __contains__(self, var):
|
metaflow/flowspec.py
CHANGED
@@ -17,6 +17,7 @@ from .exception import (
|
|
17
17
|
)
|
18
18
|
from .graph import FlowGraph
|
19
19
|
from .unbounded_foreach import UnboundedForeachInput
|
20
|
+
from .util import to_pod
|
20
21
|
from .metaflow_config import INCLUDE_FOREACH_STACK, MAXIMUM_FOREACH_VALUE_CHARS
|
21
22
|
|
22
23
|
# For Python 3 compatibility
|
@@ -201,7 +202,7 @@ class FlowSpec(metaclass=_FlowSpecMeta):
|
|
201
202
|
"decorators": [
|
202
203
|
{
|
203
204
|
"name": deco.name,
|
204
|
-
"attributes": deco.attributes,
|
205
|
+
"attributes": to_pod(deco.attributes),
|
205
206
|
"statically_defined": deco.statically_defined,
|
206
207
|
}
|
207
208
|
for deco in flow_decorators(self)
|
metaflow/graph.py
CHANGED
@@ -3,6 +3,9 @@ import ast
|
|
3
3
|
import re
|
4
4
|
|
5
5
|
|
6
|
+
from .util import to_pod
|
7
|
+
|
8
|
+
|
6
9
|
def deindent_docstring(doc):
|
7
10
|
if doc:
|
8
11
|
# Find the indent to remove from the docstring. We consider the following possibilities:
|
@@ -72,7 +75,6 @@ class DAGNode(object):
|
|
72
75
|
return "%s.%s" % (expr.value.id, expr.attr)
|
73
76
|
|
74
77
|
def _parse(self, func_ast):
|
75
|
-
|
76
78
|
self.num_args = len(func_ast.args.args)
|
77
79
|
tail = func_ast.body[-1]
|
78
80
|
|
@@ -171,6 +173,8 @@ class FlowGraph(object):
|
|
171
173
|
self.name = flow.__name__
|
172
174
|
self.nodes = self._create_nodes(flow)
|
173
175
|
self.doc = deindent_docstring(flow.__doc__)
|
176
|
+
# nodes sorted in topological order.
|
177
|
+
self.sorted_nodes = []
|
174
178
|
self._traverse_graph()
|
175
179
|
self._postprocess()
|
176
180
|
|
@@ -197,6 +201,7 @@ class FlowGraph(object):
|
|
197
201
|
|
198
202
|
def _traverse_graph(self):
|
199
203
|
def traverse(node, seen, split_parents):
|
204
|
+
self.sorted_nodes.append(node.name)
|
200
205
|
if node.type in ("split", "foreach"):
|
201
206
|
node.split_parents = split_parents
|
202
207
|
split_parents = split_parents + [node.name]
|
@@ -262,7 +267,6 @@ class FlowGraph(object):
|
|
262
267
|
)
|
263
268
|
|
264
269
|
def output_steps(self):
|
265
|
-
|
266
270
|
steps_info = {}
|
267
271
|
graph_structure = []
|
268
272
|
|
@@ -286,7 +290,7 @@ class FlowGraph(object):
|
|
286
290
|
"decorators": [
|
287
291
|
{
|
288
292
|
"name": deco.name,
|
289
|
-
"attributes": deco.attributes,
|
293
|
+
"attributes": to_pod(deco.attributes),
|
290
294
|
"statically_defined": deco.statically_defined,
|
291
295
|
}
|
292
296
|
for deco in node.decorators
|
metaflow/metaflow_config.py
CHANGED
@@ -15,6 +15,17 @@ if sys.platform == "darwin":
|
|
15
15
|
## value, either set `METAFLOW_DEFAULT_DATASTORE` in your configuration file or set
|
16
16
|
## an environment variable called `METAFLOW_DEFAULT_DATASTORE`
|
17
17
|
|
18
|
+
##
|
19
|
+
# Constants (NOTE: these need to live before any from_conf)
|
20
|
+
##
|
21
|
+
|
22
|
+
# Path to the local directory to store artifacts for 'local' datastore.
|
23
|
+
DATASTORE_LOCAL_DIR = ".metaflow"
|
24
|
+
|
25
|
+
# Local configuration file (in .metaflow) containing overrides per-project
|
26
|
+
LOCAL_CONFIG_FILE = "config.json"
|
27
|
+
|
28
|
+
|
18
29
|
###
|
19
30
|
# Default configuration
|
20
31
|
###
|
@@ -42,8 +53,6 @@ USER = from_conf("USER")
|
|
42
53
|
###
|
43
54
|
# Datastore configuration
|
44
55
|
###
|
45
|
-
# Path to the local directory to store artifacts for 'local' datastore.
|
46
|
-
DATASTORE_LOCAL_DIR = ".metaflow"
|
47
56
|
DATASTORE_SYSROOT_LOCAL = from_conf("DATASTORE_SYSROOT_LOCAL")
|
48
57
|
# S3 bucket and prefix to store artifacts for 's3' datastore.
|
49
58
|
DATASTORE_SYSROOT_S3 = from_conf("DATASTORE_SYSROOT_S3")
|
@@ -32,8 +32,40 @@ def init_config():
|
|
32
32
|
return config
|
33
33
|
|
34
34
|
|
35
|
+
def init_local_config():
|
36
|
+
# This function is heavily inspired from LocalStorage.get_datastore_root_from_config
|
37
|
+
# but simplifies certain things and also does not depend on DATASTORE_SYSROOT_LOCAL.
|
38
|
+
#
|
39
|
+
# In other words, since this config is meant to be local to a directory, it does not
|
40
|
+
# check in DATASTORE_SYSROOT_LOCAL but only up the current getcwd() path. This also
|
41
|
+
# prevents nasty circular dependencies :)
|
42
|
+
|
43
|
+
from metaflow.metaflow_config import DATASTORE_LOCAL_DIR, LOCAL_CONFIG_FILE
|
44
|
+
|
45
|
+
current_path = os.getcwd()
|
46
|
+
check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR)
|
47
|
+
check_dir = os.path.realpath(check_dir)
|
48
|
+
while not os.path.isdir(check_dir):
|
49
|
+
new_path = os.path.dirname(current_path)
|
50
|
+
if new_path == current_path: # No longer making upward progress
|
51
|
+
return {}
|
52
|
+
current_path = new_path
|
53
|
+
check_dir = os.path.join(current_path, DATASTORE_LOCAL_DIR)
|
54
|
+
path_to_config = os.path.join(check_dir, LOCAL_CONFIG_FILE)
|
55
|
+
# We found a directory to look for the config file in
|
56
|
+
if os.path.exists(path_to_config):
|
57
|
+
with open(path_to_config, encoding="utf-8") as f:
|
58
|
+
return json.load(f)
|
59
|
+
return {}
|
60
|
+
|
61
|
+
|
35
62
|
# Initialize defaults required to setup environment variables.
|
36
|
-
|
63
|
+
# (initialized lazily in from_conf since init_local_config requires
|
64
|
+
# some configuration values
|
65
|
+
|
66
|
+
METAFLOW_CONFIG = None
|
67
|
+
|
68
|
+
METAFLOW_LOCAL_CONFIG = None
|
37
69
|
|
38
70
|
_all_configs = {}
|
39
71
|
|
@@ -51,7 +83,13 @@ def config_values(include=0):
|
|
51
83
|
|
52
84
|
def from_conf(name, default=None, validate_fn=None):
|
53
85
|
"""
|
54
|
-
|
86
|
+
Pull value from the environment or configuration.
|
87
|
+
Order is:
|
88
|
+
1. Environment (use any environment variable explicitly set by user)
|
89
|
+
2. Local config (use any value set in the local config file -- so stuff in
|
90
|
+
.metaflow/project.json for example)
|
91
|
+
3. Global config (use any value set in the global config file)
|
92
|
+
4. Default
|
55
93
|
|
56
94
|
Prior to a value being returned, we will validate using validate_fn (if provided).
|
57
95
|
Only non-None values are validated.
|
@@ -59,9 +97,19 @@ def from_conf(name, default=None, validate_fn=None):
|
|
59
97
|
validate_fn should accept (name, value).
|
60
98
|
If the value validates, return None, else raise an MetaflowException.
|
61
99
|
"""
|
100
|
+
global METAFLOW_CONFIG, METAFLOW_LOCAL_CONFIG
|
101
|
+
|
102
|
+
if METAFLOW_CONFIG is None:
|
103
|
+
METAFLOW_CONFIG = init_config()
|
104
|
+
if METAFLOW_LOCAL_CONFIG is None:
|
105
|
+
METAFLOW_LOCAL_CONFIG = init_local_config()
|
106
|
+
|
62
107
|
is_default = True
|
63
108
|
env_name = "METAFLOW_%s" % name
|
64
|
-
value = os.environ.get(
|
109
|
+
value = os.environ.get(
|
110
|
+
env_name,
|
111
|
+
METAFLOW_LOCAL_CONFIG.get(env_name, METAFLOW_CONFIG.get(env_name, default)),
|
112
|
+
)
|
65
113
|
if validate_fn and value is not None:
|
66
114
|
validate_fn(env_name, value)
|
67
115
|
if default is not None:
|
metaflow/metaflow_environment.py
CHANGED
@@ -89,10 +89,16 @@ class MetaflowEnvironment(object):
|
|
89
89
|
It should work silently if everything goes well.
|
90
90
|
"""
|
91
91
|
if datastore_type == "s3":
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
92
|
+
from .plugins.aws.aws_utils import parse_s3_full_path
|
93
|
+
|
94
|
+
bucket, s3_object = parse_s3_full_path(code_package_url)
|
95
|
+
# NOTE: the script quoting is extremely sensitive due to the way shlex.split operates and this being inserted
|
96
|
+
# into a quoted command elsewhere.
|
97
|
+
return "{python} -c '{script}'".format(
|
98
|
+
python=self._python(),
|
99
|
+
script='import boto3, os; boto3.client(\\"s3\\", endpoint_url=os.getenv(\\"METAFLOW_S3_ENDPOINT_URL\\")).download_file(\\"%s\\", \\"%s\\", \\"job.tar\\")'
|
100
|
+
% (bucket, s3_object),
|
101
|
+
)
|
96
102
|
elif datastore_type == "azure":
|
97
103
|
from .plugins.azure.azure_utils import parse_azure_full_path
|
98
104
|
|
@@ -119,25 +125,34 @@ class MetaflowEnvironment(object):
|
|
119
125
|
)
|
120
126
|
|
121
127
|
def _get_install_dependencies_cmd(self, datastore_type):
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
"
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
128
|
+
base_cmd = "{} -m pip install -qqq".format(self._python())
|
129
|
+
|
130
|
+
datastore_packages = {
|
131
|
+
"s3": ["boto3"],
|
132
|
+
"azure": [
|
133
|
+
"azure-identity",
|
134
|
+
"azure-storage-blob",
|
135
|
+
"azure-keyvault-secrets",
|
136
|
+
"simple-azure-blob-downloader",
|
137
|
+
],
|
138
|
+
"gs": [
|
139
|
+
"google-cloud-storage",
|
140
|
+
"google-auth",
|
141
|
+
"simple-gcp-object-downloader",
|
142
|
+
"google-cloud-secret-manager",
|
143
|
+
],
|
144
|
+
}
|
145
|
+
|
146
|
+
if datastore_type not in datastore_packages:
|
136
147
|
raise NotImplementedError(
|
137
|
-
"
|
138
|
-
% datastore_type
|
148
|
+
"Unknown datastore type: {}".format(datastore_type)
|
139
149
|
)
|
140
|
-
|
150
|
+
|
151
|
+
cmd = "{} {}".format(
|
152
|
+
base_cmd, " ".join(datastore_packages[datastore_type] + ["requests"])
|
153
|
+
)
|
154
|
+
# skip pip installs if we know that packages might already be available
|
155
|
+
return "if [ -z $METAFLOW_SKIP_INSTALL_DEPENDENCIES ]; then {}; fi".format(cmd)
|
141
156
|
|
142
157
|
def get_package_commands(self, code_package_url, datastore_type):
|
143
158
|
cmds = [
|
@@ -172,7 +172,7 @@ def argo_workflows(obj, name=None):
|
|
172
172
|
)
|
173
173
|
@click.option(
|
174
174
|
"--enable-heartbeat-daemon/--no-enable-heartbeat-daemon",
|
175
|
-
default=
|
175
|
+
default=True,
|
176
176
|
show_default=True,
|
177
177
|
help="Use a daemon container to broadcast heartbeats.",
|
178
178
|
)
|
@@ -186,7 +186,7 @@ def argo_workflows(obj, name=None):
|
|
186
186
|
)
|
187
187
|
@click.option(
|
188
188
|
"--enable-error-msg-capture/--no-enable-error-msg-capture",
|
189
|
-
default=
|
189
|
+
default=True,
|
190
190
|
show_default=True,
|
191
191
|
help="Capture stack trace of first failed task in exit hook.",
|
192
192
|
)
|
@@ -4,6 +4,22 @@ import requests
|
|
4
4
|
from metaflow.exception import MetaflowException
|
5
5
|
|
6
6
|
|
7
|
+
def parse_s3_full_path(s3_uri):
|
8
|
+
from urllib.parse import urlparse
|
9
|
+
|
10
|
+
# <scheme>://<netloc>/<path>;<params>?<query>#<fragment>
|
11
|
+
scheme, netloc, path, _, _, _ = urlparse(s3_uri)
|
12
|
+
assert scheme == "s3"
|
13
|
+
assert netloc is not None
|
14
|
+
|
15
|
+
bucket = netloc
|
16
|
+
path = path.lstrip("/").rstrip("/")
|
17
|
+
if path == "":
|
18
|
+
path = None
|
19
|
+
|
20
|
+
return bucket, path
|
21
|
+
|
22
|
+
|
7
23
|
def get_ec2_instance_metadata():
|
8
24
|
"""
|
9
25
|
Fetches the EC2 instance metadata through AWS instance metadata service
|
@@ -97,6 +97,9 @@ class KubernetesDecorator(StepDecorator):
|
|
97
97
|
Shared memory size (in MiB) required for this step
|
98
98
|
port: int, optional
|
99
99
|
Port number to specify in the Kubernetes job object
|
100
|
+
compute_pool : str, optional, default None
|
101
|
+
Compute pool to be used for for this step.
|
102
|
+
If not specified, any accessible compute pool within the perimeter is used.
|
100
103
|
"""
|
101
104
|
|
102
105
|
name = "kubernetes"
|
@@ -121,6 +124,7 @@ class KubernetesDecorator(StepDecorator):
|
|
121
124
|
"persistent_volume_claims": None, # e.g., {"pvc-name": "/mnt/vol", "another-pvc": "/mnt/vol2"}
|
122
125
|
"shared_memory": None,
|
123
126
|
"port": None,
|
127
|
+
"compute_pool": None,
|
124
128
|
}
|
125
129
|
package_url = None
|
126
130
|
package_sha = None
|
@@ -153,6 +157,12 @@ class KubernetesDecorator(StepDecorator):
|
|
153
157
|
self.attributes["node_selector"] = parse_kube_keyvalue_list(
|
154
158
|
self.attributes["node_selector"].split(",")
|
155
159
|
)
|
160
|
+
if self.attributes["compute_pool"]:
|
161
|
+
if self.attributes["node_selector"] is None:
|
162
|
+
self.attributes["node_selector"] = {}
|
163
|
+
self.attributes["node_selector"].update(
|
164
|
+
{"outerbounds.co/compute-pool": self.attributes["compute_pool"]}
|
165
|
+
)
|
156
166
|
|
157
167
|
if self.attributes["tolerations"]:
|
158
168
|
try:
|
@@ -370,9 +380,13 @@ class KubernetesDecorator(StepDecorator):
|
|
370
380
|
cli_args.command_args.append(self.package_sha)
|
371
381
|
cli_args.command_args.append(self.package_url)
|
372
382
|
|
383
|
+
# skip certain keys as CLI arguments
|
384
|
+
_skip_keys = ["compute_pool"]
|
373
385
|
# --namespace is used to specify Metaflow namespace (a different
|
374
386
|
# concept from k8s namespace).
|
375
387
|
for k, v in self.attributes.items():
|
388
|
+
if k in _skip_keys:
|
389
|
+
continue
|
376
390
|
if k == "namespace":
|
377
391
|
cli_args.command_options["k8s_namespace"] = v
|
378
392
|
elif k in {"node_selector"} and v:
|
metaflow/runtime.py
CHANGED
@@ -76,7 +76,7 @@ class NativeRuntime(object):
|
|
76
76
|
clone_run_id=None,
|
77
77
|
clone_only=False,
|
78
78
|
reentrant=False,
|
79
|
-
|
79
|
+
steps_to_rerun=None,
|
80
80
|
max_workers=MAX_WORKERS,
|
81
81
|
max_num_splits=MAX_NUM_SPLITS,
|
82
82
|
max_log_size=MAX_LOG_SIZE,
|
@@ -110,12 +110,21 @@ class NativeRuntime(object):
|
|
110
110
|
|
111
111
|
self._clone_run_id = clone_run_id
|
112
112
|
self._clone_only = clone_only
|
113
|
-
self._clone_steps = {} if clone_steps is None else clone_steps
|
114
113
|
self._cloned_tasks = []
|
115
114
|
self._cloned_task_index = set()
|
116
115
|
self._reentrant = reentrant
|
117
116
|
self._run_url = None
|
118
117
|
|
118
|
+
# If steps_to_rerun is specified, we will not clone them in resume mode.
|
119
|
+
self._steps_to_rerun = steps_to_rerun or {}
|
120
|
+
# sorted_nodes are in topological order already, so we only need to
|
121
|
+
# iterate through the nodes once to get a stable set of rerun steps.
|
122
|
+
for step_name in self._graph.sorted_nodes:
|
123
|
+
if step_name in self._steps_to_rerun:
|
124
|
+
out_funcs = self._graph[step_name].out_funcs or []
|
125
|
+
for next_step in out_funcs:
|
126
|
+
self._steps_to_rerun.add(next_step)
|
127
|
+
|
119
128
|
self._origin_ds_set = None
|
120
129
|
if clone_run_id:
|
121
130
|
# resume logic
|
@@ -166,7 +175,7 @@ class NativeRuntime(object):
|
|
166
175
|
else:
|
167
176
|
may_clone = all(self._is_cloned[path] for path in input_paths)
|
168
177
|
|
169
|
-
if step in self.
|
178
|
+
if step in self._steps_to_rerun:
|
170
179
|
may_clone = False
|
171
180
|
|
172
181
|
if step == "_parameters":
|
@@ -301,16 +310,14 @@ class NativeRuntime(object):
|
|
301
310
|
)
|
302
311
|
except Exception as e:
|
303
312
|
self._logger(
|
304
|
-
"Cloning
|
305
|
-
self._clone_run_id, step_name, task_id, str(e)
|
313
|
+
"Cloning {}/{}/{}/{} failed with error: {}".format(
|
314
|
+
self._flow.name, self._clone_run_id, step_name, task_id, str(e)
|
306
315
|
)
|
307
316
|
)
|
308
317
|
|
309
318
|
def clone_original_run(self, generate_task_obj=False, verbose=True):
|
310
319
|
self._logger(
|
311
|
-
"
|
312
|
-
self._flow.name, self._clone_run_id
|
313
|
-
),
|
320
|
+
"Cloning {}/{}".format(self._flow.name, self._clone_run_id),
|
314
321
|
system_msg=True,
|
315
322
|
)
|
316
323
|
|
@@ -336,7 +343,11 @@ class NativeRuntime(object):
|
|
336
343
|
_, step_name, task_id = task_ds.pathspec.split("/")
|
337
344
|
pathspec_index = task_ds.pathspec_index
|
338
345
|
|
339
|
-
if
|
346
|
+
if (
|
347
|
+
task_ds["_task_ok"]
|
348
|
+
and step_name != "_parameters"
|
349
|
+
and (step_name not in self._steps_to_rerun)
|
350
|
+
):
|
340
351
|
# "_unbounded_foreach" is a special flag to indicate that the transition is an unbounded foreach.
|
341
352
|
# Both parent and splitted children tasks will have this flag set. The splitted control/mapper tasks
|
342
353
|
# have no "foreach_param" because UBF is always followed by a join step.
|
@@ -390,7 +401,9 @@ class NativeRuntime(object):
|
|
390
401
|
) in inputs
|
391
402
|
]
|
392
403
|
_, _ = futures.wait(all_tasks)
|
393
|
-
self._logger(
|
404
|
+
self._logger(
|
405
|
+
"{}/{} cloned!".format(self._flow.name, self._clone_run_id), system_msg=True
|
406
|
+
)
|
394
407
|
self._params_task.mark_resume_done()
|
395
408
|
|
396
409
|
def execute(self):
|
@@ -1149,13 +1162,13 @@ class Task(object):
|
|
1149
1162
|
self._should_skip_cloning = task_completed
|
1150
1163
|
if self._should_skip_cloning:
|
1151
1164
|
self.log(
|
1152
|
-
"
|
1165
|
+
"Skipping cloning of previously run task %s"
|
1166
|
+
% self.clone_origin,
|
1153
1167
|
system_msg=True,
|
1154
1168
|
)
|
1155
1169
|
else:
|
1156
1170
|
self.log(
|
1157
|
-
"Cloning
|
1158
|
-
% self.clone_origin,
|
1171
|
+
"Cloning previously run task %s" % self.clone_origin,
|
1159
1172
|
system_msg=True,
|
1160
1173
|
)
|
1161
1174
|
else:
|
metaflow/util.py
CHANGED
@@ -426,6 +426,25 @@ def tar_safe_extract(tar, path=".", members=None, *, numeric_owner=False):
|
|
426
426
|
tar.extractall(path, members, numeric_owner=numeric_owner)
|
427
427
|
|
428
428
|
|
429
|
+
def to_pod(value):
|
430
|
+
"""
|
431
|
+
Convert a python object to plain-old-data (POD) format.
|
432
|
+
|
433
|
+
Parameters
|
434
|
+
----------
|
435
|
+
value : Any
|
436
|
+
Value to convert to POD format. The value can be a string, number, list,
|
437
|
+
dictionary, or a nested structure of these types.
|
438
|
+
"""
|
439
|
+
if isinstance(value, (str, int, float)):
|
440
|
+
return value
|
441
|
+
if isinstance(value, dict):
|
442
|
+
return {to_pod(k): to_pod(v) for k, v in value.items()}
|
443
|
+
if isinstance(value, (list, set, tuple)):
|
444
|
+
return [to_pod(v) for v in value]
|
445
|
+
return str(value)
|
446
|
+
|
447
|
+
|
429
448
|
if sys.version_info[:2] > (3, 5):
|
430
449
|
from metaflow._vendor.packaging.version import parse as version_parse
|
431
450
|
else:
|
metaflow/version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
metaflow_version = "2.12.
|
1
|
+
metaflow_version = "2.12.13"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: metaflow
|
3
|
-
Version: 2.12.
|
3
|
+
Version: 2.12.13
|
4
4
|
Summary: Metaflow: More Data Science, Less Engineering
|
5
5
|
Author: Metaflow Developers
|
6
6
|
Author-email: help@metaflow.org
|
@@ -26,7 +26,7 @@ License-File: LICENSE
|
|
26
26
|
Requires-Dist: requests
|
27
27
|
Requires-Dist: boto3
|
28
28
|
Provides-Extra: stubs
|
29
|
-
Requires-Dist: metaflow-stubs==2.12.
|
29
|
+
Requires-Dist: metaflow-stubs==2.12.13; extra == "stubs"
|
30
30
|
|
31
31
|

|
32
32
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
metaflow/R.py,sha256=CqVfIatvmjciuICNnoyyNGrwE7Va9iXfLdFbQa52hwA,3958
|
2
2
|
metaflow/__init__.py,sha256=1TJeSmvQVELJpVGh9s9GoB6aBIK0_zEl9mpW4eSz64Q,5971
|
3
3
|
metaflow/cards.py,sha256=tP1_RrtmqdFh741pqE4t98S7SA0MtGRlGvRICRZF1Mg,426
|
4
|
-
metaflow/cli.py,sha256=
|
4
|
+
metaflow/cli.py,sha256=fa6dx6F2PKtq0oeMxRM2efA9LyE108jGZQvYCtiUS94,34274
|
5
5
|
metaflow/cli_args.py,sha256=lcgBGNTvfaiPxiUnejAe60Upt9swG6lRy1_3OqbU6MY,2616
|
6
6
|
metaflow/clone_util.py,sha256=XfUX0vssu_hPlyZfhFl1AOnKkLqvt33Qp8xNrmdocGg,2057
|
7
7
|
metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
|
@@ -10,15 +10,15 @@ metaflow/decorators.py,sha256=hbJwRlZuLbl6t7fOsTiH7Sux4Lx6zgEEpldIXoM5TQc,21540
|
|
10
10
|
metaflow/event_logger.py,sha256=joTVRqZPL87nvah4ZOwtqWX8NeraM_CXKXXGVpKGD8o,780
|
11
11
|
metaflow/events.py,sha256=ahjzkSbSnRCK9RZ-9vTfUviz_6gMvSO9DGkJ86X80-k,5300
|
12
12
|
metaflow/exception.py,sha256=KC1LHJQzzYkWib0DeQ4l_A2r8VaudywsSqIQuq1RDZU,4954
|
13
|
-
metaflow/flowspec.py,sha256=
|
14
|
-
metaflow/graph.py,sha256=
|
13
|
+
metaflow/flowspec.py,sha256=0-NnCKiX4oJ21Spb5PJkl2IvHoJxBcpi8CQrZEGB2Ag,27178
|
14
|
+
metaflow/graph.py,sha256=HFJ7V_bPSht_NHIm8BejrSqOX2fyBQpVOczRCliRw08,11975
|
15
15
|
metaflow/includefile.py,sha256=yHczcZ_U0SrasxSNhZb3DIBzx8UZnrJCl3FzvpEQLOA,19753
|
16
16
|
metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
|
17
17
|
metaflow/lint.py,sha256=5rj1MlpluxyPTSINjtMoJ7viotyNzfjtBJSAihlAwMU,10870
|
18
|
-
metaflow/metaflow_config.py,sha256=
|
19
|
-
metaflow/metaflow_config_funcs.py,sha256=
|
18
|
+
metaflow/metaflow_config.py,sha256=4PUd2-JWJs35SaobDgMg4RdpZaKjhEqPUFh2f0pjqnU,22853
|
19
|
+
metaflow/metaflow_config_funcs.py,sha256=5GlvoafV6SxykwfL8D12WXSfwjBN_NsyuKE_Q3gjGVE,6738
|
20
20
|
metaflow/metaflow_current.py,sha256=5Kri7fzj-rtIJVr5xh5kPKwZ0T73_4egZybzlDR-fgc,7136
|
21
|
-
metaflow/metaflow_environment.py,sha256=
|
21
|
+
metaflow/metaflow_environment.py,sha256=xTVowXgia8puUcA0rcNn7iJpFao7e_TRp30RAnB_TW4,7910
|
22
22
|
metaflow/metaflow_profile.py,sha256=jKPEW-hmAQO-htSxb9hXaeloLacAh41A35rMZH6G8pA,418
|
23
23
|
metaflow/metaflow_version.py,sha256=mPQ6g_3XjNdi0NrxDzwlW8ZH0nMyYpwqmJ04P7TIdP0,4774
|
24
24
|
metaflow/monitor.py,sha256=T0NMaBPvXynlJAO_avKtk8OIIRMyEuMAyF8bIp79aZU,5323
|
@@ -28,14 +28,14 @@ metaflow/parameters.py,sha256=l8qnhBG9C4wf_FkXWjq5sapUA6npLdR7pyB0PPQ-KF0,15712
|
|
28
28
|
metaflow/procpoll.py,sha256=U2tE4iK_Mwj2WDyVTx_Uglh6xZ-jixQOo4wrM9OOhxg,2859
|
29
29
|
metaflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
30
30
|
metaflow/pylint_wrapper.py,sha256=zzBY9YaSUZOGH-ypDKAv2B_7XcoyMZj-zCoCrmYqNRc,2865
|
31
|
-
metaflow/runtime.py,sha256=
|
31
|
+
metaflow/runtime.py,sha256=fbBObJJciagHWPzR3T7x9e_jez_RBnLZIHsXMvYnW_M,68875
|
32
32
|
metaflow/tagging_util.py,sha256=ctyf0Q1gBi0RyZX6J0e9DQGNkNHblV_CITfy66axXB4,2346
|
33
33
|
metaflow/task.py,sha256=uJHl8K4n3jNllWHSsG1vAZtDza0U2QbQcdg9GS_YPBE,28660
|
34
34
|
metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
|
35
35
|
metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
|
36
|
-
metaflow/util.py,sha256=
|
36
|
+
metaflow/util.py,sha256=olAvJK3y1it_k99MhLulTaAJo7OFVt5rnrD-ulIFLCU,13616
|
37
37
|
metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
|
38
|
-
metaflow/version.py,sha256=
|
38
|
+
metaflow/version.py,sha256=0_24jJexuDqVeG3cwx7dMTGu2WyVyUIektnSTwPMkDc,29
|
39
39
|
metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
|
40
40
|
metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
|
41
41
|
metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
|
@@ -110,7 +110,7 @@ metaflow/_vendor/v3_6/importlib_metadata/_meta.py,sha256=_F48Hu_jFxkfKWz5wcYS8vO
|
|
110
110
|
metaflow/_vendor/v3_6/importlib_metadata/_text.py,sha256=HCsFksZpJLeTP3NEk_ngrAeXVRRtTrtyh9eOABoRP4A,2166
|
111
111
|
metaflow/_vendor/v3_6/importlib_metadata/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
112
112
|
metaflow/client/__init__.py,sha256=1GtQB4Y_CBkzaxg32L1syNQSlfj762wmLrfrDxGi1b8,226
|
113
|
-
metaflow/client/core.py,sha256=
|
113
|
+
metaflow/client/core.py,sha256=L4COrMyQgSAWkXIPoXFFQ3OqGnoHfePsBZdJQvShU1E,74162
|
114
114
|
metaflow/client/filecache.py,sha256=Wy0yhhCqC1JZgebqi7z52GCwXYnkAqMZHTtxThvwBgM,15229
|
115
115
|
metaflow/cmd/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
116
116
|
metaflow/cmd/configure_cmd.py,sha256=o-DKnUf2FBo_HiMVyoyzQaGBSMtpbEPEdFTQZ0hkU-k,33396
|
@@ -175,7 +175,7 @@ metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
|
|
175
175
|
metaflow/plugins/argo/argo_client.py,sha256=MKKhMCbWOPzf6z5zQQiyDRHHkAXcO7ipboDZDqAAvOk,15849
|
176
176
|
metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
|
177
177
|
metaflow/plugins/argo/argo_workflows.py,sha256=hAzp11YSd98GvnUHSwZkto7bMCY6hvwCaO1Lplru3EA,169793
|
178
|
-
metaflow/plugins/argo/argo_workflows_cli.py,sha256=
|
178
|
+
metaflow/plugins/argo/argo_workflows_cli.py,sha256=ydCf7P0lR8KaZIwYTtV_hRw1J49okE0wtEW8Kcj2c4c,35704
|
179
179
|
metaflow/plugins/argo/argo_workflows_decorator.py,sha256=yprszMdbE3rBTcEA9VR0IEnPjTprUauZBc4SBb-Q7sA,7878
|
180
180
|
metaflow/plugins/argo/argo_workflows_deployer.py,sha256=yMIXAVoAuBLHCqQyFriV_Wc_Lp5D041Ay83R5pYNoXE,8066
|
181
181
|
metaflow/plugins/argo/capture_error.py,sha256=Ys9dscGrTpW-ZCirLBU0gD9qBM0BjxyxGlUMKcwewQc,1852
|
@@ -184,7 +184,7 @@ metaflow/plugins/argo/generate_input_paths.py,sha256=loYsI6RFX9LlFsHb7Fe-mzlTTtR
|
|
184
184
|
metaflow/plugins/argo/jobset_input_paths.py,sha256=_JhZWngA6p9Q_O2fx3pdzKI0WE-HPRHz_zFvY2pHPTQ,525
|
185
185
|
metaflow/plugins/aws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
186
186
|
metaflow/plugins/aws/aws_client.py,sha256=mO8UD6pxFaOnxDb3hTP3HB7Gqb_ZxoR-76LT683WHvI,4036
|
187
|
-
metaflow/plugins/aws/aws_utils.py,sha256=
|
187
|
+
metaflow/plugins/aws/aws_utils.py,sha256=dk92IRZ2QTF3PicBOtZMMOmS_FIncFqZPeL9EbCtXak,7310
|
188
188
|
metaflow/plugins/aws/batch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
189
189
|
metaflow/plugins/aws/batch/batch.py,sha256=e9ssahWM18GnipPK2sqYB-ztx9w7Eoo7YtWyEtufYxs,17787
|
190
190
|
metaflow/plugins/aws/batch/batch_cli.py,sha256=6PTbyajRgdy0XmjyJLBTdKdiOB84dcovQQ8sFXlJqko,11749
|
@@ -282,7 +282,7 @@ metaflow/plugins/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMp
|
|
282
282
|
metaflow/plugins/kubernetes/kubernetes.py,sha256=cYtDuJEqd0TOKy5vK1UybNLxROhHBJdOtYBYJrWhSMo,30035
|
283
283
|
metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=qBDdr1Lvtt-RO9pB-9_HTOPdzAmDvvJ0aiQ1OoCcrMU,10892
|
284
284
|
metaflow/plugins/kubernetes/kubernetes_client.py,sha256=GKg-gT3qhXMRQV-sG1YyoOf3Z32NXr_wwEN2ytMVSEg,2471
|
285
|
-
metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=
|
285
|
+
metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=ahptZAzrPhoxEAQ4FgY9vlUnL8siX1jY02WSmW6jilc,25294
|
286
286
|
metaflow/plugins/kubernetes/kubernetes_job.py,sha256=Cfkee8LbXC17jSXWoeNdomQRvF_8YSeXNg1gvxm6E_M,31806
|
287
287
|
metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=OBmLtX-ZUDQdCCfftUmRMernfmTNMwdTxPoCAp_NmwE,40957
|
288
288
|
metaflow/plugins/metadata/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
@@ -344,9 +344,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
|
|
344
344
|
metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
|
345
345
|
metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
|
346
346
|
metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
|
347
|
-
metaflow-2.12.
|
348
|
-
metaflow-2.12.
|
349
|
-
metaflow-2.12.
|
350
|
-
metaflow-2.12.
|
351
|
-
metaflow-2.12.
|
352
|
-
metaflow-2.12.
|
347
|
+
metaflow-2.12.13.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
|
348
|
+
metaflow-2.12.13.dist-info/METADATA,sha256=MJ8kowl4Zycp108IIh3fZq5VlMdK5AOq5-yMml7t1Mk,5906
|
349
|
+
metaflow-2.12.13.dist-info/WHEEL,sha256=GUeE9LxUgRABPG7YM0jCNs9cBsAIx0YAkzCB88PMLgc,109
|
350
|
+
metaflow-2.12.13.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
|
351
|
+
metaflow-2.12.13.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
|
352
|
+
metaflow-2.12.13.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|