metaflow 2.12.13__py2.py3-none-any.whl → 2.12.15__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow/plugins/argo/argo_workflows.py +9 -0
- metaflow/plugins/argo/argo_workflows_cli.py +11 -0
- metaflow/plugins/argo/argo_workflows_deployer.py +3 -1
- metaflow/plugins/aws/batch/batch_decorator.py +1 -0
- metaflow/plugins/aws/step_functions/step_functions_deployer.py +3 -1
- metaflow/plugins/kubernetes/kubernetes.py +26 -0
- metaflow/plugins/kubernetes/kubernetes_decorator.py +1 -0
- metaflow/plugins/pypi/conda_decorator.py +5 -5
- metaflow/plugins/pypi/conda_environment.py +1 -1
- metaflow/plugins/pypi/pypi_decorator.py +13 -4
- metaflow/runner/deployer.py +46 -3
- metaflow/runner/metaflow_runner.py +7 -1
- metaflow/runner/nbdeploy.py +3 -0
- metaflow/runner/nbrun.py +5 -0
- metaflow/version.py +1 -1
- {metaflow-2.12.13.dist-info → metaflow-2.12.15.dist-info}/METADATA +2 -2
- {metaflow-2.12.13.dist-info → metaflow-2.12.15.dist-info}/RECORD +21 -21
- {metaflow-2.12.13.dist-info → metaflow-2.12.15.dist-info}/LICENSE +0 -0
- {metaflow-2.12.13.dist-info → metaflow-2.12.15.dist-info}/WHEEL +0 -0
- {metaflow-2.12.13.dist-info → metaflow-2.12.15.dist-info}/entry_points.txt +0 -0
- {metaflow-2.12.13.dist-info → metaflow-2.12.15.dist-info}/top_level.txt +0 -0
@@ -1658,6 +1658,9 @@ class ArgoWorkflows(object):
|
|
1658
1658
|
|
1659
1659
|
# support Metaflow sandboxes
|
1660
1660
|
env["METAFLOW_INIT_SCRIPT"] = KUBERNETES_SANDBOX_INIT_SCRIPT
|
1661
|
+
env["METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT"] = (
|
1662
|
+
KUBERNETES_SANDBOX_INIT_SCRIPT
|
1663
|
+
)
|
1661
1664
|
|
1662
1665
|
# support for @secret
|
1663
1666
|
env["METAFLOW_DEFAULT_SECRETS_BACKEND_TYPE"] = DEFAULT_SECRETS_BACKEND_TYPE
|
@@ -1673,6 +1676,12 @@ class ArgoWorkflows(object):
|
|
1673
1676
|
)
|
1674
1677
|
env["METAFLOW_DATASTORE_SYSROOT_AZURE"] = DATASTORE_SYSROOT_AZURE
|
1675
1678
|
env["METAFLOW_CARD_AZUREROOT"] = CARD_AZUREROOT
|
1679
|
+
env["METAFLOW_ARGO_WORKFLOWS_KUBERNETES_SECRETS"] = (
|
1680
|
+
ARGO_WORKFLOWS_KUBERNETES_SECRETS
|
1681
|
+
)
|
1682
|
+
env["METAFLOW_ARGO_WORKFLOWS_ENV_VARS_TO_SKIP"] = (
|
1683
|
+
ARGO_WORKFLOWS_ENV_VARS_TO_SKIP
|
1684
|
+
)
|
1676
1685
|
|
1677
1686
|
# support for GCP
|
1678
1687
|
env["METAFLOW_DATASTORE_SYSROOT_GS"] = DATASTORE_SYSROOT_GS
|
@@ -36,6 +36,12 @@ from .argo_workflows import ArgoWorkflows
|
|
36
36
|
|
37
37
|
VALID_NAME = re.compile(r"^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$")
|
38
38
|
|
39
|
+
unsupported_decorators = {
|
40
|
+
"snowpark": "Step *%s* is marked for execution on Snowpark with Argo Workflows which isn't currently supported.",
|
41
|
+
"slurm": "Step *%s* is marked for execution on Slurm with Argo Workflows which isn't currently supported.",
|
42
|
+
"nvidia": "Step *%s* is marked for execution on Nvidia with Argo Workflows which isn't currently supported.",
|
43
|
+
}
|
44
|
+
|
39
45
|
|
40
46
|
class IncorrectProductionToken(MetaflowException):
|
41
47
|
headline = "Incorrect production token"
|
@@ -211,6 +217,11 @@ def create(
|
|
211
217
|
deployer_attribute_file=None,
|
212
218
|
enable_error_msg_capture=False,
|
213
219
|
):
|
220
|
+
for node in obj.graph:
|
221
|
+
for decorator, error_message in unsupported_decorators.items():
|
222
|
+
if any([d.name == decorator for d in node.decorators]):
|
223
|
+
raise MetaflowException(error_message % node.name)
|
224
|
+
|
214
225
|
validate_tags(tags)
|
215
226
|
|
216
227
|
if deployer_attribute_file:
|
@@ -226,7 +226,9 @@ def trigger(instance: DeployedFlow, **kwargs):
|
|
226
226
|
)
|
227
227
|
|
228
228
|
command_obj = instance.deployer.spm.get(pid)
|
229
|
-
content = handle_timeout(
|
229
|
+
content = handle_timeout(
|
230
|
+
tfp_runner_attribute, command_obj, instance.deployer.file_read_timeout
|
231
|
+
)
|
230
232
|
|
231
233
|
if command_obj.process.returncode == 0:
|
232
234
|
triggered_run = TriggeredRun(deployer=instance.deployer, content=content)
|
@@ -193,7 +193,9 @@ def trigger(instance: DeployedFlow, **kwargs):
|
|
193
193
|
)
|
194
194
|
|
195
195
|
command_obj = instance.deployer.spm.get(pid)
|
196
|
-
content = handle_timeout(
|
196
|
+
content = handle_timeout(
|
197
|
+
tfp_runner_attribute, command_obj, instance.deployer.file_read_timeout
|
198
|
+
)
|
197
199
|
|
198
200
|
if command_obj.process.returncode == 0:
|
199
201
|
triggered_run = TriggeredRun(deployer=instance.deployer, content=content)
|
@@ -17,6 +17,8 @@ from metaflow.metaflow_config import (
|
|
17
17
|
ARGO_EVENTS_INTERNAL_WEBHOOK_URL,
|
18
18
|
ARGO_EVENTS_SERVICE_ACCOUNT,
|
19
19
|
ARGO_EVENTS_WEBHOOK_AUTH,
|
20
|
+
ARGO_WORKFLOWS_KUBERNETES_SECRETS,
|
21
|
+
ARGO_WORKFLOWS_ENV_VARS_TO_SKIP,
|
20
22
|
AWS_SECRETS_MANAGER_DEFAULT_REGION,
|
21
23
|
AZURE_KEY_VAULT_PREFIX,
|
22
24
|
AZURE_STORAGE_BLOB_SERVICE_ENDPOINT,
|
@@ -280,6 +282,18 @@ class Kubernetes(object):
|
|
280
282
|
.environment_variable(
|
281
283
|
"METAFLOW_INIT_SCRIPT", KUBERNETES_SANDBOX_INIT_SCRIPT
|
282
284
|
)
|
285
|
+
.environment_variable(
|
286
|
+
"METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT",
|
287
|
+
KUBERNETES_SANDBOX_INIT_SCRIPT,
|
288
|
+
)
|
289
|
+
.environment_variable(
|
290
|
+
"METAFLOW_ARGO_WORKFLOWS_KUBERNETES_SECRETS",
|
291
|
+
ARGO_WORKFLOWS_KUBERNETES_SECRETS,
|
292
|
+
)
|
293
|
+
.environment_variable(
|
294
|
+
"METAFLOW_ARGO_WORKFLOWS_ENV_VARS_TO_SKIP",
|
295
|
+
ARGO_WORKFLOWS_ENV_VARS_TO_SKIP,
|
296
|
+
)
|
283
297
|
.environment_variable("METAFLOW_OTEL_ENDPOINT", OTEL_ENDPOINT)
|
284
298
|
# Skip setting METAFLOW_DATASTORE_SYSROOT_LOCAL because metadata sync
|
285
299
|
# between the local user instance and the remote Kubernetes pod
|
@@ -565,6 +579,18 @@ class Kubernetes(object):
|
|
565
579
|
.environment_variable(
|
566
580
|
"METAFLOW_INIT_SCRIPT", KUBERNETES_SANDBOX_INIT_SCRIPT
|
567
581
|
)
|
582
|
+
.environment_variable(
|
583
|
+
"METAFLOW_KUBERNETES_SANDBOX_INIT_SCRIPT",
|
584
|
+
KUBERNETES_SANDBOX_INIT_SCRIPT,
|
585
|
+
)
|
586
|
+
.environment_variable(
|
587
|
+
"METAFLOW_ARGO_WORKFLOWS_KUBERNETES_SECRETS",
|
588
|
+
ARGO_WORKFLOWS_KUBERNETES_SECRETS,
|
589
|
+
)
|
590
|
+
.environment_variable(
|
591
|
+
"METAFLOW_ARGO_WORKFLOWS_ENV_VARS_TO_SKIP",
|
592
|
+
ARGO_WORKFLOWS_ENV_VARS_TO_SKIP,
|
593
|
+
)
|
568
594
|
.environment_variable("METAFLOW_OTEL_ENDPOINT", OTEL_ENDPOINT)
|
569
595
|
# Skip setting METAFLOW_DATASTORE_SYSROOT_LOCAL because metadata sync
|
570
596
|
# between the local user instance and the remote Kubernetes pod
|
@@ -100,9 +100,9 @@ class CondaStepDecorator(StepDecorator):
|
|
100
100
|
# --environment=pypi to --environment=conda
|
101
101
|
_supported_virtual_envs.extend(["pypi"])
|
102
102
|
|
103
|
-
# TODO: Hardcoded for now to support
|
103
|
+
# TODO: Hardcoded for now to support the fast bakery environment.
|
104
104
|
# We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
|
105
|
-
_supported_virtual_envs.extend(["
|
105
|
+
_supported_virtual_envs.extend(["fast-bakery"])
|
106
106
|
|
107
107
|
# The --environment= requirement ensures that valid virtual environments are
|
108
108
|
# created for every step to execute it, greatly simplifying the @conda
|
@@ -210,7 +210,7 @@ class CondaStepDecorator(StepDecorator):
|
|
210
210
|
self.interpreter = (
|
211
211
|
self.environment.interpreter(self.step)
|
212
212
|
if not any(
|
213
|
-
decorator.name in ["batch", "kubernetes"]
|
213
|
+
decorator.name in ["batch", "kubernetes", "nvidia", "snowpark", "slurm"]
|
214
214
|
for decorator in next(
|
215
215
|
step for step in self.flow if step.name == self.step
|
216
216
|
).decorators
|
@@ -344,9 +344,9 @@ class CondaFlowDecorator(FlowDecorator):
|
|
344
344
|
# --environment=pypi to --environment=conda
|
345
345
|
_supported_virtual_envs.extend(["pypi"])
|
346
346
|
|
347
|
-
# TODO: Hardcoded for now to support
|
347
|
+
# TODO: Hardcoded for now to support the fast bakery environment.
|
348
348
|
# We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
|
349
|
-
_supported_virtual_envs.extend(["
|
349
|
+
_supported_virtual_envs.extend(["fast-bakery"])
|
350
350
|
|
351
351
|
# The --environment= requirement ensures that valid virtual environments are
|
352
352
|
# created for every step to execute it, greatly simplifying the @conda
|
@@ -268,7 +268,7 @@ class CondaEnvironment(MetaflowEnvironment):
|
|
268
268
|
for decorator in step.decorators:
|
269
269
|
# TODO: rather than relying on decorator names, rely on attributes
|
270
270
|
# to make them extensible.
|
271
|
-
if decorator.name in ["batch", "kubernetes", "nvidia"]:
|
271
|
+
if decorator.name in ["batch", "kubernetes", "nvidia", "snowpark", "slurm"]:
|
272
272
|
# TODO: Support arm architectures
|
273
273
|
target_platform = "linux-64"
|
274
274
|
break
|
@@ -24,6 +24,12 @@ class PyPIStepDecorator(StepDecorator):
|
|
24
24
|
name = "pypi"
|
25
25
|
defaults = {"packages": {}, "python": None, "disabled": None} # wheels
|
26
26
|
|
27
|
+
def __init__(self, attributes=None, statically_defined=False):
|
28
|
+
self._user_defined_attributes = (
|
29
|
+
attributes.copy() if attributes is not None else {}
|
30
|
+
)
|
31
|
+
super().__init__(attributes, statically_defined)
|
32
|
+
|
27
33
|
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
28
34
|
# The init_environment hook for Environment creates the relevant virtual
|
29
35
|
# environments. The step_init hook sets up the relevant state for that hook to
|
@@ -70,9 +76,9 @@ class PyPIStepDecorator(StepDecorator):
|
|
70
76
|
# --environment=pypi to --environment=conda
|
71
77
|
_supported_virtual_envs.extend(["pypi"])
|
72
78
|
|
73
|
-
# TODO: Hardcoded for now to support
|
79
|
+
# TODO: Hardcoded for now to support the fast bakery environment.
|
74
80
|
# We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
|
75
|
-
_supported_virtual_envs.extend(["
|
81
|
+
_supported_virtual_envs.extend(["fast-bakery"])
|
76
82
|
|
77
83
|
# The --environment= requirement ensures that valid virtual environments are
|
78
84
|
# created for every step to execute it, greatly simplifying the @pypi
|
@@ -88,6 +94,9 @@ class PyPIStepDecorator(StepDecorator):
|
|
88
94
|
)
|
89
95
|
)
|
90
96
|
|
97
|
+
def is_attribute_user_defined(self, name):
|
98
|
+
return name in self._user_defined_attributes
|
99
|
+
|
91
100
|
|
92
101
|
class PyPIFlowDecorator(FlowDecorator):
|
93
102
|
"""
|
@@ -123,9 +132,9 @@ class PyPIFlowDecorator(FlowDecorator):
|
|
123
132
|
# --environment=pypi to --environment=conda
|
124
133
|
_supported_virtual_envs.extend(["pypi"])
|
125
134
|
|
126
|
-
# TODO: Hardcoded for now to support
|
135
|
+
# TODO: Hardcoded for now to support the fast bakery environment.
|
127
136
|
# We should introduce a more robust mechanism for appending supported environments, for example from within extensions.
|
128
|
-
_supported_virtual_envs.extend(["
|
137
|
+
_supported_virtual_envs.extend(["fast-bakery"])
|
129
138
|
|
130
139
|
# The --environment= requirement ensures that valid virtual environments are
|
131
140
|
# created for every step to execute it, greatly simplifying the @conda
|
metaflow/runner/deployer.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
import os
|
2
2
|
import sys
|
3
3
|
import json
|
4
|
+
import time
|
4
5
|
import importlib
|
5
6
|
import functools
|
6
7
|
import tempfile
|
@@ -11,7 +12,9 @@ from metaflow.runner.subprocess_manager import CommandManager, SubprocessManager
|
|
11
12
|
from metaflow.runner.utils import read_from_file_when_ready
|
12
13
|
|
13
14
|
|
14
|
-
def handle_timeout(
|
15
|
+
def handle_timeout(
|
16
|
+
tfp_runner_attribute, command_obj: CommandManager, file_read_timeout: int
|
17
|
+
):
|
15
18
|
"""
|
16
19
|
Handle the timeout for a running subprocess command that reads a file
|
17
20
|
and raises an error with appropriate logs if a TimeoutError occurs.
|
@@ -35,7 +38,9 @@ def handle_timeout(tfp_runner_attribute, command_obj: CommandManager):
|
|
35
38
|
stdout and stderr logs.
|
36
39
|
"""
|
37
40
|
try:
|
38
|
-
content = read_from_file_when_ready(
|
41
|
+
content = read_from_file_when_ready(
|
42
|
+
tfp_runner_attribute.name, timeout=file_read_timeout
|
43
|
+
)
|
39
44
|
return content
|
40
45
|
except TimeoutError as e:
|
41
46
|
stdout_log = open(command_obj.log_files["stdout"]).read()
|
@@ -102,6 +107,8 @@ class Deployer(object):
|
|
102
107
|
cwd : Optional[str], default None
|
103
108
|
The directory to run the subprocess in; if not specified, the current
|
104
109
|
directory is used.
|
110
|
+
file_read_timeout : int, default 3600
|
111
|
+
The timeout until which we try to read the deployer attribute file.
|
105
112
|
**kwargs : Any
|
106
113
|
Additional arguments that you would pass to `python myflow.py` before
|
107
114
|
the deployment command.
|
@@ -114,6 +121,7 @@ class Deployer(object):
|
|
114
121
|
profile: Optional[str] = None,
|
115
122
|
env: Optional[Dict] = None,
|
116
123
|
cwd: Optional[str] = None,
|
124
|
+
file_read_timeout: int = 3600,
|
117
125
|
**kwargs
|
118
126
|
):
|
119
127
|
self.flow_file = flow_file
|
@@ -121,6 +129,7 @@ class Deployer(object):
|
|
121
129
|
self.profile = profile
|
122
130
|
self.env = env
|
123
131
|
self.cwd = cwd
|
132
|
+
self.file_read_timeout = file_read_timeout
|
124
133
|
self.top_level_kwargs = kwargs
|
125
134
|
|
126
135
|
from metaflow.plugins import DEPLOYER_IMPL_PROVIDERS
|
@@ -155,6 +164,7 @@ class Deployer(object):
|
|
155
164
|
profile=self.profile,
|
156
165
|
env=self.env,
|
157
166
|
cwd=self.cwd,
|
167
|
+
file_read_timeout=self.file_read_timeout,
|
158
168
|
**self.top_level_kwargs
|
159
169
|
)
|
160
170
|
|
@@ -197,6 +207,33 @@ class TriggeredRun(object):
|
|
197
207
|
else:
|
198
208
|
setattr(self.__class__, k, property(fget=lambda _, v=v: v))
|
199
209
|
|
210
|
+
def wait_for_run(self, timeout=None):
|
211
|
+
"""
|
212
|
+
Wait for the `run` property to become available.
|
213
|
+
|
214
|
+
Parameters
|
215
|
+
----------
|
216
|
+
timeout : int, optional
|
217
|
+
Maximum time to wait for the `run` to become available, in seconds. If None, wait indefinitely.
|
218
|
+
|
219
|
+
Raises
|
220
|
+
------
|
221
|
+
TimeoutError
|
222
|
+
If the `run` is not available within the specified timeout.
|
223
|
+
"""
|
224
|
+
start_time = time.time()
|
225
|
+
check_interval = 5
|
226
|
+
while True:
|
227
|
+
if self.run is not None:
|
228
|
+
return self.run
|
229
|
+
|
230
|
+
if timeout is not None and (time.time() - start_time) > timeout:
|
231
|
+
raise TimeoutError(
|
232
|
+
"Timed out waiting for the run object to become available."
|
233
|
+
)
|
234
|
+
|
235
|
+
time.sleep(check_interval)
|
236
|
+
|
200
237
|
@property
|
201
238
|
def run(self):
|
202
239
|
"""
|
@@ -268,6 +305,8 @@ class DeployerImpl(object):
|
|
268
305
|
cwd : Optional[str], default None
|
269
306
|
The directory to run the subprocess in; if not specified, the current
|
270
307
|
directory is used.
|
308
|
+
file_read_timeout : int, default 3600
|
309
|
+
The timeout until which we try to read the deployer attribute file.
|
271
310
|
**kwargs : Any
|
272
311
|
Additional arguments that you would pass to `python myflow.py` before
|
273
312
|
the deployment command.
|
@@ -282,6 +321,7 @@ class DeployerImpl(object):
|
|
282
321
|
profile: Optional[str] = None,
|
283
322
|
env: Optional[Dict] = None,
|
284
323
|
cwd: Optional[str] = None,
|
324
|
+
file_read_timeout: int = 3600,
|
285
325
|
**kwargs
|
286
326
|
):
|
287
327
|
if self.TYPE is None:
|
@@ -299,6 +339,7 @@ class DeployerImpl(object):
|
|
299
339
|
self.profile = profile
|
300
340
|
self.env = env
|
301
341
|
self.cwd = cwd
|
342
|
+
self.file_read_timeout = file_read_timeout
|
302
343
|
|
303
344
|
self.env_vars = os.environ.copy()
|
304
345
|
self.env_vars.update(self.env or {})
|
@@ -349,7 +390,9 @@ class DeployerImpl(object):
|
|
349
390
|
)
|
350
391
|
|
351
392
|
command_obj = self.spm.get(pid)
|
352
|
-
content = handle_timeout(
|
393
|
+
content = handle_timeout(
|
394
|
+
tfp_runner_attribute, command_obj, self.file_read_timeout
|
395
|
+
)
|
353
396
|
content = json.loads(content)
|
354
397
|
self.name = content.get("name")
|
355
398
|
self.flow_name = content.get("flow_name")
|
@@ -211,6 +211,8 @@ class Runner(object):
|
|
211
211
|
cwd : Optional[str], default None
|
212
212
|
The directory to run the subprocess in; if not specified, the current
|
213
213
|
directory is used.
|
214
|
+
file_read_timeout : int, default 3600
|
215
|
+
The timeout until which we try to read the runner attribute file.
|
214
216
|
**kwargs : Any
|
215
217
|
Additional arguments that you would pass to `python myflow.py` before
|
216
218
|
the `run` command.
|
@@ -223,6 +225,7 @@ class Runner(object):
|
|
223
225
|
profile: Optional[str] = None,
|
224
226
|
env: Optional[Dict] = None,
|
225
227
|
cwd: Optional[str] = None,
|
228
|
+
file_read_timeout: int = 3600,
|
226
229
|
**kwargs
|
227
230
|
):
|
228
231
|
# these imports are required here and not at the top
|
@@ -248,6 +251,7 @@ class Runner(object):
|
|
248
251
|
self.env_vars["METAFLOW_PROFILE"] = profile
|
249
252
|
|
250
253
|
self.cwd = cwd
|
254
|
+
self.file_read_timeout = file_read_timeout
|
251
255
|
self.spm = SubprocessManager()
|
252
256
|
self.top_level_kwargs = kwargs
|
253
257
|
self.api = MetaflowAPI.from_cli(self.flow_file, start)
|
@@ -270,7 +274,9 @@ class Runner(object):
|
|
270
274
|
clear_and_set_os_environ(self.old_env)
|
271
275
|
|
272
276
|
# Set the correct metadata from the runner_attribute file corresponding to this run.
|
273
|
-
content = read_from_file_when_ready(
|
277
|
+
content = read_from_file_when_ready(
|
278
|
+
tfp_runner_attribute.name, timeout=self.file_read_timeout
|
279
|
+
)
|
274
280
|
metadata_for_flow, pathspec = content.rsplit(":", maxsplit=1)
|
275
281
|
metadata(metadata_for_flow)
|
276
282
|
run_object = Run(pathspec, _namespace_check=False)
|
metaflow/runner/nbdeploy.py
CHANGED
@@ -61,6 +61,7 @@ class NBDeployer(object):
|
|
61
61
|
profile: Optional[str] = None,
|
62
62
|
env: Optional[Dict] = None,
|
63
63
|
base_dir: str = DEFAULT_DIR,
|
64
|
+
file_read_timeout: int = 3600,
|
64
65
|
**kwargs,
|
65
66
|
):
|
66
67
|
try:
|
@@ -78,6 +79,7 @@ class NBDeployer(object):
|
|
78
79
|
self.profile = profile
|
79
80
|
self.env = env
|
80
81
|
self.cwd = base_dir
|
82
|
+
self.file_read_timeout = file_read_timeout
|
81
83
|
self.top_level_kwargs = kwargs
|
82
84
|
|
83
85
|
self.env_vars = os.environ.copy()
|
@@ -112,6 +114,7 @@ class NBDeployer(object):
|
|
112
114
|
profile=self.profile,
|
113
115
|
env=self.env_vars,
|
114
116
|
cwd=self.cwd,
|
117
|
+
file_read_timeout=self.file_read_timeout,
|
115
118
|
**kwargs,
|
116
119
|
)
|
117
120
|
|
metaflow/runner/nbrun.py
CHANGED
@@ -45,6 +45,8 @@ class NBRunner(object):
|
|
45
45
|
base_dir : Optional[str], default None
|
46
46
|
The directory to run the subprocess in; if not specified, a temporary
|
47
47
|
directory is used.
|
48
|
+
file_read_timeout : int, default 3600
|
49
|
+
The timeout until which we try to read the runner attribute file.
|
48
50
|
**kwargs : Any
|
49
51
|
Additional arguments that you would pass to `python myflow.py` before
|
50
52
|
the `run` command.
|
@@ -58,6 +60,7 @@ class NBRunner(object):
|
|
58
60
|
profile: Optional[str] = None,
|
59
61
|
env: Optional[Dict] = None,
|
60
62
|
base_dir: str = DEFAULT_DIR,
|
63
|
+
file_read_timeout: int = 3600,
|
61
64
|
**kwargs,
|
62
65
|
):
|
63
66
|
try:
|
@@ -82,6 +85,7 @@ class NBRunner(object):
|
|
82
85
|
self.env_vars["METAFLOW_PROFILE"] = profile
|
83
86
|
|
84
87
|
self.base_dir = base_dir
|
88
|
+
self.file_read_timeout = file_read_timeout
|
85
89
|
|
86
90
|
if not self.cell:
|
87
91
|
raise ValueError("Couldn't find a cell.")
|
@@ -104,6 +108,7 @@ class NBRunner(object):
|
|
104
108
|
profile=profile,
|
105
109
|
env=self.env_vars,
|
106
110
|
cwd=self.base_dir,
|
111
|
+
file_read_timeout=self.file_read_timeout,
|
107
112
|
**kwargs,
|
108
113
|
)
|
109
114
|
|
metaflow/version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
metaflow_version = "2.12.
|
1
|
+
metaflow_version = "2.12.15"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: metaflow
|
3
|
-
Version: 2.12.
|
3
|
+
Version: 2.12.15
|
4
4
|
Summary: Metaflow: More Data Science, Less Engineering
|
5
5
|
Author: Metaflow Developers
|
6
6
|
Author-email: help@metaflow.org
|
@@ -26,7 +26,7 @@ License-File: LICENSE
|
|
26
26
|
Requires-Dist: requests
|
27
27
|
Requires-Dist: boto3
|
28
28
|
Provides-Extra: stubs
|
29
|
-
Requires-Dist: metaflow-stubs==2.12.
|
29
|
+
Requires-Dist: metaflow-stubs==2.12.15; extra == "stubs"
|
30
30
|
|
31
31
|

|
32
32
|
|
@@ -35,7 +35,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
|
|
35
35
|
metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
|
36
36
|
metaflow/util.py,sha256=olAvJK3y1it_k99MhLulTaAJo7OFVt5rnrD-ulIFLCU,13616
|
37
37
|
metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
|
38
|
-
metaflow/version.py,sha256=
|
38
|
+
metaflow/version.py,sha256=1NB7Z6CQ_ikm8j8fCxvlewScECCPh_oBSWiTiz8W2sg,29
|
39
39
|
metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
|
40
40
|
metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
|
41
41
|
metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
|
@@ -174,10 +174,10 @@ metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqN
|
|
174
174
|
metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
175
175
|
metaflow/plugins/argo/argo_client.py,sha256=MKKhMCbWOPzf6z5zQQiyDRHHkAXcO7ipboDZDqAAvOk,15849
|
176
176
|
metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
|
177
|
-
metaflow/plugins/argo/argo_workflows.py,sha256=
|
178
|
-
metaflow/plugins/argo/argo_workflows_cli.py,sha256=
|
177
|
+
metaflow/plugins/argo/argo_workflows.py,sha256=6xUkz1LKdCLbl-O-D83Y2G5mCKYcIciKts3x1PNAzCk,170173
|
178
|
+
metaflow/plugins/argo/argo_workflows_cli.py,sha256=X2j_F0xF8-K30ebM4dSLOTteDKXbr-jMN18oMpl5S6Y,36313
|
179
179
|
metaflow/plugins/argo/argo_workflows_decorator.py,sha256=yprszMdbE3rBTcEA9VR0IEnPjTprUauZBc4SBb-Q7sA,7878
|
180
|
-
metaflow/plugins/argo/argo_workflows_deployer.py,sha256=
|
180
|
+
metaflow/plugins/argo/argo_workflows_deployer.py,sha256=wSSZtThn_VPvE_Wu6NB1L0Q86LmBJh9g009v_lpvBPM,8125
|
181
181
|
metaflow/plugins/argo/capture_error.py,sha256=Ys9dscGrTpW-ZCirLBU0gD9qBM0BjxyxGlUMKcwewQc,1852
|
182
182
|
metaflow/plugins/argo/daemon.py,sha256=dJOS_UUISXBYffi3oGVKPwq4Pa4P_nGBGL15piPaPto,1776
|
183
183
|
metaflow/plugins/argo/generate_input_paths.py,sha256=loYsI6RFX9LlFsHb7Fe-mzlTTtRdySoOu7sYDy-uXK0,881
|
@@ -189,7 +189,7 @@ metaflow/plugins/aws/batch/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
|
|
189
189
|
metaflow/plugins/aws/batch/batch.py,sha256=e9ssahWM18GnipPK2sqYB-ztx9w7Eoo7YtWyEtufYxs,17787
|
190
190
|
metaflow/plugins/aws/batch/batch_cli.py,sha256=6PTbyajRgdy0XmjyJLBTdKdiOB84dcovQQ8sFXlJqko,11749
|
191
191
|
metaflow/plugins/aws/batch/batch_client.py,sha256=s9ZHhxQPPoBQijLUgn6_16QOaD4-22U_44uJbp-yLkI,28565
|
192
|
-
metaflow/plugins/aws/batch/batch_decorator.py,sha256=
|
192
|
+
metaflow/plugins/aws/batch/batch_decorator.py,sha256=kwgxEPCEoI6eZIpU5PuL442Ohg4_BfvwowoYgAnCzKE,17520
|
193
193
|
metaflow/plugins/aws/secrets_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
194
194
|
metaflow/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.py,sha256=JtFUVu00Cg0FzAizgrPLXmrMqsT7YeQMkQlgeivUxcE,7986
|
195
195
|
metaflow/plugins/aws/step_functions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -202,7 +202,7 @@ metaflow/plugins/aws/step_functions/step_functions.py,sha256=8tBs4pvdIVgNoZRm0Dz
|
|
202
202
|
metaflow/plugins/aws/step_functions/step_functions_cli.py,sha256=4CYw5xQwPaEV_iudF4_m4SSRS5LEq5UgrzVlCB6xT_U,25763
|
203
203
|
metaflow/plugins/aws/step_functions/step_functions_client.py,sha256=DKpNwAIWElvWjFANs5Ku3rgzjxFoqAD6k-EF8Xhkg3Q,4754
|
204
204
|
metaflow/plugins/aws/step_functions/step_functions_decorator.py,sha256=9hw_MX36RyFp6IowuAYaJzJg9UC5KCe1FNt1PcG7_J0,3791
|
205
|
-
metaflow/plugins/aws/step_functions/step_functions_deployer.py,sha256=
|
205
|
+
metaflow/plugins/aws/step_functions/step_functions_deployer.py,sha256=WrfQjvXnnInXwSePwoLUMb2EjqFG4RK1krO_8bW0qGI,7218
|
206
206
|
metaflow/plugins/azure/__init__.py,sha256=GuuhTVC-zSdyAf79a1wiERMq0Zts7fwVT7t9fAf234A,100
|
207
207
|
metaflow/plugins/azure/azure_credential.py,sha256=JmdGEbVzgxy8ucqnQDdTTI_atyMX9WSZUw3qYOo7RhE,2174
|
208
208
|
metaflow/plugins/azure/azure_exceptions.py,sha256=NnbwpUC23bc61HZjJmeXztY0tBNn_Y_VpIpDDuYWIZ0,433
|
@@ -279,10 +279,10 @@ metaflow/plugins/gcp/gs_tail.py,sha256=Jl_wvnzU7dub07A-DOAuP5FeccNIrPM-CeL1xKFs1
|
|
279
279
|
metaflow/plugins/gcp/gs_utils.py,sha256=ZmIGFse1qYyvAVrwga23PQUzF6dXEDLLsZ2F-YRmvow,2030
|
280
280
|
metaflow/plugins/gcp/includefile_support.py,sha256=vIDeR-MiJuUh-2S2pV7Z7FBkhIWwtHXaRrj76MWGRiY,3869
|
281
281
|
metaflow/plugins/kubernetes/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
282
|
-
metaflow/plugins/kubernetes/kubernetes.py,sha256=
|
282
|
+
metaflow/plugins/kubernetes/kubernetes.py,sha256=cr3TheUasxIBEwFZ3GEVbctaf8gW57BM5BDk80ikjPI,31063
|
283
283
|
metaflow/plugins/kubernetes/kubernetes_cli.py,sha256=qBDdr1Lvtt-RO9pB-9_HTOPdzAmDvvJ0aiQ1OoCcrMU,10892
|
284
284
|
metaflow/plugins/kubernetes/kubernetes_client.py,sha256=GKg-gT3qhXMRQV-sG1YyoOf3Z32NXr_wwEN2ytMVSEg,2471
|
285
|
-
metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=
|
285
|
+
metaflow/plugins/kubernetes/kubernetes_decorator.py,sha256=CXStYHomuJJK_Yocpdo6OJadEQv5hDfSpO7GPL61ltw,25322
|
286
286
|
metaflow/plugins/kubernetes/kubernetes_job.py,sha256=Cfkee8LbXC17jSXWoeNdomQRvF_8YSeXNg1gvxm6E_M,31806
|
287
287
|
metaflow/plugins/kubernetes/kubernetes_jobsets.py,sha256=OBmLtX-ZUDQdCCfftUmRMernfmTNMwdTxPoCAp_NmwE,40957
|
288
288
|
metaflow/plugins/metadata/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
@@ -290,11 +290,11 @@ metaflow/plugins/metadata/local.py,sha256=YhLJC5zjVJrvQFIyQ92ZBByiUmhCC762RUX7IT
|
|
290
290
|
metaflow/plugins/metadata/service.py,sha256=ihq5F7KQZlxvYwzH_-jyP2aWN_I96i2vp92j_d697s8,20204
|
291
291
|
metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
|
292
292
|
metaflow/plugins/pypi/bootstrap.py,sha256=Tvc4_QKIx-A8j5Aq8ccWZrrxNM8csN40rK8HmxDx-Z8,5106
|
293
|
-
metaflow/plugins/pypi/conda_decorator.py,sha256=
|
294
|
-
metaflow/plugins/pypi/conda_environment.py,sha256=
|
293
|
+
metaflow/plugins/pypi/conda_decorator.py,sha256=fTJVbEfgOUtsDXIfnfsNk46sKeA9uTuTqGey9OFs9Ig,14738
|
294
|
+
metaflow/plugins/pypi/conda_environment.py,sha256=erVvMgQhg6oBv4l6Yca72FKKoB0QEcitpTserPSPc44,19328
|
295
295
|
metaflow/plugins/pypi/micromamba.py,sha256=67FiIZZz0Kig9EcN7bZLObsE6Z1MFyo4Dp93fd3Grcc,12178
|
296
296
|
metaflow/plugins/pypi/pip.py,sha256=7B06mPOs5MvY33xbzPVYZlBr1iKMYaN-n8uulL9zSVg,13649
|
297
|
-
metaflow/plugins/pypi/pypi_decorator.py,sha256=
|
297
|
+
metaflow/plugins/pypi/pypi_decorator.py,sha256=h5cAnxkWjmj4Ad4q0AkABKwhHQHYfeexy12yMaaLgXQ,6443
|
298
298
|
metaflow/plugins/pypi/pypi_environment.py,sha256=FYMg8kF3lXqcLfRYWD83a9zpVjcoo_TARqMGZ763rRk,230
|
299
299
|
metaflow/plugins/pypi/utils.py,sha256=ds1Mnv_DaxGnLAYp7ozg_K6oyguGyNhvHfE-75Ia1YA,2836
|
300
300
|
metaflow/plugins/secrets/__init__.py,sha256=mhJaN2eMS_ZZVewAMR2E-JdP5i0t3v9e6Dcwd-WpruE,310
|
@@ -302,10 +302,10 @@ metaflow/plugins/secrets/inline_secrets_provider.py,sha256=EChmoBGA1i7qM3jtYwPpL
|
|
302
302
|
metaflow/plugins/secrets/secrets_decorator.py,sha256=s-sFzPWOjahhpr5fMj-ZEaHkDYAPTO0isYXGvaUwlG8,11273
|
303
303
|
metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
304
304
|
metaflow/runner/click_api.py,sha256=vrrIb5DtVYhCW_hB7wPgj6Q0o-h1bBSWiYnKTOyC454,13452
|
305
|
-
metaflow/runner/deployer.py,sha256=
|
306
|
-
metaflow/runner/metaflow_runner.py,sha256=
|
307
|
-
metaflow/runner/nbdeploy.py,sha256
|
308
|
-
metaflow/runner/nbrun.py,sha256=
|
305
|
+
metaflow/runner/deployer.py,sha256=nArjnErc0rOaZW612VRKDOT5594jwzeu86w5zW1LX6U,13558
|
306
|
+
metaflow/runner/metaflow_runner.py,sha256=AO9nwr5qUbZWmsbFdjkUJrvFlaylz7WvxslvHsIqDYc,15371
|
307
|
+
metaflow/runner/nbdeploy.py,sha256=fP1s_5MeiDyT_igP82pB5EUqX9rOy2s06Hyc-OUbOvQ,4115
|
308
|
+
metaflow/runner/nbrun.py,sha256=lmvhzMCz7iC9LSPGRijifW1wMXxa4RW_jVmpdjQi22E,7261
|
309
309
|
metaflow/runner/subprocess_manager.py,sha256=0knxWZYJx8srMv6wTPYKOC6tn4-airnyI7Vbqfb3iXY,19567
|
310
310
|
metaflow/runner/utils.py,sha256=FibdEj8CDnx1a-Je5KUQTwHuNbtkFm1unXGarj0D8ok,1394
|
311
311
|
metaflow/sidecar/__init__.py,sha256=1mmNpmQ5puZCpRmmYlCOeieZ4108Su9XQ4_EqF1FGOU,131
|
@@ -344,9 +344,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
|
|
344
344
|
metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
|
345
345
|
metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
|
346
346
|
metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
|
347
|
-
metaflow-2.12.
|
348
|
-
metaflow-2.12.
|
349
|
-
metaflow-2.12.
|
350
|
-
metaflow-2.12.
|
351
|
-
metaflow-2.12.
|
352
|
-
metaflow-2.12.
|
347
|
+
metaflow-2.12.15.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
|
348
|
+
metaflow-2.12.15.dist-info/METADATA,sha256=AMUsPF_gTNbFJkaDmtqOopwXmUXYfFCXymmbX6CUmFc,5906
|
349
|
+
metaflow-2.12.15.dist-info/WHEEL,sha256=GUeE9LxUgRABPG7YM0jCNs9cBsAIx0YAkzCB88PMLgc,109
|
350
|
+
metaflow-2.12.15.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
|
351
|
+
metaflow-2.12.15.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
|
352
|
+
metaflow-2.12.15.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|