metaflow 2.12.37__py2.py3-none-any.whl → 2.12.39__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow/cli.py +1 -1
- metaflow/cli_components/run_cmds.py +1 -1
- metaflow/cli_components/step_cmd.py +1 -1
- metaflow/decorators.py +13 -4
- metaflow/plugins/airflow/airflow_cli.py +1 -1
- metaflow/plugins/argo/argo_workflows_cli.py +1 -1
- metaflow/plugins/argo/argo_workflows_deployer_objects.py +5 -1
- metaflow/plugins/aws/step_functions/step_functions_cli.py +1 -1
- metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py +3 -0
- metaflow/plugins/pypi/conda_decorator.py +14 -26
- metaflow/plugins/pypi/pypi_decorator.py +11 -10
- metaflow/runner/deployer.py +1 -1
- metaflow/runner/deployer_impl.py +2 -2
- metaflow/runner/metaflow_runner.py +4 -1
- metaflow/runner/nbdeploy.py +2 -0
- metaflow/runner/nbrun.py +1 -1
- metaflow/runner/subprocess_manager.py +3 -1
- metaflow/runner/utils.py +37 -20
- metaflow/version.py +1 -1
- {metaflow-2.12.37.dist-info → metaflow-2.12.39.dist-info}/METADATA +2 -2
- {metaflow-2.12.37.dist-info → metaflow-2.12.39.dist-info}/RECORD +25 -25
- {metaflow-2.12.37.dist-info → metaflow-2.12.39.dist-info}/LICENSE +0 -0
- {metaflow-2.12.37.dist-info → metaflow-2.12.39.dist-info}/WHEEL +0 -0
- {metaflow-2.12.37.dist-info → metaflow-2.12.39.dist-info}/entry_points.txt +0 -0
- {metaflow-2.12.37.dist-info → metaflow-2.12.39.dist-info}/top_level.txt +0 -0
metaflow/cli.py
CHANGED
@@ -461,7 +461,7 @@ def start(
|
|
461
461
|
)
|
462
462
|
if all_decospecs:
|
463
463
|
decorators._attach_decorators(ctx.obj.flow, all_decospecs)
|
464
|
-
decorators._init(ctx.obj.flow
|
464
|
+
decorators._init(ctx.obj.flow)
|
465
465
|
# Regenerate graph if we attached more decorators
|
466
466
|
ctx.obj.graph = FlowGraph(ctx.obj.flow.__class__)
|
467
467
|
|
@@ -40,7 +40,7 @@ def before_run(obj, tags, decospecs):
|
|
40
40
|
)
|
41
41
|
if all_decospecs:
|
42
42
|
decorators._attach_decorators(obj.flow, all_decospecs)
|
43
|
-
decorators._init(obj.flow
|
43
|
+
decorators._init(obj.flow)
|
44
44
|
obj.graph = FlowGraph(obj.flow.__class__)
|
45
45
|
|
46
46
|
obj.check(obj.graph, obj.flow, obj.environment, pylint=obj.pylint)
|
@@ -138,7 +138,7 @@ def step(
|
|
138
138
|
|
139
139
|
if decospecs:
|
140
140
|
decorators._attach_decorators_to_step(func, decospecs)
|
141
|
-
decorators._init(ctx.obj.flow
|
141
|
+
decorators._init(ctx.obj.flow)
|
142
142
|
|
143
143
|
step_kwargs = ctx.params
|
144
144
|
# Remove argument `step_name` from `step_kwargs`.
|
metaflow/decorators.py
CHANGED
@@ -27,6 +27,11 @@ except NameError:
|
|
27
27
|
unicode = str
|
28
28
|
basestring = str
|
29
29
|
|
30
|
+
# Contains the decorators on which _init was called. We want to ensure it is called
|
31
|
+
# only once on each decorator and, as the _init() function below can be called in
|
32
|
+
# several places, we need to track which decorator had their init function called
|
33
|
+
_inited_decorators = set()
|
34
|
+
|
30
35
|
|
31
36
|
class BadStepDecoratorException(MetaflowException):
|
32
37
|
headline = "Syntax error"
|
@@ -553,12 +558,16 @@ def _attach_decorators_to_step(step, decospecs):
|
|
553
558
|
def _init(flow, only_non_static=False):
|
554
559
|
for decorators in flow._flow_decorators.values():
|
555
560
|
for deco in decorators:
|
556
|
-
if
|
557
|
-
|
561
|
+
if deco in _inited_decorators:
|
562
|
+
continue
|
563
|
+
deco.init()
|
564
|
+
_inited_decorators.add(deco)
|
558
565
|
for flowstep in flow:
|
559
566
|
for deco in flowstep.decorators:
|
560
|
-
if
|
561
|
-
|
567
|
+
if deco in _inited_decorators:
|
568
|
+
continue
|
569
|
+
deco.init()
|
570
|
+
_inited_decorators.add(deco)
|
562
571
|
|
563
572
|
|
564
573
|
def _init_flow_decorators(
|
@@ -283,7 +283,7 @@ def make_flow(
|
|
283
283
|
):
|
284
284
|
# Attach @kubernetes.
|
285
285
|
decorators._attach_decorators(obj.flow, [KubernetesDecorator.name])
|
286
|
-
decorators._init(obj.flow
|
286
|
+
decorators._init(obj.flow)
|
287
287
|
|
288
288
|
decorators._init_step_decorators(
|
289
289
|
obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger
|
@@ -470,7 +470,7 @@ def make_flow(
|
|
470
470
|
decorators._attach_decorators(
|
471
471
|
obj.flow, [KubernetesDecorator.name, EnvironmentDecorator.name]
|
472
472
|
)
|
473
|
-
decorators._init(obj.flow
|
473
|
+
decorators._init(obj.flow)
|
474
474
|
|
475
475
|
decorators._init_step_decorators(
|
476
476
|
obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger
|
@@ -97,6 +97,7 @@ class ArgoWorkflowsTriggeredRun(TriggeredRun):
|
|
97
97
|
)
|
98
98
|
|
99
99
|
command_obj = self.deployer.spm.get(pid)
|
100
|
+
command_obj.sync_wait()
|
100
101
|
return command_obj.process.returncode == 0
|
101
102
|
|
102
103
|
def unsuspend(self, **kwargs) -> bool:
|
@@ -131,6 +132,7 @@ class ArgoWorkflowsTriggeredRun(TriggeredRun):
|
|
131
132
|
)
|
132
133
|
|
133
134
|
command_obj = self.deployer.spm.get(pid)
|
135
|
+
command_obj.sync_wait()
|
134
136
|
return command_obj.process.returncode == 0
|
135
137
|
|
136
138
|
def terminate(self, **kwargs) -> bool:
|
@@ -165,6 +167,7 @@ class ArgoWorkflowsTriggeredRun(TriggeredRun):
|
|
165
167
|
)
|
166
168
|
|
167
169
|
command_obj = self.deployer.spm.get(pid)
|
170
|
+
command_obj.sync_wait()
|
168
171
|
return command_obj.process.returncode == 0
|
169
172
|
|
170
173
|
@property
|
@@ -319,6 +322,7 @@ class ArgoWorkflowsDeployedFlow(DeployedFlow):
|
|
319
322
|
)
|
320
323
|
|
321
324
|
command_obj = self.deployer.spm.get(pid)
|
325
|
+
command_obj.sync_wait()
|
322
326
|
return command_obj.process.returncode == 0
|
323
327
|
|
324
328
|
def trigger(self, **kwargs) -> ArgoWorkflowsTriggeredRun:
|
@@ -361,7 +365,7 @@ class ArgoWorkflowsDeployedFlow(DeployedFlow):
|
|
361
365
|
content = handle_timeout(
|
362
366
|
attribute_file_fd, command_obj, self.deployer.file_read_timeout
|
363
367
|
)
|
364
|
-
|
368
|
+
command_obj.sync_wait()
|
365
369
|
if command_obj.process.returncode == 0:
|
366
370
|
return ArgoWorkflowsTriggeredRun(
|
367
371
|
deployer=self.deployer, content=content
|
@@ -326,7 +326,7 @@ def make_flow(
|
|
326
326
|
|
327
327
|
# Attach AWS Batch decorator to the flow
|
328
328
|
decorators._attach_decorators(obj.flow, [BatchDecorator.name])
|
329
|
-
decorators._init(obj.flow
|
329
|
+
decorators._init(obj.flow)
|
330
330
|
decorators._init_step_decorators(
|
331
331
|
obj.flow, obj.graph, obj.environment, obj.flow_datastore, obj.logger
|
332
332
|
)
|
@@ -46,6 +46,7 @@ class StepFunctionsTriggeredRun(TriggeredRun):
|
|
46
46
|
)
|
47
47
|
|
48
48
|
command_obj = self.deployer.spm.get(pid)
|
49
|
+
command_obj.sync_wait()
|
49
50
|
return command_obj.process.returncode == 0
|
50
51
|
|
51
52
|
|
@@ -174,6 +175,7 @@ class StepFunctionsDeployedFlow(DeployedFlow):
|
|
174
175
|
)
|
175
176
|
|
176
177
|
command_obj = self.deployer.spm.get(pid)
|
178
|
+
command_obj.sync_wait()
|
177
179
|
return command_obj.process.returncode == 0
|
178
180
|
|
179
181
|
def trigger(self, **kwargs) -> StepFunctionsTriggeredRun:
|
@@ -217,6 +219,7 @@ class StepFunctionsDeployedFlow(DeployedFlow):
|
|
217
219
|
attribute_file_fd, command_obj, self.deployer.file_read_timeout
|
218
220
|
)
|
219
221
|
|
222
|
+
command_obj.sync_wait()
|
220
223
|
if command_obj.process.returncode == 0:
|
221
224
|
return StepFunctionsTriggeredRun(
|
222
225
|
deployer=self.deployer, content=content
|
@@ -50,31 +50,26 @@ class CondaStepDecorator(StepDecorator):
|
|
50
50
|
# conda channels, users can specify channel::package as the package name.
|
51
51
|
|
52
52
|
def __init__(self, attributes=None, statically_defined=False):
|
53
|
-
self.
|
54
|
-
attributes.
|
53
|
+
self._attributes_with_user_values = (
|
54
|
+
set(attributes.keys()) if attributes is not None else set()
|
55
55
|
)
|
56
|
+
|
56
57
|
super(CondaStepDecorator, self).__init__(attributes, statically_defined)
|
57
58
|
|
58
59
|
def init(self):
|
59
60
|
super(CondaStepDecorator, self).init()
|
60
61
|
|
61
|
-
# We have to go back and fixup _user_defined_attributes for potential
|
62
|
-
# config resolution
|
63
|
-
self._user_defined_attributes = {
|
64
|
-
k: v
|
65
|
-
for k, v in self.attributes.items()
|
66
|
-
if k in self._user_defined_attributes
|
67
|
-
}
|
68
|
-
|
69
62
|
# Support legacy 'libraries=' attribute for the decorator.
|
70
63
|
self.attributes["packages"] = {
|
71
64
|
**self.attributes["libraries"],
|
72
65
|
**self.attributes["packages"],
|
73
66
|
}
|
74
67
|
del self.attributes["libraries"]
|
68
|
+
if self.attributes["packages"]:
|
69
|
+
self._attributes_with_user_values.add("packages")
|
75
70
|
|
76
71
|
def is_attribute_user_defined(self, name):
|
77
|
-
return name in self.
|
72
|
+
return name in self._attributes_with_user_values
|
78
73
|
|
79
74
|
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
80
75
|
# The init_environment hook for Environment creates the relevant virtual
|
@@ -94,10 +89,10 @@ class CondaStepDecorator(StepDecorator):
|
|
94
89
|
**super_attributes["packages"],
|
95
90
|
**self.attributes["packages"],
|
96
91
|
}
|
97
|
-
self.
|
98
|
-
|
99
|
-
|
100
|
-
|
92
|
+
self._attributes_with_user_values.update(
|
93
|
+
conda_base._attributes_with_user_values
|
94
|
+
)
|
95
|
+
|
101
96
|
self.attributes["python"] = (
|
102
97
|
self.attributes["python"] or super_attributes["python"]
|
103
98
|
)
|
@@ -344,22 +339,15 @@ class CondaFlowDecorator(FlowDecorator):
|
|
344
339
|
}
|
345
340
|
|
346
341
|
def __init__(self, attributes=None, statically_defined=False):
|
347
|
-
self.
|
348
|
-
attributes.
|
342
|
+
self._attributes_with_user_values = (
|
343
|
+
set(attributes.keys()) if attributes is not None else set()
|
349
344
|
)
|
345
|
+
|
350
346
|
super(CondaFlowDecorator, self).__init__(attributes, statically_defined)
|
351
347
|
|
352
348
|
def init(self):
|
353
349
|
super(CondaFlowDecorator, self).init()
|
354
350
|
|
355
|
-
# We have to go back and fixup _user_defined_attributes for potential
|
356
|
-
# config resolution
|
357
|
-
self._user_defined_attributes = {
|
358
|
-
k: v
|
359
|
-
for k, v in self.attributes.items()
|
360
|
-
if k in self._user_defined_attributes
|
361
|
-
}
|
362
|
-
|
363
351
|
# Support legacy 'libraries=' attribute for the decorator.
|
364
352
|
self.attributes["packages"] = {
|
365
353
|
**self.attributes["libraries"],
|
@@ -370,7 +358,7 @@ class CondaFlowDecorator(FlowDecorator):
|
|
370
358
|
self.attributes["python"] = str(self.attributes["python"])
|
371
359
|
|
372
360
|
def is_attribute_user_defined(self, name):
|
373
|
-
return name in self.
|
361
|
+
return name in self._attributes_with_user_values
|
374
362
|
|
375
363
|
def flow_init(
|
376
364
|
self, flow, graph, environment, flow_datastore, metadata, logger, echo, options
|
@@ -25,9 +25,10 @@ class PyPIStepDecorator(StepDecorator):
|
|
25
25
|
defaults = {"packages": {}, "python": None, "disabled": None} # wheels
|
26
26
|
|
27
27
|
def __init__(self, attributes=None, statically_defined=False):
|
28
|
-
self.
|
29
|
-
attributes.
|
28
|
+
self._attributes_with_user_values = (
|
29
|
+
set(attributes.keys()) if attributes is not None else set()
|
30
30
|
)
|
31
|
+
|
31
32
|
super().__init__(attributes, statically_defined)
|
32
33
|
|
33
34
|
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
@@ -42,10 +43,9 @@ class PyPIStepDecorator(StepDecorator):
|
|
42
43
|
if "pypi_base" in self.flow._flow_decorators:
|
43
44
|
pypi_base = self.flow._flow_decorators["pypi_base"][0]
|
44
45
|
super_attributes = pypi_base.attributes
|
45
|
-
self.
|
46
|
-
|
47
|
-
|
48
|
-
}
|
46
|
+
self._attributes_with_user_values.update(
|
47
|
+
pypi_base._attributes_with_user_values
|
48
|
+
)
|
49
49
|
self.attributes["packages"] = {
|
50
50
|
**super_attributes["packages"],
|
51
51
|
**self.attributes["packages"],
|
@@ -106,7 +106,7 @@ class PyPIStepDecorator(StepDecorator):
|
|
106
106
|
environment.set_local_root(LocalStorage.get_datastore_root_from_config(logger))
|
107
107
|
|
108
108
|
def is_attribute_user_defined(self, name):
|
109
|
-
return name in self.
|
109
|
+
return name in self._attributes_with_user_values
|
110
110
|
|
111
111
|
|
112
112
|
class PyPIFlowDecorator(FlowDecorator):
|
@@ -129,9 +129,10 @@ class PyPIFlowDecorator(FlowDecorator):
|
|
129
129
|
defaults = {"packages": {}, "python": None, "disabled": None}
|
130
130
|
|
131
131
|
def __init__(self, attributes=None, statically_defined=False):
|
132
|
-
self.
|
133
|
-
attributes.
|
132
|
+
self._attributes_with_user_values = (
|
133
|
+
set(attributes.keys()) if attributes is not None else set()
|
134
134
|
)
|
135
|
+
|
135
136
|
super().__init__(attributes, statically_defined)
|
136
137
|
|
137
138
|
def flow_init(
|
@@ -140,7 +141,7 @@ class PyPIFlowDecorator(FlowDecorator):
|
|
140
141
|
from metaflow import decorators
|
141
142
|
|
142
143
|
decorators._attach_decorators(flow, ["pypi"])
|
143
|
-
decorators._init(flow
|
144
|
+
decorators._init(flow)
|
144
145
|
|
145
146
|
# @pypi uses a conda environment to create a virtual environment.
|
146
147
|
# The conda environment can be created through micromamba.
|
metaflow/runner/deployer.py
CHANGED
@@ -64,7 +64,7 @@ class Deployer(metaclass=DeployerMeta):
|
|
64
64
|
The directory to run the subprocess in; if not specified, the current
|
65
65
|
directory is used.
|
66
66
|
file_read_timeout : int, default 3600
|
67
|
-
The timeout until which we try to read the deployer attribute file.
|
67
|
+
The timeout until which we try to read the deployer attribute file (in seconds).
|
68
68
|
**kwargs : Any
|
69
69
|
Additional arguments that you would pass to `python myflow.py` before
|
70
70
|
the deployment command.
|
metaflow/runner/deployer_impl.py
CHANGED
@@ -37,7 +37,7 @@ class DeployerImpl(object):
|
|
37
37
|
The directory to run the subprocess in; if not specified, the current
|
38
38
|
directory is used.
|
39
39
|
file_read_timeout : int, default 3600
|
40
|
-
The timeout until which we try to read the deployer attribute file.
|
40
|
+
The timeout until which we try to read the deployer attribute file (in seconds).
|
41
41
|
**kwargs : Any
|
42
42
|
Additional arguments that you would pass to `python myflow.py` before
|
43
43
|
the deployment command.
|
@@ -144,7 +144,7 @@ class DeployerImpl(object):
|
|
144
144
|
# Additional info is used to pass additional deployer specific information.
|
145
145
|
# It is used in non-OSS deployers (extensions).
|
146
146
|
self.additional_info = content.get("additional_info", {})
|
147
|
-
|
147
|
+
command_obj.sync_wait()
|
148
148
|
if command_obj.process.returncode == 0:
|
149
149
|
return create_class(deployer=self)
|
150
150
|
|
@@ -221,7 +221,7 @@ class Runner(object):
|
|
221
221
|
The directory to run the subprocess in; if not specified, the current
|
222
222
|
directory is used.
|
223
223
|
file_read_timeout : int, default 3600
|
224
|
-
The timeout until which we try to read the runner attribute file.
|
224
|
+
The timeout until which we try to read the runner attribute file (in seconds).
|
225
225
|
**kwargs : Any
|
226
226
|
Additional arguments that you would pass to `python myflow.py` before
|
227
227
|
the `run` command.
|
@@ -272,6 +272,9 @@ class Runner(object):
|
|
272
272
|
|
273
273
|
def __get_executing_run(self, attribute_file_fd, command_obj):
|
274
274
|
content = handle_timeout(attribute_file_fd, command_obj, self.file_read_timeout)
|
275
|
+
|
276
|
+
command_obj.sync_wait()
|
277
|
+
|
275
278
|
content = json.loads(content)
|
276
279
|
pathspec = "%s/%s" % (content.get("flow_name"), content.get("run_id"))
|
277
280
|
|
metaflow/runner/nbdeploy.py
CHANGED
@@ -46,6 +46,8 @@ class NBDeployer(object):
|
|
46
46
|
base_dir : str, optional, default None
|
47
47
|
The directory to run the subprocess in; if not specified, the current
|
48
48
|
working directory is used.
|
49
|
+
file_read_timeout : int, default 3600
|
50
|
+
The timeout until which we try to read the deployer attribute file (in seconds).
|
49
51
|
**kwargs : Any
|
50
52
|
Additional arguments that you would pass to `python myflow.py` i.e. options
|
51
53
|
listed in `python myflow.py --help`
|
metaflow/runner/nbrun.py
CHANGED
@@ -44,7 +44,7 @@ class NBRunner(object):
|
|
44
44
|
The directory to run the subprocess in; if not specified, the current
|
45
45
|
working directory is used.
|
46
46
|
file_read_timeout : int, default 3600
|
47
|
-
The timeout until which we try to read the runner attribute file.
|
47
|
+
The timeout until which we try to read the runner attribute file (in seconds).
|
48
48
|
**kwargs : Any
|
49
49
|
Additional arguments that you would pass to `python myflow.py` before
|
50
50
|
the `run` command.
|
@@ -120,6 +120,9 @@ class SubprocessManager(object):
|
|
120
120
|
"""
|
121
121
|
Run a command synchronously and return its process ID.
|
122
122
|
|
123
|
+
Note: in no case does this wait for the process to *finish*. Use sync_wait()
|
124
|
+
to wait for the command to finish.
|
125
|
+
|
123
126
|
Parameters
|
124
127
|
----------
|
125
128
|
command : List[str]
|
@@ -145,7 +148,6 @@ class SubprocessManager(object):
|
|
145
148
|
command_obj = CommandManager(command, env, cwd)
|
146
149
|
pid = command_obj.run(show_output=show_output)
|
147
150
|
self.commands[pid] = command_obj
|
148
|
-
command_obj.sync_wait()
|
149
151
|
return pid
|
150
152
|
|
151
153
|
async def async_run_command(
|
metaflow/runner/utils.py
CHANGED
@@ -91,7 +91,7 @@ def read_from_fifo_when_ready(
|
|
91
91
|
encoding : str, optional
|
92
92
|
Encoding to use while reading the file, by default "utf-8".
|
93
93
|
timeout : int, optional
|
94
|
-
Timeout for reading the file in
|
94
|
+
Timeout for reading the file in seconds, by default 3600.
|
95
95
|
|
96
96
|
Returns
|
97
97
|
-------
|
@@ -107,30 +107,47 @@ def read_from_fifo_when_ready(
|
|
107
107
|
content to the FIFO.
|
108
108
|
"""
|
109
109
|
content = bytearray()
|
110
|
-
|
111
110
|
poll = select.poll()
|
112
111
|
poll.register(fifo_fd, select.POLLIN)
|
113
|
-
|
112
|
+
max_timeout = 3 # Wait for 10 * 3 = 30 ms after last write
|
114
113
|
while True:
|
115
|
-
|
116
|
-
poll.poll(timeout)
|
117
|
-
timeout -= 1000 * (time.time() - poll_begin)
|
118
|
-
|
119
|
-
if timeout <= 0:
|
114
|
+
if timeout < 0:
|
120
115
|
raise TimeoutError("Timeout while waiting for the file content")
|
121
116
|
|
117
|
+
poll_begin = time.time()
|
118
|
+
# We poll for a very short time to be also able to check if the file was closed
|
119
|
+
# If the file is closed, we assume that we only have one writer so if we have
|
120
|
+
# data, we break out. This is to work around issues in macos
|
121
|
+
events = poll.poll(min(10, timeout * 1000))
|
122
|
+
timeout -= time.time() - poll_begin
|
123
|
+
|
122
124
|
try:
|
123
|
-
data = os.read(fifo_fd,
|
124
|
-
|
125
|
+
data = os.read(fifo_fd, 8192)
|
126
|
+
if data:
|
125
127
|
content += data
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
128
|
+
else:
|
129
|
+
if len(events):
|
130
|
+
# We read an EOF -- consider the file done
|
131
|
+
break
|
132
|
+
else:
|
133
|
+
# We had no events (just a timeout) and the read didn't return
|
134
|
+
# an exception so the file is still open; we continue waiting for data
|
135
|
+
# Unfortunately, on MacOS, it seems that even *after* the file is
|
136
|
+
# closed on the other end, we still don't get a BlockingIOError so
|
137
|
+
# we hack our way and timeout if there is no write in 30ms which is
|
138
|
+
# a relative eternity for file writes.
|
139
|
+
if content:
|
140
|
+
if max_timeout <= 0:
|
141
|
+
break
|
142
|
+
max_timeout -= 1
|
143
|
+
continue
|
131
144
|
except BlockingIOError:
|
132
|
-
|
133
|
-
|
145
|
+
has_blocking_error = True
|
146
|
+
if content:
|
147
|
+
# The file was closed
|
148
|
+
break
|
149
|
+
# else, if we have no content, we continue waiting for the file to be open
|
150
|
+
# and written to.
|
134
151
|
|
135
152
|
if not content and check_process_exited(command_obj):
|
136
153
|
raise CalledProcessError(command_obj.process.returncode, command_obj.command)
|
@@ -156,7 +173,7 @@ async def async_read_from_fifo_when_ready(
|
|
156
173
|
encoding : str, optional
|
157
174
|
Encoding to use while reading the file, by default "utf-8".
|
158
175
|
timeout : int, optional
|
159
|
-
Timeout for reading the file in
|
176
|
+
Timeout for reading the file in seconds, by default 3600.
|
160
177
|
|
161
178
|
Returns
|
162
179
|
-------
|
@@ -206,7 +223,7 @@ def handle_timeout(
|
|
206
223
|
command_obj : CommandManager
|
207
224
|
Command manager object that encapsulates the running command details.
|
208
225
|
file_read_timeout : int
|
209
|
-
Timeout for reading the file
|
226
|
+
Timeout for reading the file, in seconds
|
210
227
|
|
211
228
|
Returns
|
212
229
|
-------
|
@@ -243,7 +260,7 @@ async def async_handle_timeout(
|
|
243
260
|
command_obj : CommandManager
|
244
261
|
Command manager object that encapsulates the running command details.
|
245
262
|
file_read_timeout : int
|
246
|
-
Timeout for reading the file
|
263
|
+
Timeout for reading the file, in seconds
|
247
264
|
|
248
265
|
Returns
|
249
266
|
-------
|
metaflow/version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
metaflow_version = "2.12.
|
1
|
+
metaflow_version = "2.12.39"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: metaflow
|
3
|
-
Version: 2.12.
|
3
|
+
Version: 2.12.39
|
4
4
|
Summary: Metaflow: More Data Science, Less Engineering
|
5
5
|
Author: Metaflow Developers
|
6
6
|
Author-email: help@metaflow.org
|
@@ -26,7 +26,7 @@ License-File: LICENSE
|
|
26
26
|
Requires-Dist: requests
|
27
27
|
Requires-Dist: boto3
|
28
28
|
Provides-Extra: stubs
|
29
|
-
Requires-Dist: metaflow-stubs==2.12.
|
29
|
+
Requires-Dist: metaflow-stubs==2.12.39; extra == "stubs"
|
30
30
|
|
31
31
|

|
32
32
|
|
@@ -1,12 +1,12 @@
|
|
1
1
|
metaflow/R.py,sha256=CqVfIatvmjciuICNnoyyNGrwE7Va9iXfLdFbQa52hwA,3958
|
2
2
|
metaflow/__init__.py,sha256=yeHIcwunlMyp_1Y6D9iuCbWi4atVMoEp2F_6-CagnDs,5819
|
3
3
|
metaflow/cards.py,sha256=tP1_RrtmqdFh741pqE4t98S7SA0MtGRlGvRICRZF1Mg,426
|
4
|
-
metaflow/cli.py,sha256=
|
4
|
+
metaflow/cli.py,sha256=aE3CtoLcD8wsdR-mUDVbi8RuyrQOUzfzq_iqjHROjZ8,18429
|
5
5
|
metaflow/cli_args.py,sha256=nz6ZN-gv-NFTFrfHCBJRy_JtzV42iltXI4V4-sBTukE,3626
|
6
6
|
metaflow/clone_util.py,sha256=LSuVbFpPUh92UW32DBcnZbL0FFw-4w3CLa0tpEbCkzk,2066
|
7
7
|
metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
|
8
8
|
metaflow/debug.py,sha256=HEmt_16tJtqHXQXsqD9pqOFe3CWR5GZ7VwpaYQgnRdU,1466
|
9
|
-
metaflow/decorators.py,sha256=
|
9
|
+
metaflow/decorators.py,sha256=BxzcxPkc8SvVTDD5pNhE3bmy7bYn6pBPEF-WYjnw-MY,23911
|
10
10
|
metaflow/event_logger.py,sha256=joTVRqZPL87nvah4ZOwtqWX8NeraM_CXKXXGVpKGD8o,780
|
11
11
|
metaflow/events.py,sha256=ahjzkSbSnRCK9RZ-9vTfUviz_6gMvSO9DGkJ86X80-k,5300
|
12
12
|
metaflow/exception.py,sha256=KC1LHJQzzYkWib0DeQ4l_A2r8VaudywsSqIQuq1RDZU,4954
|
@@ -36,7 +36,7 @@ metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
|
|
36
36
|
metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
|
37
37
|
metaflow/util.py,sha256=wA25u2oXP-mHuDJM8Yx9gnPGZa9dP5tP1RWD9EULPow,14698
|
38
38
|
metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
|
39
|
-
metaflow/version.py,sha256=
|
39
|
+
metaflow/version.py,sha256=IDS-f-gfZhRkS0U_D4XJsfMkG2u0ReP0nYK6QpQeTI4,29
|
40
40
|
metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
|
41
41
|
metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
|
42
42
|
metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
|
@@ -113,8 +113,8 @@ metaflow/_vendor/v3_6/importlib_metadata/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCe
|
|
113
113
|
metaflow/cli_components/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
114
114
|
metaflow/cli_components/dump_cmd.py,sha256=SZEX51BWNd1o3H2uHDkYA8KRvou5X8g5rTwpdu5vnNQ,2704
|
115
115
|
metaflow/cli_components/init_cmd.py,sha256=Er-BO59UEUV1HIsg81bRtZWT2D2IZNMp93l-AoZLCls,1519
|
116
|
-
metaflow/cli_components/run_cmds.py,sha256=
|
117
|
-
metaflow/cli_components/step_cmd.py,sha256=
|
116
|
+
metaflow/cli_components/run_cmds.py,sha256=hgSPO5it4xG22ObZ3y5o8Vpf5DbZokX-Rjg-lACPLcY,10352
|
117
|
+
metaflow/cli_components/step_cmd.py,sha256=Mkztidy-wxYH7KyLSVsWK7SeYPJIeihpDClzTn9-2oo,5057
|
118
118
|
metaflow/cli_components/utils.py,sha256=gpoDociadjnJD7MuiJup_MDR02ZJjjleejr0jPBu29c,6057
|
119
119
|
metaflow/client/__init__.py,sha256=1GtQB4Y_CBkzaxg32L1syNQSlfj762wmLrfrDxGi1b8,226
|
120
120
|
metaflow/client/core.py,sha256=Ka6G12h6pS_046cej0c8npvaBGpiKyvJ1bGbRFZL6Kg,75437
|
@@ -167,7 +167,7 @@ metaflow/plugins/test_unbounded_foreach_decorator.py,sha256=33p5aCWnyk9MT5DmXcm4
|
|
167
167
|
metaflow/plugins/timeout_decorator.py,sha256=ZOUmg5HIm_9kteMC7qbzj2tTVBESwRfp1hPJd8MJMBg,3586
|
168
168
|
metaflow/plugins/airflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
169
169
|
metaflow/plugins/airflow/airflow.py,sha256=GDaKLdzzySttJfhl_OiYjkK_ubIaRKR4YgcLaKRCQLk,32293
|
170
|
-
metaflow/plugins/airflow/airflow_cli.py,sha256=
|
170
|
+
metaflow/plugins/airflow/airflow_cli.py,sha256=k6UfaDzkipUOIhoGYUV6bnS9kUNRvPNi5uN_TspTFnQ,14723
|
171
171
|
metaflow/plugins/airflow/airflow_decorator.py,sha256=IWT6M9gga8t65FR4Wi7pIZvOupk3hE75B5NRg9tMEps,1781
|
172
172
|
metaflow/plugins/airflow/airflow_utils.py,sha256=dvRllfQeOWfDUseFnOocIGaL3gRI_A7cEHnC1w01vfk,28905
|
173
173
|
metaflow/plugins/airflow/dag.py,sha256=zYV3QsyqGIOxgipbiEb4dX-r6aippNbXjuT6Jt2s4xI,129
|
@@ -182,10 +182,10 @@ metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
|
|
182
182
|
metaflow/plugins/argo/argo_client.py,sha256=Z_A1TO9yw4Y-a8VAlwrFS0BwunWzXpbtik-j_xjcuHE,16303
|
183
183
|
metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
|
184
184
|
metaflow/plugins/argo/argo_workflows.py,sha256=lPPPKlHgz2cKYHM8mdq8Aywoi6vYytyCzxd5PrsWGFE,175733
|
185
|
-
metaflow/plugins/argo/argo_workflows_cli.py,sha256=
|
185
|
+
metaflow/plugins/argo/argo_workflows_cli.py,sha256=k8rDbHMXF-A0CxxEnVMvk_9OYobPIyjF-PHBGYfH8b4,36752
|
186
186
|
metaflow/plugins/argo/argo_workflows_decorator.py,sha256=QdM1rK9gM-lDhyZldK8WqvFqJDvfJ7i3JPR5Uzaq2as,7887
|
187
187
|
metaflow/plugins/argo/argo_workflows_deployer.py,sha256=6kHxEnYXJwzNCM9swI8-0AckxtPWqwhZLerYkX8fxUM,4444
|
188
|
-
metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=
|
188
|
+
metaflow/plugins/argo/argo_workflows_deployer_objects.py,sha256=gl5Gu0gE6K079YKxqlyYuuk6nRagEq2g1Wuh_1uSv48,12481
|
189
189
|
metaflow/plugins/argo/capture_error.py,sha256=Ys9dscGrTpW-ZCirLBU0gD9qBM0BjxyxGlUMKcwewQc,1852
|
190
190
|
metaflow/plugins/argo/generate_input_paths.py,sha256=loYsI6RFX9LlFsHb7Fe-mzlTTtRdySoOu7sYDy-uXK0,881
|
191
191
|
metaflow/plugins/argo/jobset_input_paths.py,sha256=_JhZWngA6p9Q_O2fx3pdzKI0WE-HPRHz_zFvY2pHPTQ,525
|
@@ -206,11 +206,11 @@ metaflow/plugins/aws/step_functions/production_token.py,sha256=_o4emv3rozYZoWpaj
|
|
206
206
|
metaflow/plugins/aws/step_functions/schedule_decorator.py,sha256=Ab1rW8O_no4HNZm4__iBmFDCDW0Z8-TgK4lnxHHA6HI,1940
|
207
207
|
metaflow/plugins/aws/step_functions/set_batch_environment.py,sha256=ibiGWFHDjKcLfprH3OsX-g2M9lUsh6J-bp7v2cdLhD4,1294
|
208
208
|
metaflow/plugins/aws/step_functions/step_functions.py,sha256=ZQ1qKLieQ99lWm6RI0zagEW8eOKKxGBiGwnBgFJS1DQ,53146
|
209
|
-
metaflow/plugins/aws/step_functions/step_functions_cli.py,sha256=
|
209
|
+
metaflow/plugins/aws/step_functions/step_functions_cli.py,sha256=4RsVtkw_3FO9Y2HrOcffTKnzSBG5sh3IWX-F6vDZM3Y,26107
|
210
210
|
metaflow/plugins/aws/step_functions/step_functions_client.py,sha256=DKpNwAIWElvWjFANs5Ku3rgzjxFoqAD6k-EF8Xhkg3Q,4754
|
211
211
|
metaflow/plugins/aws/step_functions/step_functions_decorator.py,sha256=LoZC5BuQLqyFtfE-sGla26l2xXlCKN9aSvIlzPKV134,3800
|
212
212
|
metaflow/plugins/aws/step_functions/step_functions_deployer.py,sha256=JKYtDhKivtXUWPklprZFzkqezh14loGDmk8mNk6QtpI,3714
|
213
|
-
metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py,sha256
|
213
|
+
metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py,sha256=-ElC48q739UJk4qGEYj6H7p8nZ4xsm1iJwh5B7kEsFE,7319
|
214
214
|
metaflow/plugins/azure/__init__.py,sha256=GuuhTVC-zSdyAf79a1wiERMq0Zts7fwVT7t9fAf234A,100
|
215
215
|
metaflow/plugins/azure/azure_credential.py,sha256=JmdGEbVzgxy8ucqnQDdTTI_atyMX9WSZUw3qYOo7RhE,2174
|
216
216
|
metaflow/plugins/azure/azure_exceptions.py,sha256=NnbwpUC23bc61HZjJmeXztY0tBNn_Y_VpIpDDuYWIZ0,433
|
@@ -299,11 +299,11 @@ metaflow/plugins/metadata_providers/local.py,sha256=9UAxe9caN6kU1lkSlIoJbRGgTqsM
|
|
299
299
|
metaflow/plugins/metadata_providers/service.py,sha256=NKZfFMamx6upP6aFRJfXlfYIhySgFNzz6kbp1yPD7LA,20222
|
300
300
|
metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
|
301
301
|
metaflow/plugins/pypi/bootstrap.py,sha256=FI-itExqIz7DUzLnnkGwoB60rFBviygpIFThUtqk_4E,5227
|
302
|
-
metaflow/plugins/pypi/conda_decorator.py,sha256=
|
302
|
+
metaflow/plugins/pypi/conda_decorator.py,sha256=piFcE4uGmWhhbGlxMK0GHd7BGEyqy6r9BFy8Mjoi80Q,15937
|
303
303
|
metaflow/plugins/pypi/conda_environment.py,sha256=IGHIphHm1e8UEJX-PvyTesfKRCpxtJIc1pxJ5Wen-aU,19765
|
304
304
|
metaflow/plugins/pypi/micromamba.py,sha256=QaZYMy5w4esW2w_Lb9kZdWU07EtZD_Ky00MVlA4FJw0,14079
|
305
305
|
metaflow/plugins/pypi/pip.py,sha256=Uewmt6-meLyPhNLiAOAkDdfd1P4Go07bkQUD0uE5VIs,13827
|
306
|
-
metaflow/plugins/pypi/pypi_decorator.py,sha256=
|
306
|
+
metaflow/plugins/pypi/pypi_decorator.py,sha256=ybNgo-T5Z_0W2KNuED0pdjyI0qygZ4a1MXAzKqdHt_E,7250
|
307
307
|
metaflow/plugins/pypi/pypi_environment.py,sha256=FYMg8kF3lXqcLfRYWD83a9zpVjcoo_TARqMGZ763rRk,230
|
308
308
|
metaflow/plugins/pypi/utils.py,sha256=ds1Mnv_DaxGnLAYp7ozg_K6oyguGyNhvHfE-75Ia1YA,2836
|
309
309
|
metaflow/plugins/secrets/__init__.py,sha256=mhJaN2eMS_ZZVewAMR2E-JdP5i0t3v9e6Dcwd-WpruE,310
|
@@ -311,13 +311,13 @@ metaflow/plugins/secrets/inline_secrets_provider.py,sha256=EChmoBGA1i7qM3jtYwPpL
|
|
311
311
|
metaflow/plugins/secrets/secrets_decorator.py,sha256=s-sFzPWOjahhpr5fMj-ZEaHkDYAPTO0isYXGvaUwlG8,11273
|
312
312
|
metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
313
313
|
metaflow/runner/click_api.py,sha256=BicwlQRGrLhy2xOYo89IV9pu5L8ZkGGTUI2FTsrq33I,15944
|
314
|
-
metaflow/runner/deployer.py,sha256=
|
315
|
-
metaflow/runner/deployer_impl.py,sha256=
|
316
|
-
metaflow/runner/metaflow_runner.py,sha256=
|
317
|
-
metaflow/runner/nbdeploy.py,sha256=
|
318
|
-
metaflow/runner/nbrun.py,sha256=
|
319
|
-
metaflow/runner/subprocess_manager.py,sha256=
|
320
|
-
metaflow/runner/utils.py,sha256=
|
314
|
+
metaflow/runner/deployer.py,sha256=Yas_SZCss3kfJw3hLC8_IyzgiytUFGoEGHz-l-rBBKk,8980
|
315
|
+
metaflow/runner/deployer_impl.py,sha256=fylQone4_utUFTeHUfnGpGHNkJqaxrkxRxpG6f-b38g,5673
|
316
|
+
metaflow/runner/metaflow_runner.py,sha256=B8WLQ2-M82enY4h48el9MME3Yrkt5NoB88WRXrlSItw,14874
|
317
|
+
metaflow/runner/nbdeploy.py,sha256=Sp5w-6nCZwjHaRBHWxi8udya-RYnJOB76KNLjB4L7Gs,4166
|
318
|
+
metaflow/runner/nbrun.py,sha256=LhJu-Teoi7wTkNxg0kpNPVXFxH_9P4lvtp0ysMEIFJ8,7299
|
319
|
+
metaflow/runner/subprocess_manager.py,sha256=K6uZXnqdgeW0vHUAVwoolSpDSLp1EVHiBtyD7f_vwac,22050
|
320
|
+
metaflow/runner/utils.py,sha256=UZfhYVdF-c5NyMrDzZW_C_3z8bBG_duW7rRv7S6_PZc,10006
|
321
321
|
metaflow/sidecar/__init__.py,sha256=1mmNpmQ5puZCpRmmYlCOeieZ4108Su9XQ4_EqF1FGOU,131
|
322
322
|
metaflow/sidecar/sidecar.py,sha256=EspKXvPPNiyRToaUZ51PS5TT_PzrBNAurn_wbFnmGr0,1334
|
323
323
|
metaflow/sidecar/sidecar_messages.py,sha256=zPsCoYgDIcDkkvdC9MEpJTJ3y6TSGm2JWkRc4vxjbFA,1071
|
@@ -358,9 +358,9 @@ metaflow/user_configs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3h
|
|
358
358
|
metaflow/user_configs/config_decorators.py,sha256=Tj0H88UT8Q6pylXxHXgiA6cqnNlw4d3mR7M8J9g3ZUg,20139
|
359
359
|
metaflow/user_configs/config_options.py,sha256=3l293IQHOE-DqNzDt7kc1vLWMLuevoJceYVQ43dEhQY,19165
|
360
360
|
metaflow/user_configs/config_parameters.py,sha256=cBlnhLoax-QbpD46oofuLUJhT3_8WHscLNDSpO8hWOU,13098
|
361
|
-
metaflow-2.12.
|
362
|
-
metaflow-2.12.
|
363
|
-
metaflow-2.12.
|
364
|
-
metaflow-2.12.
|
365
|
-
metaflow-2.12.
|
366
|
-
metaflow-2.12.
|
361
|
+
metaflow-2.12.39.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
|
362
|
+
metaflow-2.12.39.dist-info/METADATA,sha256=fwphR4UNEn8SAcWMFBHPYDYkB3x1iZM-0XEwghcmp0E,5908
|
363
|
+
metaflow-2.12.39.dist-info/WHEEL,sha256=pxeNX5JdtCe58PUSYP9upmc7jdRPgvT0Gm9kb1SHlVw,109
|
364
|
+
metaflow-2.12.39.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
|
365
|
+
metaflow-2.12.39.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
|
366
|
+
metaflow-2.12.39.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|
File without changes
|