ob-metaflow 2.12.0.1__py2.py3-none-any.whl → 2.12.4.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ob-metaflow might be problematic. Click here for more details.
- metaflow/R.py +10 -7
- metaflow/cli.py +12 -3
- metaflow/client/core.py +55 -15
- metaflow/decorators.py +8 -7
- metaflow/flowspec.py +21 -15
- metaflow/metaflow_config.py +3 -0
- metaflow/parameters.py +42 -1
- metaflow/plugins/airflow/airflow.py +1 -1
- metaflow/plugins/airflow/sensors/base_sensor.py +4 -4
- metaflow/plugins/airflow/sensors/external_task_sensor.py +2 -2
- metaflow/plugins/airflow/sensors/s3_sensor.py +2 -2
- metaflow/plugins/argo/argo_workflows.py +11 -6
- metaflow/plugins/aws/step_functions/step_functions.py +1 -1
- metaflow/plugins/env_escape/stub.py +3 -1
- metaflow/plugins/pypi/bootstrap.py +1 -1
- metaflow/runner/click_api.py +18 -9
- metaflow/runner/metaflow_runner.py +4 -0
- metaflow/runtime.py +2 -8
- metaflow/version.py +1 -1
- {ob_metaflow-2.12.0.1.dist-info → ob_metaflow-2.12.4.1.dist-info}/METADATA +2 -2
- {ob_metaflow-2.12.0.1.dist-info → ob_metaflow-2.12.4.1.dist-info}/RECORD +25 -25
- {ob_metaflow-2.12.0.1.dist-info → ob_metaflow-2.12.4.1.dist-info}/LICENSE +0 -0
- {ob_metaflow-2.12.0.1.dist-info → ob_metaflow-2.12.4.1.dist-info}/WHEEL +0 -0
- {ob_metaflow-2.12.0.1.dist-info → ob_metaflow-2.12.4.1.dist-info}/entry_points.txt +0 -0
- {ob_metaflow-2.12.0.1.dist-info → ob_metaflow-2.12.4.1.dist-info}/top_level.txt +0 -0
metaflow/R.py
CHANGED
|
@@ -3,6 +3,7 @@ import sys
|
|
|
3
3
|
from importlib import util as imp_util, machinery as imp_machinery
|
|
4
4
|
from tempfile import NamedTemporaryFile
|
|
5
5
|
|
|
6
|
+
from . import parameters
|
|
6
7
|
from .util import to_bytes
|
|
7
8
|
|
|
8
9
|
R_FUNCTIONS = {}
|
|
@@ -125,15 +126,17 @@ def run(
|
|
|
125
126
|
flow = module.FLOW(use_cli=False)
|
|
126
127
|
|
|
127
128
|
from . import exception
|
|
128
|
-
from . import cli
|
|
129
129
|
|
|
130
130
|
try:
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
131
|
+
with parameters.flow_context(flow.__class__) as _:
|
|
132
|
+
from . import cli
|
|
133
|
+
|
|
134
|
+
cli.main(
|
|
135
|
+
flow,
|
|
136
|
+
args=metaflow_args,
|
|
137
|
+
handle_exceptions=False,
|
|
138
|
+
entrypoint=full_cmdline[: -len(metaflow_args)],
|
|
139
|
+
)
|
|
137
140
|
except exception.MetaflowException as e:
|
|
138
141
|
cli.print_metaflow_exception(e)
|
|
139
142
|
os.remove(tmp.name)
|
metaflow/cli.py
CHANGED
|
@@ -10,7 +10,6 @@ from metaflow._vendor import click
|
|
|
10
10
|
|
|
11
11
|
from . import decorators, lint, metaflow_version, namespace, parameters, plugins
|
|
12
12
|
from .cli_args import cli_args
|
|
13
|
-
from .client.core import get_metadata
|
|
14
13
|
from .datastore import FlowDataStore, TaskDataStore, TaskDataStoreSet
|
|
15
14
|
from .exception import CommandException, MetaflowException
|
|
16
15
|
from .graph import FlowGraph
|
|
@@ -700,7 +699,12 @@ def resume(
|
|
|
700
699
|
runtime.persist_constants()
|
|
701
700
|
write_file(
|
|
702
701
|
runner_attribute_file,
|
|
703
|
-
"%s:%s"
|
|
702
|
+
"%s@%s:%s"
|
|
703
|
+
% (
|
|
704
|
+
obj.metadata.__class__.TYPE,
|
|
705
|
+
obj.metadata.__class__.INFO,
|
|
706
|
+
"/".join((obj.flow.name, runtime.run_id)),
|
|
707
|
+
),
|
|
704
708
|
)
|
|
705
709
|
if clone_only:
|
|
706
710
|
runtime.clone_original_run()
|
|
@@ -763,7 +767,12 @@ def run(
|
|
|
763
767
|
runtime.persist_constants()
|
|
764
768
|
write_file(
|
|
765
769
|
runner_attribute_file,
|
|
766
|
-
"%s:%s"
|
|
770
|
+
"%s@%s:%s"
|
|
771
|
+
% (
|
|
772
|
+
obj.metadata.__class__.TYPE,
|
|
773
|
+
obj.metadata.__class__.INFO,
|
|
774
|
+
"/".join((obj.flow.name, runtime.run_id)),
|
|
775
|
+
),
|
|
767
776
|
)
|
|
768
777
|
runtime.execute()
|
|
769
778
|
|
metaflow/client/core.py
CHANGED
|
@@ -268,11 +268,13 @@ class MetaflowObject(object):
|
|
|
268
268
|
_object: Optional["MetaflowObject"] = None,
|
|
269
269
|
_parent: Optional["MetaflowObject"] = None,
|
|
270
270
|
_namespace_check: bool = True,
|
|
271
|
+
_current_namespace: Optional[str] = None,
|
|
271
272
|
):
|
|
272
273
|
self._metaflow = Metaflow()
|
|
273
274
|
self._parent = _parent
|
|
274
275
|
self._path_components = None
|
|
275
276
|
self._attempt = attempt
|
|
277
|
+
self._current_namespace = _current_namespace or get_namespace()
|
|
276
278
|
self._namespace_check = _namespace_check
|
|
277
279
|
|
|
278
280
|
if self._attempt is not None:
|
|
@@ -339,8 +341,8 @@ class MetaflowObject(object):
|
|
|
339
341
|
self._user_tags = frozenset(self._object.get("tags") or [])
|
|
340
342
|
self._system_tags = frozenset(self._object.get("system_tags") or [])
|
|
341
343
|
|
|
342
|
-
if self._namespace_check and not self.
|
|
343
|
-
raise MetaflowNamespaceMismatch(
|
|
344
|
+
if self._namespace_check and not self._is_in_namespace(self._current_namespace):
|
|
345
|
+
raise MetaflowNamespaceMismatch(self._current_namespace)
|
|
344
346
|
|
|
345
347
|
def _get_object(self, *path_components):
|
|
346
348
|
result = self._metaflow.metadata.get_object(
|
|
@@ -365,8 +367,8 @@ class MetaflowObject(object):
|
|
|
365
367
|
query_filter = {}
|
|
366
368
|
|
|
367
369
|
# skip namespace filtering if _namespace_check is unset.
|
|
368
|
-
if self._namespace_check and
|
|
369
|
-
query_filter = {"any_tags":
|
|
370
|
+
if self._namespace_check and self._current_namespace:
|
|
371
|
+
query_filter = {"any_tags": self._current_namespace}
|
|
370
372
|
|
|
371
373
|
unfiltered_children = self._metaflow.metadata.get_object(
|
|
372
374
|
self._NAME,
|
|
@@ -383,7 +385,10 @@ class MetaflowObject(object):
|
|
|
383
385
|
attempt=self._attempt,
|
|
384
386
|
_object=obj,
|
|
385
387
|
_parent=self,
|
|
386
|
-
_namespace_check=
|
|
388
|
+
_namespace_check=self._namespace_check,
|
|
389
|
+
_current_namespace=self._current_namespace
|
|
390
|
+
if self._namespace_check
|
|
391
|
+
else None,
|
|
387
392
|
)
|
|
388
393
|
for obj in unfiltered_children
|
|
389
394
|
),
|
|
@@ -422,6 +427,23 @@ class MetaflowObject(object):
|
|
|
422
427
|
|
|
423
428
|
If the current namespace is None, this will always return True.
|
|
424
429
|
|
|
430
|
+
Returns
|
|
431
|
+
-------
|
|
432
|
+
bool
|
|
433
|
+
Whether or not the object is in the current namespace
|
|
434
|
+
"""
|
|
435
|
+
return self._is_in_namespace(current_namespace)
|
|
436
|
+
|
|
437
|
+
def _is_in_namespace(self, ns: str) -> bool:
|
|
438
|
+
"""
|
|
439
|
+
Returns whether this object is in namespace passed in.
|
|
440
|
+
|
|
441
|
+
If the current namespace is None, this will always return True.
|
|
442
|
+
|
|
443
|
+
Parameters
|
|
444
|
+
----------
|
|
445
|
+
ns : str
|
|
446
|
+
Namespace to check if the object is in.
|
|
425
447
|
Returns
|
|
426
448
|
-------
|
|
427
449
|
bool
|
|
@@ -430,7 +452,7 @@ class MetaflowObject(object):
|
|
|
430
452
|
if self._NAME == "flow":
|
|
431
453
|
return any(True for _ in self)
|
|
432
454
|
else:
|
|
433
|
-
return
|
|
455
|
+
return ns is None or ns in self._tags
|
|
434
456
|
|
|
435
457
|
def __str__(self):
|
|
436
458
|
if self._attempt is not None:
|
|
@@ -479,6 +501,9 @@ class MetaflowObject(object):
|
|
|
479
501
|
_object=obj,
|
|
480
502
|
_parent=self,
|
|
481
503
|
_namespace_check=self._namespace_check,
|
|
504
|
+
_current_namespace=self._current_namespace
|
|
505
|
+
if self._namespace_check
|
|
506
|
+
else None,
|
|
482
507
|
)
|
|
483
508
|
else:
|
|
484
509
|
raise KeyError(id)
|
|
@@ -509,7 +534,20 @@ class MetaflowObject(object):
|
|
|
509
534
|
pathspec=pathspec, attempt=attempt, _namespace_check=namespace_check
|
|
510
535
|
)
|
|
511
536
|
|
|
512
|
-
|
|
537
|
+
def _unpickle_2124(self, data):
|
|
538
|
+
if len(data) != 4:
|
|
539
|
+
raise MetaflowInternalError(
|
|
540
|
+
"Unexpected size of array: {}".format(len(data))
|
|
541
|
+
)
|
|
542
|
+
pathspec, attempt, ns, namespace_check = data
|
|
543
|
+
self.__init__(
|
|
544
|
+
pathspec=pathspec,
|
|
545
|
+
attempt=attempt,
|
|
546
|
+
_namespace_check=namespace_check,
|
|
547
|
+
_current_namespace=ns,
|
|
548
|
+
)
|
|
549
|
+
|
|
550
|
+
_UNPICKLE_FUNC = {"2.8.4": _unpickle_284, "2.12.4": _unpickle_2124}
|
|
513
551
|
|
|
514
552
|
def __setstate__(self, state):
|
|
515
553
|
"""
|
|
@@ -529,12 +567,13 @@ class MetaflowObject(object):
|
|
|
529
567
|
self._UNPICKLE_FUNC[version](self, state["data"])
|
|
530
568
|
else:
|
|
531
569
|
# For backward compatibility: handles pickled objects that were serialized without a __getstate__ override
|
|
532
|
-
# We set namespace_check to False if it doesn't exist
|
|
533
|
-
#
|
|
570
|
+
# We set namespace_check to False if it doesn't exist so that the user can
|
|
571
|
+
# continue accessing this object once unpickled.
|
|
534
572
|
self.__init__(
|
|
535
573
|
pathspec=state.get("_pathspec", None),
|
|
536
574
|
attempt=state.get("_attempt", None),
|
|
537
575
|
_namespace_check=state.get("_namespace_check", False),
|
|
576
|
+
_current_namespace=None,
|
|
538
577
|
)
|
|
539
578
|
|
|
540
579
|
def __getstate__(self):
|
|
@@ -546,16 +585,17 @@ class MetaflowObject(object):
|
|
|
546
585
|
from this object) are pickled (serialized) in a later version of Metaflow, it may not be possible
|
|
547
586
|
to unpickle (deserialize) them in a previous version of Metaflow.
|
|
548
587
|
"""
|
|
549
|
-
# Note that we
|
|
550
|
-
#
|
|
551
|
-
#
|
|
552
|
-
#
|
|
588
|
+
# Note that we now record the namespace at the time of the object creation so
|
|
589
|
+
# we don't need to force namespace_check to be False and can properly continue
|
|
590
|
+
# checking for the namespace even after unpickling since we will know which
|
|
591
|
+
# namespace to check.
|
|
553
592
|
return {
|
|
554
|
-
"version": "2.
|
|
593
|
+
"version": "2.12.4",
|
|
555
594
|
"data": [
|
|
556
595
|
self.pathspec,
|
|
557
596
|
self._attempt,
|
|
558
|
-
|
|
597
|
+
self._current_namespace,
|
|
598
|
+
self._namespace_check,
|
|
559
599
|
],
|
|
560
600
|
}
|
|
561
601
|
|
metaflow/decorators.py
CHANGED
|
@@ -11,6 +11,8 @@ from .exception import (
|
|
|
11
11
|
InvalidDecoratorAttribute,
|
|
12
12
|
)
|
|
13
13
|
|
|
14
|
+
from .parameters import current_flow
|
|
15
|
+
|
|
14
16
|
from metaflow._vendor import click
|
|
15
17
|
|
|
16
18
|
try:
|
|
@@ -174,13 +176,9 @@ class Decorator(object):
|
|
|
174
176
|
|
|
175
177
|
|
|
176
178
|
class FlowDecorator(Decorator):
|
|
177
|
-
_flow_decorators = []
|
|
178
179
|
options = {}
|
|
179
180
|
|
|
180
181
|
def __init__(self, *args, **kwargs):
|
|
181
|
-
# Note that this assumes we are executing one flow per process, so we have a global list of
|
|
182
|
-
# _flow_decorators. A similar setup is used in parameters.
|
|
183
|
-
self._flow_decorators.append(self)
|
|
184
182
|
super(FlowDecorator, self).__init__(*args, **kwargs)
|
|
185
183
|
|
|
186
184
|
def flow_init(
|
|
@@ -206,7 +204,10 @@ class FlowDecorator(Decorator):
|
|
|
206
204
|
# compare this to parameters.add_custom_parameters
|
|
207
205
|
def add_decorator_options(cmd):
|
|
208
206
|
seen = {}
|
|
209
|
-
|
|
207
|
+
flow_cls = getattr(current_flow, "flow_cls", None)
|
|
208
|
+
if flow_cls is None:
|
|
209
|
+
return cmd
|
|
210
|
+
for deco in flow_decorators(flow_cls):
|
|
210
211
|
for option, kwargs in deco.options.items():
|
|
211
212
|
if option in seen:
|
|
212
213
|
msg = (
|
|
@@ -222,8 +223,8 @@ def add_decorator_options(cmd):
|
|
|
222
223
|
return cmd
|
|
223
224
|
|
|
224
225
|
|
|
225
|
-
def flow_decorators():
|
|
226
|
-
return
|
|
226
|
+
def flow_decorators(flow_cls):
|
|
227
|
+
return [d for deco_list in flow_cls._flow_decorators.values() for d in deco_list]
|
|
227
228
|
|
|
228
229
|
|
|
229
230
|
class StepDecorator(Decorator):
|
metaflow/flowspec.py
CHANGED
|
@@ -53,7 +53,18 @@ class ParallelUBF(UnboundedForeachInput):
|
|
|
53
53
|
return item or 0 # item is None for the control task, but it is also split 0
|
|
54
54
|
|
|
55
55
|
|
|
56
|
-
class
|
|
56
|
+
class _FlowSpecMeta(type):
|
|
57
|
+
def __new__(cls, name, bases, dct):
|
|
58
|
+
f = super().__new__(cls, name, bases, dct)
|
|
59
|
+
# This makes sure to give _flow_decorators to each
|
|
60
|
+
# child class (and not share it with the FlowSpec base
|
|
61
|
+
# class). This is important to not make a "global"
|
|
62
|
+
# _flow_decorators
|
|
63
|
+
f._flow_decorators = {}
|
|
64
|
+
return f
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class FlowSpec(metaclass=_FlowSpecMeta):
|
|
57
68
|
"""
|
|
58
69
|
Main class from which all Flows should inherit.
|
|
59
70
|
|
|
@@ -83,8 +94,6 @@ class FlowSpec(object):
|
|
|
83
94
|
# names starting with `_` as those are already excluded from `_get_parameters`.
|
|
84
95
|
_NON_PARAMETERS = {"cmd", "foreach_stack", "index", "input", "script_name", "name"}
|
|
85
96
|
|
|
86
|
-
_flow_decorators = {}
|
|
87
|
-
|
|
88
97
|
def __init__(self, use_cli=True):
|
|
89
98
|
"""
|
|
90
99
|
Construct a FlowSpec
|
|
@@ -104,15 +113,11 @@ class FlowSpec(object):
|
|
|
104
113
|
self._graph = FlowGraph(self.__class__)
|
|
105
114
|
self._steps = [getattr(self, node.name) for node in self._graph]
|
|
106
115
|
|
|
107
|
-
# This must be set before calling cli.main() below (or specifically, add_custom_parameters)
|
|
108
|
-
parameters.parameters = [p for _, p in self._get_parameters()]
|
|
109
|
-
|
|
110
116
|
if use_cli:
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
from . import cli
|
|
117
|
+
with parameters.flow_context(self.__class__) as _:
|
|
118
|
+
from . import cli
|
|
114
119
|
|
|
115
|
-
|
|
120
|
+
cli.main(self)
|
|
116
121
|
|
|
117
122
|
@property
|
|
118
123
|
def script_name(self) -> str:
|
|
@@ -192,18 +197,19 @@ class FlowSpec(object):
|
|
|
192
197
|
"attributes": deco.attributes,
|
|
193
198
|
"statically_defined": deco.statically_defined,
|
|
194
199
|
}
|
|
195
|
-
for deco in flow_decorators()
|
|
200
|
+
for deco in flow_decorators(self)
|
|
196
201
|
if not deco.name.startswith("_")
|
|
197
202
|
],
|
|
198
203
|
}
|
|
199
204
|
self._graph_info = graph_info
|
|
200
205
|
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
206
|
+
@classmethod
|
|
207
|
+
def _get_parameters(cls):
|
|
208
|
+
for var in dir(cls):
|
|
209
|
+
if var[0] == "_" or var in cls._NON_PARAMETERS:
|
|
204
210
|
continue
|
|
205
211
|
try:
|
|
206
|
-
val = getattr(
|
|
212
|
+
val = getattr(cls, var)
|
|
207
213
|
except:
|
|
208
214
|
continue
|
|
209
215
|
if isinstance(val, Parameter):
|
metaflow/metaflow_config.py
CHANGED
|
@@ -26,6 +26,9 @@ DEFAULT_METADATA = from_conf("DEFAULT_METADATA", "local")
|
|
|
26
26
|
DEFAULT_MONITOR = from_conf("DEFAULT_MONITOR", "nullSidecarMonitor")
|
|
27
27
|
DEFAULT_PACKAGE_SUFFIXES = from_conf("DEFAULT_PACKAGE_SUFFIXES", ".py,.R,.RDS")
|
|
28
28
|
DEFAULT_AWS_CLIENT_PROVIDER = from_conf("DEFAULT_AWS_CLIENT_PROVIDER", "boto3")
|
|
29
|
+
DEFAULT_AZURE_CLIENT_PROVIDER = from_conf(
|
|
30
|
+
"DEFAULT_AZURE_CLIENT_PROVIDER", "azure-default"
|
|
31
|
+
)
|
|
29
32
|
DEFAULT_GCP_CLIENT_PROVIDER = from_conf("DEFAULT_GCP_CLIENT_PROVIDER", "gcp-default")
|
|
30
33
|
DEFAULT_SECRETS_BACKEND_TYPE = from_conf("DEFAULT_SECRETS_BACKEND_TYPE")
|
|
31
34
|
DEFAULT_SECRETS_ROLE = from_conf("DEFAULT_SECRETS_ROLE")
|
metaflow/parameters.py
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
1
|
import json
|
|
2
|
+
|
|
3
|
+
from contextlib import contextmanager
|
|
4
|
+
from threading import local
|
|
5
|
+
|
|
2
6
|
from typing import Any, Callable, Dict, NamedTuple, Optional, Type, Union
|
|
3
7
|
|
|
4
8
|
from metaflow._vendor import click
|
|
@@ -31,7 +35,40 @@ ParameterContext = NamedTuple(
|
|
|
31
35
|
],
|
|
32
36
|
)
|
|
33
37
|
|
|
34
|
-
|
|
38
|
+
|
|
39
|
+
# When we launch a flow, we need to know the parameters so we can
|
|
40
|
+
# attach them with add_custom_parameters to commands. This used to be a global
|
|
41
|
+
# but causes problems when multiple FlowSpec are loaded (as can happen when using
|
|
42
|
+
# the Runner or just if multiple Flows are defined and instantiated). To minimally
|
|
43
|
+
# impact code, we now create the CLI with a thread local value of the FlowSpec
|
|
44
|
+
# that is being used to create the CLI which enables us to extract the parameters
|
|
45
|
+
# directly from the Flow.
|
|
46
|
+
current_flow = local()
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@contextmanager
|
|
50
|
+
def flow_context(flow_cls):
|
|
51
|
+
"""
|
|
52
|
+
Context manager to set the current flow for the thread. This is used
|
|
53
|
+
to extract the parameters from the FlowSpec that is being used to create
|
|
54
|
+
the CLI.
|
|
55
|
+
"""
|
|
56
|
+
# Use a stack because with the runner this can get called multiple times in
|
|
57
|
+
# a nested fashion
|
|
58
|
+
current_flow.flow_cls_stack = getattr(current_flow, "flow_cls_stack", [])
|
|
59
|
+
current_flow.flow_cls_stack.insert(0, flow_cls)
|
|
60
|
+
current_flow.flow_cls = current_flow.flow_cls_stack[0]
|
|
61
|
+
try:
|
|
62
|
+
yield
|
|
63
|
+
finally:
|
|
64
|
+
current_flow.flow_cls_stack = current_flow.flow_cls_stack[1:]
|
|
65
|
+
if len(current_flow.flow_cls_stack) == 0:
|
|
66
|
+
del current_flow.flow_cls_stack
|
|
67
|
+
del current_flow.flow_cls
|
|
68
|
+
else:
|
|
69
|
+
current_flow.flow_cls = current_flow.flow_cls_stack[0]
|
|
70
|
+
|
|
71
|
+
|
|
35
72
|
context_proto = None
|
|
36
73
|
|
|
37
74
|
|
|
@@ -391,6 +428,10 @@ def add_custom_parameters(deploy_mode=False):
|
|
|
391
428
|
cmd.has_flow_params = True
|
|
392
429
|
# Iterate over parameters in reverse order so cmd.params lists options
|
|
393
430
|
# in the order they are defined in the FlowSpec subclass
|
|
431
|
+
flow_cls = getattr(current_flow, "flow_cls", None)
|
|
432
|
+
if flow_cls is None:
|
|
433
|
+
return cmd
|
|
434
|
+
parameters = [p for _, p in flow_cls._get_parameters()]
|
|
394
435
|
for arg in parameters[::-1]:
|
|
395
436
|
kwargs = arg.option_kwargs(deploy_mode)
|
|
396
437
|
cmd.params.insert(0, click.Option(("--" + arg.name,), **kwargs))
|
|
@@ -539,7 +539,7 @@ class Airflow(object):
|
|
|
539
539
|
# FlowDecorators can define their own top-level options. They are
|
|
540
540
|
# responsible for adding their own top-level options and values through
|
|
541
541
|
# the get_top_level_options() hook. See similar logic in runtime.py.
|
|
542
|
-
for deco in flow_decorators():
|
|
542
|
+
for deco in flow_decorators(self.flow):
|
|
543
543
|
top_opts_dict.update(deco.get_top_level_options())
|
|
544
544
|
|
|
545
545
|
top_opts = list(dict_to_cli_options(top_opts_dict))
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import uuid
|
|
2
|
-
from metaflow.decorators import FlowDecorator
|
|
2
|
+
from metaflow.decorators import FlowDecorator, flow_decorators
|
|
3
3
|
from ..exception import AirflowException
|
|
4
4
|
from ..airflow_utils import AirflowTask, id_creator, TASK_ID_HASH_LEN
|
|
5
5
|
|
|
@@ -49,7 +49,7 @@ class AirflowSensorDecorator(FlowDecorator):
|
|
|
49
49
|
operator_type=self.operator_type,
|
|
50
50
|
).set_operator_args(**{k: v for k, v in task_args.items() if v is not None})
|
|
51
51
|
|
|
52
|
-
def validate(self):
|
|
52
|
+
def validate(self, flow):
|
|
53
53
|
"""
|
|
54
54
|
Validate if the arguments for the sensor are correct.
|
|
55
55
|
"""
|
|
@@ -58,7 +58,7 @@ class AirflowSensorDecorator(FlowDecorator):
|
|
|
58
58
|
if self.attributes["name"] is None:
|
|
59
59
|
deco_index = [
|
|
60
60
|
d._id
|
|
61
|
-
for d in
|
|
61
|
+
for d in flow_decorators(flow)
|
|
62
62
|
if issubclass(d.__class__, AirflowSensorDecorator)
|
|
63
63
|
].index(self._id)
|
|
64
64
|
self._airflow_task_name = "%s-%s" % (
|
|
@@ -71,4 +71,4 @@ class AirflowSensorDecorator(FlowDecorator):
|
|
|
71
71
|
def flow_init(
|
|
72
72
|
self, flow, graph, environment, flow_datastore, metadata, logger, echo, options
|
|
73
73
|
):
|
|
74
|
-
self.validate()
|
|
74
|
+
self.validate(flow)
|
|
@@ -85,7 +85,7 @@ class ExternalTaskSensorDecorator(AirflowSensorDecorator):
|
|
|
85
85
|
)
|
|
86
86
|
return task_args
|
|
87
87
|
|
|
88
|
-
def validate(self):
|
|
88
|
+
def validate(self, flow):
|
|
89
89
|
if self.attributes["external_dag_id"] is None:
|
|
90
90
|
raise AirflowException(
|
|
91
91
|
"`%s` argument of `@%s`cannot be `None`."
|
|
@@ -131,4 +131,4 @@ class ExternalTaskSensorDecorator(AirflowSensorDecorator):
|
|
|
131
131
|
self.name,
|
|
132
132
|
)
|
|
133
133
|
)
|
|
134
|
-
super().validate()
|
|
134
|
+
super().validate(flow)
|
|
@@ -58,9 +58,9 @@ class S3KeySensorDecorator(AirflowSensorDecorator):
|
|
|
58
58
|
# `verify` is a airflow variable.
|
|
59
59
|
)
|
|
60
60
|
|
|
61
|
-
def validate(self):
|
|
61
|
+
def validate(self, flow):
|
|
62
62
|
if self.attributes["bucket_key"] is None:
|
|
63
63
|
raise AirflowException(
|
|
64
64
|
"`bucket_key` for `@%s`cannot be empty." % (self.name)
|
|
65
65
|
)
|
|
66
|
-
super().validate()
|
|
66
|
+
super().validate(flow)
|
|
@@ -184,6 +184,12 @@ class ArgoWorkflows(object):
|
|
|
184
184
|
# allowed by Metaflow - guaranteeing uniqueness.
|
|
185
185
|
return name.replace("_", "-")
|
|
186
186
|
|
|
187
|
+
@staticmethod
|
|
188
|
+
def _sensor_name(name):
|
|
189
|
+
# Unfortunately, Argo Events Sensor names don't allow for
|
|
190
|
+
# dots (sensors run into an error) which rules out self.name :(
|
|
191
|
+
return name.replace(".", "-")
|
|
192
|
+
|
|
187
193
|
@staticmethod
|
|
188
194
|
def list_templates(flow_name, all=False):
|
|
189
195
|
client = ArgoClient(namespace=KUBERNETES_NAMESPACE)
|
|
@@ -216,7 +222,7 @@ class ArgoWorkflows(object):
|
|
|
216
222
|
|
|
217
223
|
# The workflow might have sensors attached to it, which consume actual resources.
|
|
218
224
|
# Try to delete these as well.
|
|
219
|
-
sensor_deleted = client.delete_sensor(name)
|
|
225
|
+
sensor_deleted = client.delete_sensor(ArgoWorkflows._sensor_name(name))
|
|
220
226
|
|
|
221
227
|
# After cleaning up related resources, delete the workflow in question.
|
|
222
228
|
# Failure in deleting is treated as critical and will be made visible to the user
|
|
@@ -333,10 +339,9 @@ class ArgoWorkflows(object):
|
|
|
333
339
|
argo_client.schedule_workflow_template(
|
|
334
340
|
self.name, self._schedule, self._timezone
|
|
335
341
|
)
|
|
336
|
-
# Register sensor.
|
|
337
|
-
# dots (sensors run into an error) which rules out self.name :(
|
|
342
|
+
# Register sensor.
|
|
338
343
|
# Metaflow will overwrite any existing sensor.
|
|
339
|
-
sensor_name = self.name
|
|
344
|
+
sensor_name = ArgoWorkflows._sensor_name(self.name)
|
|
340
345
|
if self._sensor:
|
|
341
346
|
argo_client.register_sensor(sensor_name, self._sensor.to_json())
|
|
342
347
|
else:
|
|
@@ -1232,7 +1237,7 @@ class ArgoWorkflows(object):
|
|
|
1232
1237
|
# FlowDecorators can define their own top-level options. They are
|
|
1233
1238
|
# responsible for adding their own top-level options and values through
|
|
1234
1239
|
# the get_top_level_options() hook. See similar logic in runtime.py.
|
|
1235
|
-
for deco in flow_decorators():
|
|
1240
|
+
for deco in flow_decorators(self.flow):
|
|
1236
1241
|
top_opts_dict.update(deco.get_top_level_options())
|
|
1237
1242
|
|
|
1238
1243
|
top_level = list(dict_to_cli_options(top_opts_dict)) + [
|
|
@@ -2039,7 +2044,7 @@ class ArgoWorkflows(object):
|
|
|
2039
2044
|
.metadata(
|
|
2040
2045
|
# Sensor metadata.
|
|
2041
2046
|
ObjectMeta()
|
|
2042
|
-
.name(self.name
|
|
2047
|
+
.name(ArgoWorkflows._sensor_name(self.name))
|
|
2043
2048
|
.namespace(KUBERNETES_NAMESPACE)
|
|
2044
2049
|
.label("app.kubernetes.io/name", "metaflow-sensor")
|
|
2045
2050
|
.label("app.kubernetes.io/part-of", "metaflow")
|
|
@@ -881,7 +881,7 @@ class StepFunctions(object):
|
|
|
881
881
|
# FlowDecorators can define their own top-level options. They are
|
|
882
882
|
# responsible for adding their own top-level options and values through
|
|
883
883
|
# the get_top_level_options() hook. See similar logic in runtime.py.
|
|
884
|
-
for deco in flow_decorators():
|
|
884
|
+
for deco in flow_decorators(self.flow):
|
|
885
885
|
top_opts_dict.update(deco.get_top_level_options())
|
|
886
886
|
|
|
887
887
|
top_opts = list(dict_to_cli_options(top_opts_dict))
|
|
@@ -146,7 +146,7 @@ class Stub(with_metaclass(StubMetaClass, object)):
|
|
|
146
146
|
return object.__getattribute__(self, name)
|
|
147
147
|
|
|
148
148
|
def __getattr__(self, name):
|
|
149
|
-
if name in DELETED_ATTRS:
|
|
149
|
+
if name in DELETED_ATTRS or self.___is_returned_exception___:
|
|
150
150
|
raise AttributeError()
|
|
151
151
|
return fwd_request(self, OP_GETATTR, name)
|
|
152
152
|
|
|
@@ -166,6 +166,8 @@ class Stub(with_metaclass(StubMetaClass, object)):
|
|
|
166
166
|
):
|
|
167
167
|
object.__setattr__(self, name, value)
|
|
168
168
|
else:
|
|
169
|
+
if self.___is_returned_exception___:
|
|
170
|
+
raise AttributeError()
|
|
169
171
|
fwd_request(self, OP_SETATTR, name, value)
|
|
170
172
|
|
|
171
173
|
def __dir__(self):
|
|
@@ -123,7 +123,7 @@ if __name__ == "__main__":
|
|
|
123
123
|
[
|
|
124
124
|
f"""set -e;
|
|
125
125
|
export PATH=$PATH:$(pwd)/micromamba;
|
|
126
|
-
micromamba run --prefix {prefix} pip --disable-pip-version-check install --root-user-action=ignore --no-compile {pypi_pkgs_dir}/*.whl"""
|
|
126
|
+
micromamba run --prefix {prefix} python -m pip --disable-pip-version-check install --root-user-action=ignore --no-compile {pypi_pkgs_dir}/*.whl --no-user"""
|
|
127
127
|
]
|
|
128
128
|
)
|
|
129
129
|
|
metaflow/runner/click_api.py
CHANGED
|
@@ -32,9 +32,10 @@ from metaflow._vendor.click.types import (
|
|
|
32
32
|
UUIDParameterType,
|
|
33
33
|
)
|
|
34
34
|
from metaflow._vendor.typeguard import TypeCheckError, check_type
|
|
35
|
-
from metaflow.
|
|
35
|
+
from metaflow.decorators import add_decorator_options
|
|
36
|
+
from metaflow.exception import MetaflowException
|
|
36
37
|
from metaflow.includefile import FilePathClass
|
|
37
|
-
from metaflow.parameters import JSONTypeClass
|
|
38
|
+
from metaflow.parameters import JSONTypeClass, flow_context
|
|
38
39
|
|
|
39
40
|
click_to_python_types = {
|
|
40
41
|
StringParamType: str,
|
|
@@ -140,7 +141,7 @@ def get_inspect_param_obj(p: Union[click.Argument, click.Option], kind: str):
|
|
|
140
141
|
loaded_modules = {}
|
|
141
142
|
|
|
142
143
|
|
|
143
|
-
def
|
|
144
|
+
def extract_flow_class_from_file(flow_file: str) -> FlowSpec:
|
|
144
145
|
# Check if the module has already been loaded
|
|
145
146
|
if flow_file in loaded_modules:
|
|
146
147
|
module = loaded_modules[flow_file]
|
|
@@ -153,14 +154,16 @@ def extract_flowspec_params(flow_file: str) -> List[Parameter]:
|
|
|
153
154
|
loaded_modules[flow_file] = module
|
|
154
155
|
classes = inspect.getmembers(module, inspect.isclass)
|
|
155
156
|
|
|
156
|
-
|
|
157
|
+
flow_cls = None
|
|
157
158
|
for _, kls in classes:
|
|
158
159
|
if kls != FlowSpec and issubclass(kls, FlowSpec):
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
160
|
+
if flow_cls is not None:
|
|
161
|
+
raise MetaflowException(
|
|
162
|
+
"Multiple FlowSpec classes found in %s" % flow_file
|
|
163
|
+
)
|
|
164
|
+
flow_cls = kls
|
|
162
165
|
|
|
163
|
-
return
|
|
166
|
+
return flow_cls
|
|
164
167
|
|
|
165
168
|
|
|
166
169
|
class MetaflowAPI(object):
|
|
@@ -180,7 +183,11 @@ class MetaflowAPI(object):
|
|
|
180
183
|
|
|
181
184
|
@classmethod
|
|
182
185
|
def from_cli(cls, flow_file: str, cli_collection: Callable) -> Callable:
|
|
183
|
-
|
|
186
|
+
flow_cls = extract_flow_class_from_file(flow_file)
|
|
187
|
+
flow_parameters = [p for _, p in flow_cls._get_parameters()]
|
|
188
|
+
with flow_context(flow_cls) as _:
|
|
189
|
+
add_decorator_options(cli_collection)
|
|
190
|
+
|
|
184
191
|
class_dict = {"__module__": "metaflow", "_API_NAME": flow_file}
|
|
185
192
|
command_groups = cli_collection.sources
|
|
186
193
|
for each_group in command_groups:
|
|
@@ -377,6 +384,8 @@ def extract_command(
|
|
|
377
384
|
|
|
378
385
|
|
|
379
386
|
if __name__ == "__main__":
|
|
387
|
+
from metaflow.cli import start
|
|
388
|
+
|
|
380
389
|
api = MetaflowAPI.from_cli("../try.py", start)
|
|
381
390
|
|
|
382
391
|
command = api(metadata="local").run(
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import importlib
|
|
1
2
|
import os
|
|
2
3
|
import sys
|
|
3
4
|
import tempfile
|
|
@@ -250,6 +251,9 @@ class Runner(object):
|
|
|
250
251
|
# from metaflow import Runner
|
|
251
252
|
# This ability is made possible by the statement:
|
|
252
253
|
# 'from .metaflow_runner import Runner' in '__init__.py'
|
|
254
|
+
|
|
255
|
+
if "metaflow.cli" in sys.modules:
|
|
256
|
+
importlib.reload(sys.modules["metaflow.cli"])
|
|
253
257
|
from metaflow.cli import start
|
|
254
258
|
from metaflow.runner.click_api import MetaflowAPI
|
|
255
259
|
|
metaflow/runtime.py
CHANGED
|
@@ -157,7 +157,6 @@ class NativeRuntime(object):
|
|
|
157
157
|
deco.runtime_init(flow, graph, package, self._run_id)
|
|
158
158
|
|
|
159
159
|
def _new_task(self, step, input_paths=None, **kwargs):
|
|
160
|
-
|
|
161
160
|
if input_paths is None:
|
|
162
161
|
may_clone = True
|
|
163
162
|
else:
|
|
@@ -315,7 +314,6 @@ class NativeRuntime(object):
|
|
|
315
314
|
# main scheduling loop
|
|
316
315
|
exception = None
|
|
317
316
|
while self._run_queue or self._active_tasks[0] > 0:
|
|
318
|
-
|
|
319
317
|
# 1. are any of the current workers finished?
|
|
320
318
|
finished_tasks = list(self._poll_workers())
|
|
321
319
|
# 2. push new tasks triggered by the finished tasks to the queue
|
|
@@ -583,7 +581,6 @@ class NativeRuntime(object):
|
|
|
583
581
|
)
|
|
584
582
|
|
|
585
583
|
def _queue_task_foreach(self, task, next_steps):
|
|
586
|
-
|
|
587
584
|
# CHECK: this condition should be enforced by the linter but
|
|
588
585
|
# let's assert that the assumption holds
|
|
589
586
|
if len(next_steps) > 1:
|
|
@@ -798,8 +795,8 @@ class Task(object):
|
|
|
798
795
|
task_id=None,
|
|
799
796
|
resume_identifier=None,
|
|
800
797
|
):
|
|
801
|
-
|
|
802
798
|
self.step = step
|
|
799
|
+
self.flow = flow
|
|
803
800
|
self.flow_name = flow.name
|
|
804
801
|
self.run_id = run_id
|
|
805
802
|
self.task_id = None
|
|
@@ -935,7 +932,6 @@ class Task(object):
|
|
|
935
932
|
# yet written the _resume_leader metadata, we will wait for a few seconds.
|
|
936
933
|
# We will wait for resume leader for at most 3 times.
|
|
937
934
|
for resume_leader_wait_retry in range(3):
|
|
938
|
-
|
|
939
935
|
if ds.has_metadata("_resume_leader", add_attempt=False):
|
|
940
936
|
resume_leader = ds.load_metadata(
|
|
941
937
|
["_resume_leader"], add_attempt=False
|
|
@@ -1323,7 +1319,7 @@ class CLIArgs(object):
|
|
|
1323
1319
|
# FlowDecorators can define their own top-level options. They are
|
|
1324
1320
|
# responsible for adding their own top-level options and values through
|
|
1325
1321
|
# the get_top_level_options() hook.
|
|
1326
|
-
for deco in flow_decorators():
|
|
1322
|
+
for deco in flow_decorators(self.task.flow):
|
|
1327
1323
|
self.top_level_options.update(deco.get_top_level_options())
|
|
1328
1324
|
|
|
1329
1325
|
self.commands = ["step"]
|
|
@@ -1342,11 +1338,9 @@ class CLIArgs(object):
|
|
|
1342
1338
|
self.env = {}
|
|
1343
1339
|
|
|
1344
1340
|
def get_args(self):
|
|
1345
|
-
|
|
1346
1341
|
# TODO: Make one with dict_to_cli_options; see cli_args.py for more detail
|
|
1347
1342
|
def _options(mapping):
|
|
1348
1343
|
for k, v in mapping.items():
|
|
1349
|
-
|
|
1350
1344
|
# None or False arguments are ignored
|
|
1351
1345
|
# v needs to be explicitly False, not falsy, e.g. 0 is an acceptable value
|
|
1352
1346
|
if v is None or v is False:
|
metaflow/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
metaflow_version = "2.12.
|
|
1
|
+
metaflow_version = "2.12.4.1"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: ob-metaflow
|
|
3
|
-
Version: 2.12.
|
|
3
|
+
Version: 2.12.4.1
|
|
4
4
|
Summary: Metaflow: More Data Science, Less Engineering
|
|
5
5
|
Author: Netflix, Outerbounds & the Metaflow Community
|
|
6
6
|
Author-email: help@outerbounds.co
|
|
@@ -12,7 +12,7 @@ Requires-Dist: boto3
|
|
|
12
12
|
Requires-Dist: pylint
|
|
13
13
|
Requires-Dist: kubernetes
|
|
14
14
|
Provides-Extra: stubs
|
|
15
|
-
Requires-Dist: metaflow-stubs ==2.12.
|
|
15
|
+
Requires-Dist: metaflow-stubs ==2.12.4.1 ; extra == 'stubs'
|
|
16
16
|
|
|
17
17
|

|
|
18
18
|
|
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
metaflow/R.py,sha256=
|
|
1
|
+
metaflow/R.py,sha256=CqVfIatvmjciuICNnoyyNGrwE7Va9iXfLdFbQa52hwA,3958
|
|
2
2
|
metaflow/__init__.py,sha256=TsG2wQWHpg5G9Sl7Iowiovw2wZxbR1iBRi7hXCfleCw,6254
|
|
3
3
|
metaflow/cards.py,sha256=tP1_RrtmqdFh741pqE4t98S7SA0MtGRlGvRICRZF1Mg,426
|
|
4
|
-
metaflow/cli.py,sha256=
|
|
4
|
+
metaflow/cli.py,sha256=4NTGG4_UCd53HhscDVMQGl4VcV8O6C33iFKfTc-n2tg,33736
|
|
5
5
|
metaflow/cli_args.py,sha256=lcgBGNTvfaiPxiUnejAe60Upt9swG6lRy1_3OqbU6MY,2616
|
|
6
6
|
metaflow/clone_util.py,sha256=XfUX0vssu_hPlyZfhFl1AOnKkLqvt33Qp8xNrmdocGg,2057
|
|
7
7
|
metaflow/cmd_with_io.py,sha256=kl53HkAIyv0ecpItv08wZYczv7u3msD1VCcciqigqf0,588
|
|
8
8
|
metaflow/debug.py,sha256=HEmt_16tJtqHXQXsqD9pqOFe3CWR5GZ7VwpaYQgnRdU,1466
|
|
9
|
-
metaflow/decorators.py,sha256=
|
|
9
|
+
metaflow/decorators.py,sha256=ESe0mDsyTKsfZw2Ovn83rU-GsNUCM-ZYCljGeC8ChnY,21548
|
|
10
10
|
metaflow/event_logger.py,sha256=joTVRqZPL87nvah4ZOwtqWX8NeraM_CXKXXGVpKGD8o,780
|
|
11
11
|
metaflow/events.py,sha256=ahjzkSbSnRCK9RZ-9vTfUviz_6gMvSO9DGkJ86X80-k,5300
|
|
12
12
|
metaflow/exception.py,sha256=KC1LHJQzzYkWib0DeQ4l_A2r8VaudywsSqIQuq1RDZU,4954
|
|
13
|
-
metaflow/flowspec.py,sha256=
|
|
13
|
+
metaflow/flowspec.py,sha256=_s7Kgm3l72GNUseBsb6S43YLK7U4Ju5dmw8CBRXVisQ,27021
|
|
14
14
|
metaflow/graph.py,sha256=ZPxyG8uwVMk5YYgX4pQEQaPZtZM5Wy-G4NtJK73IEuA,11818
|
|
15
15
|
metaflow/includefile.py,sha256=yHczcZ_U0SrasxSNhZb3DIBzx8UZnrJCl3FzvpEQLOA,19753
|
|
16
16
|
metaflow/integrations.py,sha256=LlsaoePRg03DjENnmLxZDYto3NwWc9z_PtU6nJxLldg,1480
|
|
17
17
|
metaflow/lint.py,sha256=_kYAbAtsP7IG1Rd0FqNbo8I8Zs66_0WXbaZJFARO3dE,10394
|
|
18
|
-
metaflow/metaflow_config.py,sha256=
|
|
18
|
+
metaflow/metaflow_config.py,sha256=mEVBlA0bzUXk0O6744zsp73VPr2JfoVO0SLlYB_faHM,22507
|
|
19
19
|
metaflow/metaflow_config_funcs.py,sha256=pCaiQ2ez9wXixJI3ehmf3QiW9lUqFrZnBZx1my_0wIg,4874
|
|
20
20
|
metaflow/metaflow_current.py,sha256=sCENPBiji3LcPbwgOG0ukGd_yEc5tST8EowES8DzRtA,7430
|
|
21
21
|
metaflow/metaflow_environment.py,sha256=HJfhI3GrU-YbY7Etu9M-1q7EbwtEGFzCBhrUs45OLXY,7403
|
|
@@ -24,18 +24,18 @@ metaflow/metaflow_version.py,sha256=mPQ6g_3XjNdi0NrxDzwlW8ZH0nMyYpwqmJ04P7TIdP0,
|
|
|
24
24
|
metaflow/monitor.py,sha256=T0NMaBPvXynlJAO_avKtk8OIIRMyEuMAyF8bIp79aZU,5323
|
|
25
25
|
metaflow/multicore_utils.py,sha256=vdTNgczVLODifscUbbveJbuSDOl3Y9pAxhr7sqYiNf4,4760
|
|
26
26
|
metaflow/package.py,sha256=sOvRpnvqVaQa6eR8lwcfb5HYCGqmpYFPm-cLgOEdl04,7377
|
|
27
|
-
metaflow/parameters.py,sha256=
|
|
27
|
+
metaflow/parameters.py,sha256=e4uVGrVMZXmfn070P-zRmVeax2OjV4dfnR6iyaE-wt4,15712
|
|
28
28
|
metaflow/procpoll.py,sha256=22ppTUyaTYVn1UUG4RNG1LnCKBwRbaTmhYiYN_7OVN8,2861
|
|
29
29
|
metaflow/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
30
30
|
metaflow/pylint_wrapper.py,sha256=zzBY9YaSUZOGH-ypDKAv2B_7XcoyMZj-zCoCrmYqNRc,2865
|
|
31
|
-
metaflow/runtime.py,sha256=
|
|
31
|
+
metaflow/runtime.py,sha256=KQbLI4zH9V6L6YwOgYz28CPMOT_akoCWEBb49avSQvo,63993
|
|
32
32
|
metaflow/tagging_util.py,sha256=ctyf0Q1gBi0RyZX6J0e9DQGNkNHblV_CITfy66axXB4,2346
|
|
33
33
|
metaflow/task.py,sha256=tb5LNyx4u-yAQTM5MK1-wMw62dLfBeR9ojIZuYHeb0c,25592
|
|
34
34
|
metaflow/tuple_util.py,sha256=_G5YIEhuugwJ_f6rrZoelMFak3DqAR2tt_5CapS1XTY,830
|
|
35
35
|
metaflow/unbounded_foreach.py,sha256=p184WMbrMJ3xKYHwewj27ZhRUsSj_kw1jlye5gA9xJk,387
|
|
36
36
|
metaflow/util.py,sha256=m5womQ7y-jXehuMyHPfByDbZ4HwTJxzs869cPOlMR8s,13057
|
|
37
37
|
metaflow/vendor.py,sha256=FchtA9tH22JM-eEtJ2c9FpUdMn8sSb1VHuQS56EcdZk,5139
|
|
38
|
-
metaflow/version.py,sha256=
|
|
38
|
+
metaflow/version.py,sha256=CoH1Xzc0kdTQCdiCC8LGtk-agf4UVZpgPffJ0MRQ120,30
|
|
39
39
|
metaflow/_vendor/__init__.py,sha256=y_CiwUD3l4eAKvTVDZeqgVujMy31cAM1qjAB-HfI-9s,353
|
|
40
40
|
metaflow/_vendor/typing_extensions.py,sha256=0nUs5p1A_UrZigrAVBoOEM6TxU37zzPDUtiij1ZwpNc,110417
|
|
41
41
|
metaflow/_vendor/zipp.py,sha256=ajztOH-9I7KA_4wqDYygtHa6xUBVZgFpmZ8FE74HHHI,8425
|
|
@@ -110,7 +110,7 @@ metaflow/_vendor/v3_6/importlib_metadata/_meta.py,sha256=_F48Hu_jFxkfKWz5wcYS8vO
|
|
|
110
110
|
metaflow/_vendor/v3_6/importlib_metadata/_text.py,sha256=HCsFksZpJLeTP3NEk_ngrAeXVRRtTrtyh9eOABoRP4A,2166
|
|
111
111
|
metaflow/_vendor/v3_6/importlib_metadata/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
112
112
|
metaflow/client/__init__.py,sha256=1GtQB4Y_CBkzaxg32L1syNQSlfj762wmLrfrDxGi1b8,226
|
|
113
|
-
metaflow/client/core.py,sha256=
|
|
113
|
+
metaflow/client/core.py,sha256=XjYNMytrm7pJmcaV2c2lu4NTFAnDRRqFO7WfcA04R4s,73721
|
|
114
114
|
metaflow/client/filecache.py,sha256=QdD1sW6w4Nnza-ioz4I1fEZI843X33AFIV3eSxq-cuU,14868
|
|
115
115
|
metaflow/cmd/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
116
116
|
metaflow/cmd/configure_cmd.py,sha256=o-DKnUf2FBo_HiMVyoyzQaGBSMtpbEPEdFTQZ0hkU-k,33396
|
|
@@ -159,7 +159,7 @@ metaflow/plugins/tag_cli.py,sha256=O_ZI4ILwGX3xKrLewUUF-zdJjCDi3JmsTb4ow87_RuY,1
|
|
|
159
159
|
metaflow/plugins/test_unbounded_foreach_decorator.py,sha256=cB_2OWb38eYfmbVck72ZwU0qgzi6hqJXZAxglpHU_qg,5216
|
|
160
160
|
metaflow/plugins/timeout_decorator.py,sha256=R-X8rKeMqd-xhfJFqskWb6ZpmZt2JB14U1BZJSRriwM,3648
|
|
161
161
|
metaflow/plugins/airflow/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
162
|
-
metaflow/plugins/airflow/airflow.py,sha256=
|
|
162
|
+
metaflow/plugins/airflow/airflow.py,sha256=n6NUVR3HMTkX6G3H246otXlSSeLhtmcPZp9ei2SIBeg,32156
|
|
163
163
|
metaflow/plugins/airflow/airflow_cli.py,sha256=fUi6IsRMi6mvL6Twrszk7rZq7_4PmdYr9evJnBpXXPc,14440
|
|
164
164
|
metaflow/plugins/airflow/airflow_decorator.py,sha256=H9-QnRP4x8tSomLmmpGeuVUI48-CxHR7tlvn_ceX1Zs,1772
|
|
165
165
|
metaflow/plugins/airflow/airflow_utils.py,sha256=qd6lV2X4VpCO2sLsRc35JMOU4DVz_tQacrM_wWNkQug,28865
|
|
@@ -168,13 +168,13 @@ metaflow/plugins/airflow/exception.py,sha256=JrsW0s7zrtiDVvOAqsGB75qMoRSNehPQmcu
|
|
|
168
168
|
metaflow/plugins/airflow/plumbing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
169
169
|
metaflow/plugins/airflow/plumbing/set_parameters.py,sha256=u8pAtHrUYW1LgD6CJertSEArMm8eBzzKX8g39INP0YI,645
|
|
170
170
|
metaflow/plugins/airflow/sensors/__init__.py,sha256=50-7HmV7GA6awpU2GbQ0hzCAR6iNK_UyPq-4vg9FDFg,190
|
|
171
|
-
metaflow/plugins/airflow/sensors/base_sensor.py,sha256=
|
|
172
|
-
metaflow/plugins/airflow/sensors/external_task_sensor.py,sha256=
|
|
173
|
-
metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=
|
|
171
|
+
metaflow/plugins/airflow/sensors/base_sensor.py,sha256=s-OQBfPWZ_T3wn96Ua59CCEj1rSz1r0Ar1JnoqD4YV0,2383
|
|
172
|
+
metaflow/plugins/airflow/sensors/external_task_sensor.py,sha256=zhYlrZnXT20KW8-fVk0fCNtTyNiKJB5PMVASacu30r0,6034
|
|
173
|
+
metaflow/plugins/airflow/sensors/s3_sensor.py,sha256=iDReG-7FKnumrtQg-HY6cCUAAqNA90nARrjjjEEk_x4,3275
|
|
174
174
|
metaflow/plugins/argo/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
175
175
|
metaflow/plugins/argo/argo_client.py,sha256=MKKhMCbWOPzf6z5zQQiyDRHHkAXcO7ipboDZDqAAvOk,15849
|
|
176
176
|
metaflow/plugins/argo/argo_events.py,sha256=_C1KWztVqgi3zuH57pInaE9OzABc2NnncC-zdwOMZ-w,5909
|
|
177
|
-
metaflow/plugins/argo/argo_workflows.py,sha256=
|
|
177
|
+
metaflow/plugins/argo/argo_workflows.py,sha256=OAbj6S3uuyS-od_Fstrt2eif-8M2mHhPRgbQjCVRXho,130335
|
|
178
178
|
metaflow/plugins/argo/argo_workflows_cli.py,sha256=sZTpgfmc50eT3e0qIxpVqUgWhTcYlO1HM4gU6Oaya8g,33259
|
|
179
179
|
metaflow/plugins/argo/argo_workflows_decorator.py,sha256=K5t4uIk2IXPdK7v7DEjj3buSB8ikLjLycKjbZUYeiaw,6781
|
|
180
180
|
metaflow/plugins/argo/generate_input_paths.py,sha256=loYsI6RFX9LlFsHb7Fe-mzlTTtRdySoOu7sYDy-uXK0,881
|
|
@@ -194,7 +194,7 @@ metaflow/plugins/aws/step_functions/event_bridge_client.py,sha256=U9-tqKdih4KR-Z
|
|
|
194
194
|
metaflow/plugins/aws/step_functions/production_token.py,sha256=_o4emv3rozYZoWpaj1Y6UfKhTMlYpQc7GDDDBfZ2G7s,1898
|
|
195
195
|
metaflow/plugins/aws/step_functions/schedule_decorator.py,sha256=Ab1rW8O_no4HNZm4__iBmFDCDW0Z8-TgK4lnxHHA6HI,1940
|
|
196
196
|
metaflow/plugins/aws/step_functions/set_batch_environment.py,sha256=ibiGWFHDjKcLfprH3OsX-g2M9lUsh6J-bp7v2cdLhD4,1294
|
|
197
|
-
metaflow/plugins/aws/step_functions/step_functions.py,sha256=
|
|
197
|
+
metaflow/plugins/aws/step_functions/step_functions.py,sha256=S51wz_75n5Um8qZ-ICactX3-Fnpv0qb-ORLa-prBrB8,51853
|
|
198
198
|
metaflow/plugins/aws/step_functions/step_functions_cli.py,sha256=KlH9jJL0VfsT0JqBhLwaWdYjaccU8UEArKAFnIJbSoU,24426
|
|
199
199
|
metaflow/plugins/aws/step_functions/step_functions_client.py,sha256=DKpNwAIWElvWjFANs5Ku3rgzjxFoqAD6k-EF8Xhkg3Q,4754
|
|
200
200
|
metaflow/plugins/aws/step_functions/step_functions_decorator.py,sha256=9hw_MX36RyFp6IowuAYaJzJg9UC5KCe1FNt1PcG7_J0,3791
|
|
@@ -252,7 +252,7 @@ metaflow/plugins/env_escape/data_transferer.py,sha256=wm1Aqf0rTWaq8JgqpiRN0g3N3h
|
|
|
252
252
|
metaflow/plugins/env_escape/exception_transferer.py,sha256=rpcpwDYRf5XqsZvShyQe5eRQ9uBGNdqmyAeB1h-Lofw,5746
|
|
253
253
|
metaflow/plugins/env_escape/override_decorators.py,sha256=JC_YcNIFUfLq6z2zEAVUT-PsaNxKxCqr0JbLTo1cCpo,3605
|
|
254
254
|
metaflow/plugins/env_escape/server.py,sha256=V5aG12xlS5rybkyKH_Uc0zIM615UHeQoyQKr2l2tF3g,21870
|
|
255
|
-
metaflow/plugins/env_escape/stub.py,sha256=
|
|
255
|
+
metaflow/plugins/env_escape/stub.py,sha256=4DSlcbGa62hPylCaAhgt_7z_hqGYXjprh70fbthnUkA,17250
|
|
256
256
|
metaflow/plugins/env_escape/utils.py,sha256=ThwneuINFoNiaqzjhHsVWSPGr-1UtskgcSpw58UARCU,1243
|
|
257
257
|
metaflow/plugins/env_escape/communication/__init__.py,sha256=Ff5AB88gOAvBzN2pp_2YNiD0PhUIt2SFE8nyOAKnxXg,38
|
|
258
258
|
metaflow/plugins/env_escape/communication/bytestream.py,sha256=weQBm-c6yPlGv1TAmQbYanqvQ0IRDh7x_6hZPvWh_Uw,1866
|
|
@@ -284,7 +284,7 @@ metaflow/plugins/metadata/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-
|
|
|
284
284
|
metaflow/plugins/metadata/local.py,sha256=YhLJC5zjVJrvQFIyQ92ZBByiUmhCC762RUX7ITX12O8,22428
|
|
285
285
|
metaflow/plugins/metadata/service.py,sha256=ihq5F7KQZlxvYwzH_-jyP2aWN_I96i2vp92j_d697s8,20204
|
|
286
286
|
metaflow/plugins/pypi/__init__.py,sha256=0YFZpXvX7HCkyBFglatual7XGifdA1RwC3U4kcizyak,1037
|
|
287
|
-
metaflow/plugins/pypi/bootstrap.py,sha256=
|
|
287
|
+
metaflow/plugins/pypi/bootstrap.py,sha256=Hik3PZ_RQC8T6hEf-NE2Xr_jq2ZIUkpgUtJlx-rqJgU,5107
|
|
288
288
|
metaflow/plugins/pypi/conda_decorator.py,sha256=-bPxNtZKjxqOo4sj89uIp8ZVrCIontWhAp7wwRFjYpg,14189
|
|
289
289
|
metaflow/plugins/pypi/conda_environment.py,sha256=wruUTU-xyFaEm331_7DcCkRuveAPS-2llZW4TZfGG4M,19236
|
|
290
290
|
metaflow/plugins/pypi/micromamba.py,sha256=wlVN2fm4WXFh3jVNtpDfu4XEz6VJKbmFNp0QvqlMIuI,12179
|
|
@@ -296,8 +296,8 @@ metaflow/plugins/secrets/__init__.py,sha256=mhJaN2eMS_ZZVewAMR2E-JdP5i0t3v9e6Dcw
|
|
|
296
296
|
metaflow/plugins/secrets/inline_secrets_provider.py,sha256=EChmoBGA1i7qM3jtYwPpLZDBybXLergiDlN63E0u3x8,294
|
|
297
297
|
metaflow/plugins/secrets/secrets_decorator.py,sha256=Y41XCOEGt7asxx6FEOBdKMqYFlU_jcA5u7erHN8DqJM,10514
|
|
298
298
|
metaflow/runner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
299
|
-
metaflow/runner/click_api.py,sha256=
|
|
300
|
-
metaflow/runner/metaflow_runner.py,sha256=
|
|
299
|
+
metaflow/runner/click_api.py,sha256=vrrIb5DtVYhCW_hB7wPgj6Q0o-h1bBSWiYnKTOyC454,13452
|
|
300
|
+
metaflow/runner/metaflow_runner.py,sha256=A-ZTdO2SEiPWAtce7Rc4zoXf7VZFj9f0oW50KFY3z5Y,15658
|
|
301
301
|
metaflow/runner/nbrun.py,sha256=MgkPREcq8Qvb8a2F1ExwIPfDRHW23ixSC1-3cxGiFUc,7551
|
|
302
302
|
metaflow/runner/subprocess_manager.py,sha256=0knxWZYJx8srMv6wTPYKOC6tn4-airnyI7Vbqfb3iXY,19567
|
|
303
303
|
metaflow/sidecar/__init__.py,sha256=1mmNpmQ5puZCpRmmYlCOeieZ4108Su9XQ4_EqF1FGOU,131
|
|
@@ -332,9 +332,9 @@ metaflow/tutorials/07-worldview/README.md,sha256=5vQTrFqulJ7rWN6r20dhot9lI2sVj9W
|
|
|
332
332
|
metaflow/tutorials/07-worldview/worldview.ipynb,sha256=ztPZPI9BXxvW1QdS2Tfe7LBuVzvFvv0AToDnsDJhLdE,2237
|
|
333
333
|
metaflow/tutorials/08-autopilot/README.md,sha256=GnePFp_q76jPs991lMUqfIIh5zSorIeWznyiUxzeUVE,1039
|
|
334
334
|
metaflow/tutorials/08-autopilot/autopilot.ipynb,sha256=DQoJlILV7Mq9vfPBGW-QV_kNhWPjS5n6SJLqePjFYLY,3191
|
|
335
|
-
ob_metaflow-2.12.
|
|
336
|
-
ob_metaflow-2.12.
|
|
337
|
-
ob_metaflow-2.12.
|
|
338
|
-
ob_metaflow-2.12.
|
|
339
|
-
ob_metaflow-2.12.
|
|
340
|
-
ob_metaflow-2.12.
|
|
335
|
+
ob_metaflow-2.12.4.1.dist-info/LICENSE,sha256=nl_Lt5v9VvJ-5lWJDT4ddKAG-VZ-2IaLmbzpgYDz2hU,11343
|
|
336
|
+
ob_metaflow-2.12.4.1.dist-info/METADATA,sha256=cuItIzueh-1UHr2rqGSpzrrj2VaSpbDl_qQJEUANgtw,5143
|
|
337
|
+
ob_metaflow-2.12.4.1.dist-info/WHEEL,sha256=DZajD4pwLWue70CAfc7YaxT1wLUciNBvN_TTcvXpltE,110
|
|
338
|
+
ob_metaflow-2.12.4.1.dist-info/entry_points.txt,sha256=IKwTN1T3I5eJL3uo_vnkyxVffcgnRdFbKwlghZfn27k,57
|
|
339
|
+
ob_metaflow-2.12.4.1.dist-info/top_level.txt,sha256=v1pDHoWaSaKeuc5fKTRSfsXCKSdW1zvNVmvA-i0if3o,9
|
|
340
|
+
ob_metaflow-2.12.4.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|