ob-metaflow 2.15.13.1__py2.py3-none-any.whl → 2.19.7.1rc0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow/__init__.py +10 -3
- metaflow/_vendor/imghdr/__init__.py +186 -0
- metaflow/_vendor/yaml/__init__.py +427 -0
- metaflow/_vendor/yaml/composer.py +139 -0
- metaflow/_vendor/yaml/constructor.py +748 -0
- metaflow/_vendor/yaml/cyaml.py +101 -0
- metaflow/_vendor/yaml/dumper.py +62 -0
- metaflow/_vendor/yaml/emitter.py +1137 -0
- metaflow/_vendor/yaml/error.py +75 -0
- metaflow/_vendor/yaml/events.py +86 -0
- metaflow/_vendor/yaml/loader.py +63 -0
- metaflow/_vendor/yaml/nodes.py +49 -0
- metaflow/_vendor/yaml/parser.py +589 -0
- metaflow/_vendor/yaml/reader.py +185 -0
- metaflow/_vendor/yaml/representer.py +389 -0
- metaflow/_vendor/yaml/resolver.py +227 -0
- metaflow/_vendor/yaml/scanner.py +1435 -0
- metaflow/_vendor/yaml/serializer.py +111 -0
- metaflow/_vendor/yaml/tokens.py +104 -0
- metaflow/cards.py +4 -0
- metaflow/cli.py +125 -21
- metaflow/cli_components/init_cmd.py +1 -0
- metaflow/cli_components/run_cmds.py +204 -40
- metaflow/cli_components/step_cmd.py +160 -4
- metaflow/client/__init__.py +1 -0
- metaflow/client/core.py +198 -130
- metaflow/client/filecache.py +59 -32
- metaflow/cmd/code/__init__.py +2 -1
- metaflow/cmd/develop/stub_generator.py +49 -18
- metaflow/cmd/develop/stubs.py +9 -27
- metaflow/cmd/make_wrapper.py +30 -0
- metaflow/datastore/__init__.py +1 -0
- metaflow/datastore/content_addressed_store.py +40 -9
- metaflow/datastore/datastore_set.py +10 -1
- metaflow/datastore/flow_datastore.py +124 -4
- metaflow/datastore/spin_datastore.py +91 -0
- metaflow/datastore/task_datastore.py +92 -6
- metaflow/debug.py +5 -0
- metaflow/decorators.py +331 -82
- metaflow/extension_support/__init__.py +414 -356
- metaflow/extension_support/_empty_file.py +2 -2
- metaflow/flowspec.py +322 -82
- metaflow/graph.py +178 -15
- metaflow/includefile.py +25 -3
- metaflow/lint.py +94 -3
- metaflow/meta_files.py +13 -0
- metaflow/metadata_provider/metadata.py +13 -2
- metaflow/metaflow_config.py +66 -4
- metaflow/metaflow_environment.py +91 -25
- metaflow/metaflow_profile.py +18 -0
- metaflow/metaflow_version.py +16 -1
- metaflow/package/__init__.py +673 -0
- metaflow/packaging_sys/__init__.py +880 -0
- metaflow/packaging_sys/backend.py +128 -0
- metaflow/packaging_sys/distribution_support.py +153 -0
- metaflow/packaging_sys/tar_backend.py +99 -0
- metaflow/packaging_sys/utils.py +54 -0
- metaflow/packaging_sys/v1.py +527 -0
- metaflow/parameters.py +6 -2
- metaflow/plugins/__init__.py +6 -0
- metaflow/plugins/airflow/airflow.py +11 -1
- metaflow/plugins/airflow/airflow_cli.py +16 -5
- metaflow/plugins/argo/argo_client.py +42 -20
- metaflow/plugins/argo/argo_events.py +6 -6
- metaflow/plugins/argo/argo_workflows.py +1023 -344
- metaflow/plugins/argo/argo_workflows_cli.py +396 -94
- metaflow/plugins/argo/argo_workflows_decorator.py +9 -0
- metaflow/plugins/argo/argo_workflows_deployer_objects.py +75 -49
- metaflow/plugins/argo/capture_error.py +5 -2
- metaflow/plugins/argo/conditional_input_paths.py +35 -0
- metaflow/plugins/argo/exit_hooks.py +209 -0
- metaflow/plugins/argo/param_val.py +19 -0
- metaflow/plugins/aws/aws_client.py +6 -0
- metaflow/plugins/aws/aws_utils.py +33 -1
- metaflow/plugins/aws/batch/batch.py +72 -5
- metaflow/plugins/aws/batch/batch_cli.py +24 -3
- metaflow/plugins/aws/batch/batch_decorator.py +57 -6
- metaflow/plugins/aws/step_functions/step_functions.py +28 -3
- metaflow/plugins/aws/step_functions/step_functions_cli.py +49 -4
- metaflow/plugins/aws/step_functions/step_functions_deployer.py +3 -0
- metaflow/plugins/aws/step_functions/step_functions_deployer_objects.py +30 -0
- metaflow/plugins/cards/card_cli.py +20 -1
- metaflow/plugins/cards/card_creator.py +24 -1
- metaflow/plugins/cards/card_datastore.py +21 -49
- metaflow/plugins/cards/card_decorator.py +58 -6
- metaflow/plugins/cards/card_modules/basic.py +38 -9
- metaflow/plugins/cards/card_modules/bundle.css +1 -1
- metaflow/plugins/cards/card_modules/chevron/renderer.py +1 -1
- metaflow/plugins/cards/card_modules/components.py +592 -3
- metaflow/plugins/cards/card_modules/convert_to_native_type.py +34 -5
- metaflow/plugins/cards/card_modules/json_viewer.py +232 -0
- metaflow/plugins/cards/card_modules/main.css +1 -0
- metaflow/plugins/cards/card_modules/main.js +56 -41
- metaflow/plugins/cards/card_modules/test_cards.py +22 -6
- metaflow/plugins/cards/component_serializer.py +1 -8
- metaflow/plugins/cards/metadata.py +22 -0
- metaflow/plugins/catch_decorator.py +9 -0
- metaflow/plugins/datastores/local_storage.py +12 -6
- metaflow/plugins/datastores/spin_storage.py +12 -0
- metaflow/plugins/datatools/s3/s3.py +49 -17
- metaflow/plugins/datatools/s3/s3op.py +113 -66
- metaflow/plugins/env_escape/client_modules.py +102 -72
- metaflow/plugins/events_decorator.py +127 -121
- metaflow/plugins/exit_hook/__init__.py +0 -0
- metaflow/plugins/exit_hook/exit_hook_decorator.py +46 -0
- metaflow/plugins/exit_hook/exit_hook_script.py +52 -0
- metaflow/plugins/kubernetes/kubernetes.py +12 -1
- metaflow/plugins/kubernetes/kubernetes_cli.py +11 -0
- metaflow/plugins/kubernetes/kubernetes_decorator.py +25 -6
- metaflow/plugins/kubernetes/kubernetes_job.py +12 -4
- metaflow/plugins/kubernetes/kubernetes_jobsets.py +31 -30
- metaflow/plugins/metadata_providers/local.py +76 -82
- metaflow/plugins/metadata_providers/service.py +13 -9
- metaflow/plugins/metadata_providers/spin.py +16 -0
- metaflow/plugins/package_cli.py +36 -24
- metaflow/plugins/parallel_decorator.py +11 -2
- metaflow/plugins/parsers.py +16 -0
- metaflow/plugins/pypi/bootstrap.py +7 -1
- metaflow/plugins/pypi/conda_decorator.py +41 -82
- metaflow/plugins/pypi/conda_environment.py +14 -6
- metaflow/plugins/pypi/micromamba.py +9 -1
- metaflow/plugins/pypi/pip.py +41 -5
- metaflow/plugins/pypi/pypi_decorator.py +4 -4
- metaflow/plugins/pypi/utils.py +22 -0
- metaflow/plugins/secrets/__init__.py +3 -0
- metaflow/plugins/secrets/secrets_decorator.py +14 -178
- metaflow/plugins/secrets/secrets_func.py +49 -0
- metaflow/plugins/secrets/secrets_spec.py +101 -0
- metaflow/plugins/secrets/utils.py +74 -0
- metaflow/plugins/test_unbounded_foreach_decorator.py +2 -2
- metaflow/plugins/timeout_decorator.py +0 -1
- metaflow/plugins/uv/bootstrap.py +29 -1
- metaflow/plugins/uv/uv_environment.py +5 -3
- metaflow/pylint_wrapper.py +5 -1
- metaflow/runner/click_api.py +79 -26
- metaflow/runner/deployer.py +208 -6
- metaflow/runner/deployer_impl.py +32 -12
- metaflow/runner/metaflow_runner.py +266 -33
- metaflow/runner/subprocess_manager.py +21 -1
- metaflow/runner/utils.py +27 -16
- metaflow/runtime.py +660 -66
- metaflow/task.py +255 -26
- metaflow/user_configs/config_options.py +33 -21
- metaflow/user_configs/config_parameters.py +220 -58
- metaflow/user_decorators/__init__.py +0 -0
- metaflow/user_decorators/common.py +144 -0
- metaflow/user_decorators/mutable_flow.py +512 -0
- metaflow/user_decorators/mutable_step.py +424 -0
- metaflow/user_decorators/user_flow_decorator.py +264 -0
- metaflow/user_decorators/user_step_decorator.py +749 -0
- metaflow/util.py +197 -7
- metaflow/vendor.py +23 -7
- metaflow/version.py +1 -1
- {ob_metaflow-2.15.13.1.data → ob_metaflow-2.19.7.1rc0.data}/data/share/metaflow/devtools/Makefile +13 -2
- {ob_metaflow-2.15.13.1.data → ob_metaflow-2.19.7.1rc0.data}/data/share/metaflow/devtools/Tiltfile +107 -7
- {ob_metaflow-2.15.13.1.data → ob_metaflow-2.19.7.1rc0.data}/data/share/metaflow/devtools/pick_services.sh +1 -0
- {ob_metaflow-2.15.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/METADATA +2 -3
- {ob_metaflow-2.15.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/RECORD +162 -121
- {ob_metaflow-2.15.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/WHEEL +1 -1
- metaflow/_vendor/v3_5/__init__.py +0 -1
- metaflow/_vendor/v3_5/importlib_metadata/__init__.py +0 -644
- metaflow/_vendor/v3_5/importlib_metadata/_compat.py +0 -152
- metaflow/_vendor/v3_5/zipp.py +0 -329
- metaflow/info_file.py +0 -25
- metaflow/package.py +0 -203
- metaflow/user_configs/config_decorators.py +0 -568
- {ob_metaflow-2.15.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/entry_points.txt +0 -0
- {ob_metaflow-2.15.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/licenses/LICENSE +0 -0
- {ob_metaflow-2.15.13.1.dist-info → ob_metaflow-2.19.7.1rc0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
|
|
2
|
+
__all__ = ['Composer', 'ComposerError']
|
|
3
|
+
|
|
4
|
+
from .error import MarkedYAMLError
|
|
5
|
+
from .events import *
|
|
6
|
+
from .nodes import *
|
|
7
|
+
|
|
8
|
+
class ComposerError(MarkedYAMLError):
|
|
9
|
+
pass
|
|
10
|
+
|
|
11
|
+
class Composer:
|
|
12
|
+
|
|
13
|
+
def __init__(self):
|
|
14
|
+
self.anchors = {}
|
|
15
|
+
|
|
16
|
+
def check_node(self):
|
|
17
|
+
# Drop the STREAM-START event.
|
|
18
|
+
if self.check_event(StreamStartEvent):
|
|
19
|
+
self.get_event()
|
|
20
|
+
|
|
21
|
+
# If there are more documents available?
|
|
22
|
+
return not self.check_event(StreamEndEvent)
|
|
23
|
+
|
|
24
|
+
def get_node(self):
|
|
25
|
+
# Get the root node of the next document.
|
|
26
|
+
if not self.check_event(StreamEndEvent):
|
|
27
|
+
return self.compose_document()
|
|
28
|
+
|
|
29
|
+
def get_single_node(self):
|
|
30
|
+
# Drop the STREAM-START event.
|
|
31
|
+
self.get_event()
|
|
32
|
+
|
|
33
|
+
# Compose a document if the stream is not empty.
|
|
34
|
+
document = None
|
|
35
|
+
if not self.check_event(StreamEndEvent):
|
|
36
|
+
document = self.compose_document()
|
|
37
|
+
|
|
38
|
+
# Ensure that the stream contains no more documents.
|
|
39
|
+
if not self.check_event(StreamEndEvent):
|
|
40
|
+
event = self.get_event()
|
|
41
|
+
raise ComposerError("expected a single document in the stream",
|
|
42
|
+
document.start_mark, "but found another document",
|
|
43
|
+
event.start_mark)
|
|
44
|
+
|
|
45
|
+
# Drop the STREAM-END event.
|
|
46
|
+
self.get_event()
|
|
47
|
+
|
|
48
|
+
return document
|
|
49
|
+
|
|
50
|
+
def compose_document(self):
|
|
51
|
+
# Drop the DOCUMENT-START event.
|
|
52
|
+
self.get_event()
|
|
53
|
+
|
|
54
|
+
# Compose the root node.
|
|
55
|
+
node = self.compose_node(None, None)
|
|
56
|
+
|
|
57
|
+
# Drop the DOCUMENT-END event.
|
|
58
|
+
self.get_event()
|
|
59
|
+
|
|
60
|
+
self.anchors = {}
|
|
61
|
+
return node
|
|
62
|
+
|
|
63
|
+
def compose_node(self, parent, index):
|
|
64
|
+
if self.check_event(AliasEvent):
|
|
65
|
+
event = self.get_event()
|
|
66
|
+
anchor = event.anchor
|
|
67
|
+
if anchor not in self.anchors:
|
|
68
|
+
raise ComposerError(None, None, "found undefined alias %r"
|
|
69
|
+
% anchor, event.start_mark)
|
|
70
|
+
return self.anchors[anchor]
|
|
71
|
+
event = self.peek_event()
|
|
72
|
+
anchor = event.anchor
|
|
73
|
+
if anchor is not None:
|
|
74
|
+
if anchor in self.anchors:
|
|
75
|
+
raise ComposerError("found duplicate anchor %r; first occurrence"
|
|
76
|
+
% anchor, self.anchors[anchor].start_mark,
|
|
77
|
+
"second occurrence", event.start_mark)
|
|
78
|
+
self.descend_resolver(parent, index)
|
|
79
|
+
if self.check_event(ScalarEvent):
|
|
80
|
+
node = self.compose_scalar_node(anchor)
|
|
81
|
+
elif self.check_event(SequenceStartEvent):
|
|
82
|
+
node = self.compose_sequence_node(anchor)
|
|
83
|
+
elif self.check_event(MappingStartEvent):
|
|
84
|
+
node = self.compose_mapping_node(anchor)
|
|
85
|
+
self.ascend_resolver()
|
|
86
|
+
return node
|
|
87
|
+
|
|
88
|
+
def compose_scalar_node(self, anchor):
|
|
89
|
+
event = self.get_event()
|
|
90
|
+
tag = event.tag
|
|
91
|
+
if tag is None or tag == '!':
|
|
92
|
+
tag = self.resolve(ScalarNode, event.value, event.implicit)
|
|
93
|
+
node = ScalarNode(tag, event.value,
|
|
94
|
+
event.start_mark, event.end_mark, style=event.style)
|
|
95
|
+
if anchor is not None:
|
|
96
|
+
self.anchors[anchor] = node
|
|
97
|
+
return node
|
|
98
|
+
|
|
99
|
+
def compose_sequence_node(self, anchor):
|
|
100
|
+
start_event = self.get_event()
|
|
101
|
+
tag = start_event.tag
|
|
102
|
+
if tag is None or tag == '!':
|
|
103
|
+
tag = self.resolve(SequenceNode, None, start_event.implicit)
|
|
104
|
+
node = SequenceNode(tag, [],
|
|
105
|
+
start_event.start_mark, None,
|
|
106
|
+
flow_style=start_event.flow_style)
|
|
107
|
+
if anchor is not None:
|
|
108
|
+
self.anchors[anchor] = node
|
|
109
|
+
index = 0
|
|
110
|
+
while not self.check_event(SequenceEndEvent):
|
|
111
|
+
node.value.append(self.compose_node(node, index))
|
|
112
|
+
index += 1
|
|
113
|
+
end_event = self.get_event()
|
|
114
|
+
node.end_mark = end_event.end_mark
|
|
115
|
+
return node
|
|
116
|
+
|
|
117
|
+
def compose_mapping_node(self, anchor):
|
|
118
|
+
start_event = self.get_event()
|
|
119
|
+
tag = start_event.tag
|
|
120
|
+
if tag is None or tag == '!':
|
|
121
|
+
tag = self.resolve(MappingNode, None, start_event.implicit)
|
|
122
|
+
node = MappingNode(tag, [],
|
|
123
|
+
start_event.start_mark, None,
|
|
124
|
+
flow_style=start_event.flow_style)
|
|
125
|
+
if anchor is not None:
|
|
126
|
+
self.anchors[anchor] = node
|
|
127
|
+
while not self.check_event(MappingEndEvent):
|
|
128
|
+
#key_event = self.peek_event()
|
|
129
|
+
item_key = self.compose_node(node, None)
|
|
130
|
+
#if item_key in node.value:
|
|
131
|
+
# raise ComposerError("while composing a mapping", start_event.start_mark,
|
|
132
|
+
# "found duplicate key", key_event.start_mark)
|
|
133
|
+
item_value = self.compose_node(node, item_key)
|
|
134
|
+
#node.value[item_key] = item_value
|
|
135
|
+
node.value.append((item_key, item_value))
|
|
136
|
+
end_event = self.get_event()
|
|
137
|
+
node.end_mark = end_event.end_mark
|
|
138
|
+
return node
|
|
139
|
+
|