ob-metaflow-stubs 2.11.4.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +2753 -0
- metaflow-stubs/cards.pyi +266 -0
- metaflow-stubs/cli.pyi +137 -0
- metaflow-stubs/client/__init__.pyi +993 -0
- metaflow-stubs/client/core.pyi +1425 -0
- metaflow-stubs/client/filecache.pyi +87 -0
- metaflow-stubs/events.pyi +107 -0
- metaflow-stubs/exception.pyi +98 -0
- metaflow-stubs/flowspec.pyi +297 -0
- metaflow-stubs/generated_for.txt +1 -0
- metaflow-stubs/includefile.pyi +524 -0
- metaflow-stubs/metadata/metadata.pyi +377 -0
- metaflow-stubs/metadata/util.pyi +18 -0
- metaflow-stubs/metaflow_config.pyi +263 -0
- metaflow-stubs/metaflow_current.pyi +327 -0
- metaflow-stubs/mflog/mflog.pyi +22 -0
- metaflow-stubs/multicore_utils.pyi +62 -0
- metaflow-stubs/parameters.pyi +114 -0
- metaflow-stubs/plugins/__init__.pyi +209 -0
- metaflow-stubs/plugins/airflow/__init__.pyi +9 -0
- metaflow-stubs/plugins/airflow/airflow.pyi +179 -0
- metaflow-stubs/plugins/airflow/airflow_cli.pyi +90 -0
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +50 -0
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +137 -0
- metaflow-stubs/plugins/airflow/exception.pyi +27 -0
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +26 -0
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +60 -0
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +54 -0
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +50 -0
- metaflow-stubs/plugins/argo/__init__.pyi +9 -0
- metaflow-stubs/plugins/argo/argo_client.pyi +77 -0
- metaflow-stubs/plugins/argo/argo_events.pyi +79 -0
- metaflow-stubs/plugins/argo/argo_workflows.pyi +604 -0
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +180 -0
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +169 -0
- metaflow-stubs/plugins/aws/__init__.pyi +9 -0
- metaflow-stubs/plugins/aws/aws_client.pyi +22 -0
- metaflow-stubs/plugins/aws/aws_utils.pyi +93 -0
- metaflow-stubs/plugins/aws/batch/__init__.pyi +9 -0
- metaflow-stubs/plugins/aws/batch/batch.pyi +120 -0
- metaflow-stubs/plugins/aws/batch/batch_cli.pyi +42 -0
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +159 -0
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +145 -0
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +9 -0
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +73 -0
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +9 -0
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +22 -0
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +27 -0
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +18 -0
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +17 -0
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +220 -0
- metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +139 -0
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +36 -0
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +49 -0
- metaflow-stubs/plugins/azure/__init__.pyi +9 -0
- metaflow-stubs/plugins/azure/azure_credential.pyi +28 -0
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +28 -0
- metaflow-stubs/plugins/azure/azure_utils.pyi +76 -0
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +52 -0
- metaflow-stubs/plugins/azure/includefile_support.pyi +63 -0
- metaflow-stubs/plugins/cards/__init__.pyi +9 -0
- metaflow-stubs/plugins/cards/card_cli.pyi +557 -0
- metaflow-stubs/plugins/cards/card_client.pyi +178 -0
- metaflow-stubs/plugins/cards/card_creator.pyi +26 -0
- metaflow-stubs/plugins/cards/card_datastore.pyi +111 -0
- metaflow-stubs/plugins/cards/card_decorator.pyi +133 -0
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +67 -0
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +263 -0
- metaflow-stubs/plugins/cards/card_modules/card.pyi +62 -0
- metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +78 -0
- metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +77 -0
- metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +11 -0
- metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +114 -0
- metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +75 -0
- metaflow-stubs/plugins/cards/card_modules/components.pyi +251 -0
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +45 -0
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +24 -0
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +155 -0
- metaflow-stubs/plugins/cards/card_resolver.pyi +60 -0
- metaflow-stubs/plugins/cards/component_serializer.pyi +227 -0
- metaflow-stubs/plugins/cards/exception.pyi +71 -0
- metaflow-stubs/plugins/catch_decorator.pyi +58 -0
- metaflow-stubs/plugins/datatools/__init__.pyi +339 -0
- metaflow-stubs/plugins/datatools/local.pyi +82 -0
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +589 -0
- metaflow-stubs/plugins/datatools/s3/s3.pyi +875 -0
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +34 -0
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +45 -0
- metaflow-stubs/plugins/debug_logger.pyi +25 -0
- metaflow-stubs/plugins/debug_monitor.pyi +25 -0
- metaflow-stubs/plugins/environment_decorator.pyi +17 -0
- metaflow-stubs/plugins/events_decorator.pyi +34 -0
- metaflow-stubs/plugins/frameworks/__init__.pyi +9 -0
- metaflow-stubs/plugins/frameworks/pytorch.pyi +42 -0
- metaflow-stubs/plugins/gcp/__init__.pyi +9 -0
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +22 -0
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +26 -0
- metaflow-stubs/plugins/gcp/gs_utils.pyi +38 -0
- metaflow-stubs/plugins/gcp/includefile_support.pyi +63 -0
- metaflow-stubs/plugins/kubernetes/__init__.pyi +9 -0
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +127 -0
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +73 -0
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +62 -0
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +165 -0
- metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +100 -0
- metaflow-stubs/plugins/package_cli.pyi +9 -0
- metaflow-stubs/plugins/parallel_decorator.pyi +34 -0
- metaflow-stubs/plugins/project_decorator.pyi +36 -0
- metaflow-stubs/plugins/pypi/__init__.pyi +18 -0
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +59 -0
- metaflow-stubs/plugins/pypi/conda_environment.pyi +86 -0
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +22 -0
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +50 -0
- metaflow-stubs/plugins/pypi/utils.pyi +28 -0
- metaflow-stubs/plugins/resources_decorator.pyi +15 -0
- metaflow-stubs/plugins/retry_decorator.pyi +28 -0
- metaflow-stubs/plugins/secrets/__init__.pyi +21 -0
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +30 -0
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +75 -0
- metaflow-stubs/plugins/storage_executor.pyi +33 -0
- metaflow-stubs/plugins/tag_cli.pyi +370 -0
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +54 -0
- metaflow-stubs/plugins/timeout_decorator.pyi +39 -0
- metaflow-stubs/procpoll.pyi +51 -0
- metaflow-stubs/py.typed +0 -0
- metaflow-stubs/pylint_wrapper.pyi +31 -0
- metaflow-stubs/tagging_util.pyi +52 -0
- ob_metaflow_stubs-2.11.4.1.dist-info/METADATA +22 -0
- ob_metaflow_stubs-2.11.4.1.dist-info/RECORD +131 -0
- ob_metaflow_stubs-2.11.4.1.dist-info/WHEEL +6 -0
- ob_metaflow_stubs-2.11.4.1.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1425 @@
|
|
1
|
+
##################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.11.4.1 #
|
4
|
+
# Generated on 2024-02-28T05:30:43.900651 #
|
5
|
+
##################################################################################
|
6
|
+
|
7
|
+
from __future__ import annotations
|
8
|
+
|
9
|
+
import typing
|
10
|
+
if typing.TYPE_CHECKING:
|
11
|
+
import metaflow.exception
|
12
|
+
import metaflow.metaflow_current
|
13
|
+
import metaflow.client.core
|
14
|
+
import metaflow.events
|
15
|
+
import datetime
|
16
|
+
import metaflow
|
17
|
+
import tarfile
|
18
|
+
import typing
|
19
|
+
|
20
|
+
current: metaflow.metaflow_current.Current
|
21
|
+
|
22
|
+
class Trigger(object, metaclass=type):
|
23
|
+
def __init__(self, _meta = None):
|
24
|
+
...
|
25
|
+
@classmethod
|
26
|
+
def from_runs(cls, run_objs: typing.List["metaflow.Run"]):
|
27
|
+
...
|
28
|
+
@property
|
29
|
+
def event(self) -> typing.Optional[metaflow.events.MetaflowEvent]:
|
30
|
+
"""
|
31
|
+
The `MetaflowEvent` object corresponding to the triggering event.
|
32
|
+
|
33
|
+
If multiple events triggered the run, this property is the latest event.
|
34
|
+
|
35
|
+
Returns
|
36
|
+
-------
|
37
|
+
MetaflowEvent, optional
|
38
|
+
The latest event that triggered the run, if applicable.
|
39
|
+
"""
|
40
|
+
...
|
41
|
+
@property
|
42
|
+
def events(self) -> typing.Optional[typing.List[metaflow.events.MetaflowEvent]]:
|
43
|
+
"""
|
44
|
+
The list of `MetaflowEvent` objects correspondings to all the triggering events.
|
45
|
+
|
46
|
+
Returns
|
47
|
+
-------
|
48
|
+
List[MetaflowEvent], optional
|
49
|
+
List of all events that triggered the run
|
50
|
+
"""
|
51
|
+
...
|
52
|
+
@property
|
53
|
+
def run(self) -> typing.Optional["metaflow.Run"]:
|
54
|
+
"""
|
55
|
+
The corresponding `Run` object if the triggering event is a Metaflow run.
|
56
|
+
|
57
|
+
In case multiple runs triggered the run, this property is the latest run.
|
58
|
+
Returns `None` if none of the triggering events are a `Run`.
|
59
|
+
|
60
|
+
Returns
|
61
|
+
-------
|
62
|
+
Run, optional
|
63
|
+
Latest Run that triggered this run, if applicable.
|
64
|
+
"""
|
65
|
+
...
|
66
|
+
@property
|
67
|
+
def runs(self) -> typing.Optional[typing.List["metaflow.Run"]]:
|
68
|
+
"""
|
69
|
+
The list of `Run` objects in the triggering events.
|
70
|
+
Returns `None` if none of the triggering events are `Run` objects.
|
71
|
+
|
72
|
+
Returns
|
73
|
+
-------
|
74
|
+
List[Run], optional
|
75
|
+
List of runs that triggered this run, if applicable.
|
76
|
+
"""
|
77
|
+
...
|
78
|
+
def __getitem__(self, key: str) -> typing.Union["metaflow.Run", metaflow.events.MetaflowEvent]:
|
79
|
+
"""
|
80
|
+
If triggering events are runs, `key` corresponds to the flow name of the triggering run.
|
81
|
+
Otherwise, `key` corresponds to the event name and a `MetaflowEvent` object is returned.
|
82
|
+
|
83
|
+
Returns
|
84
|
+
-------
|
85
|
+
Union[Run, MetaflowEvent]
|
86
|
+
`Run` object if triggered by a run. Otherwise returns a `MetaflowEvent`.
|
87
|
+
"""
|
88
|
+
...
|
89
|
+
def __iter__(self):
|
90
|
+
...
|
91
|
+
def __contains__(self, ident: str) -> bool:
|
92
|
+
...
|
93
|
+
...
|
94
|
+
|
95
|
+
class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
|
96
|
+
...
|
97
|
+
|
98
|
+
class MetaflowInvalidPathspec(metaflow.exception.MetaflowException, metaclass=type):
|
99
|
+
def __init__(self, msg):
|
100
|
+
...
|
101
|
+
...
|
102
|
+
|
103
|
+
class MetaflowNamespaceMismatch(metaflow.exception.MetaflowException, metaclass=type):
|
104
|
+
def __init__(self, namespace):
|
105
|
+
...
|
106
|
+
...
|
107
|
+
|
108
|
+
class MetaflowNotFound(metaflow.exception.MetaflowException, metaclass=type):
|
109
|
+
...
|
110
|
+
|
111
|
+
class IncludedFile(object, metaclass=type):
|
112
|
+
def __init__(self, descriptor: typing.Dict[str, typing.Any]):
|
113
|
+
...
|
114
|
+
@property
|
115
|
+
def descriptor(self):
|
116
|
+
...
|
117
|
+
@property
|
118
|
+
def size(self):
|
119
|
+
...
|
120
|
+
def decode(self, name, var_type = "Artifact"):
|
121
|
+
...
|
122
|
+
...
|
123
|
+
|
124
|
+
DEFAULT_METADATA: str
|
125
|
+
|
126
|
+
MAX_ATTEMPTS: int
|
127
|
+
|
128
|
+
ENVIRONMENTS: list
|
129
|
+
|
130
|
+
METADATA_PROVIDERS: list
|
131
|
+
|
132
|
+
CONTROL_TASK_TAG: str
|
133
|
+
|
134
|
+
INFO_FILE: str
|
135
|
+
|
136
|
+
class FileCache(object, metaclass=type):
|
137
|
+
def __init__(self, cache_dir = None, max_size = None):
|
138
|
+
...
|
139
|
+
@property
|
140
|
+
def cache_dir(self):
|
141
|
+
...
|
142
|
+
def get_logs_stream(self, ds_type, ds_root, stream, attempt, flow_name, run_id, step_name, task_id):
|
143
|
+
...
|
144
|
+
def get_log_legacy(self, ds_type, location, logtype, attempt, flow_name, run_id, step_name, task_id):
|
145
|
+
...
|
146
|
+
def get_legacy_log_size(self, ds_type, location, logtype, attempt, flow_name, run_id, step_name, task_id):
|
147
|
+
...
|
148
|
+
def get_log_size(self, ds_type, ds_root, logtype, attempt, flow_name, run_id, step_name, task_id):
|
149
|
+
...
|
150
|
+
def get_data(self, ds_type, flow_name, location, key):
|
151
|
+
...
|
152
|
+
def get_artifact_size_by_location(self, ds_type, location, attempt, flow_name, run_id, step_name, task_id, name):
|
153
|
+
"""
|
154
|
+
Gets the size of the artifact content (in bytes) for the name at the location
|
155
|
+
"""
|
156
|
+
...
|
157
|
+
def get_artifact_size(self, ds_type, ds_root, attempt, flow_name, run_id, step_name, task_id, name):
|
158
|
+
"""
|
159
|
+
Gets the size of the artifact content (in bytes) for the name
|
160
|
+
"""
|
161
|
+
...
|
162
|
+
def get_artifact_by_location(self, ds_type, location, data_metadata, flow_name, run_id, step_name, task_id, name):
|
163
|
+
...
|
164
|
+
def get_artifact(self, ds_type, ds_root, data_metadata, flow_name, run_id, step_name, task_id, name):
|
165
|
+
...
|
166
|
+
def get_all_artifacts(self, ds_type, ds_root, data_metadata, flow_name, run_id, step_name, task_id):
|
167
|
+
...
|
168
|
+
def get_artifacts(self, ds_type, ds_root, data_metadata, flow_name, run_id, step_name, task_id, names):
|
169
|
+
...
|
170
|
+
def create_file(self, path, value):
|
171
|
+
...
|
172
|
+
def read_file(self, path):
|
173
|
+
...
|
174
|
+
...
|
175
|
+
|
176
|
+
class Metadata(tuple, metaclass=type):
|
177
|
+
@staticmethod
|
178
|
+
def __new__(_cls, name, value, created_at, type, task):
|
179
|
+
"""
|
180
|
+
Create new instance of Metadata(name, value, created_at, type, task)
|
181
|
+
"""
|
182
|
+
...
|
183
|
+
def __repr__(self):
|
184
|
+
"""
|
185
|
+
Return a nicely formatted representation string
|
186
|
+
"""
|
187
|
+
...
|
188
|
+
def __getnewargs__(self):
|
189
|
+
"""
|
190
|
+
Return self as a plain tuple. Used by copy and pickle.
|
191
|
+
"""
|
192
|
+
...
|
193
|
+
...
|
194
|
+
|
195
|
+
filecache: None
|
196
|
+
|
197
|
+
current_namespace: bool
|
198
|
+
|
199
|
+
current_metadata: bool
|
200
|
+
|
201
|
+
def metadata(ms: str) -> str:
|
202
|
+
"""
|
203
|
+
Switch Metadata provider.
|
204
|
+
|
205
|
+
This call has a global effect. Selecting the local metadata will,
|
206
|
+
for example, not allow access to information stored in remote
|
207
|
+
metadata providers.
|
208
|
+
|
209
|
+
Note that you don't typically have to call this function directly. Usually
|
210
|
+
the metadata provider is set through the Metaflow configuration file. If you
|
211
|
+
need to switch between multiple providers, you can use the `METAFLOW_PROFILE`
|
212
|
+
environment variable to switch between configurations.
|
213
|
+
|
214
|
+
Parameters
|
215
|
+
----------
|
216
|
+
ms : str
|
217
|
+
Can be a path (selects local metadata), a URL starting with http (selects
|
218
|
+
the service metadata) or an explicit specification <metadata_type>@<info>; as an
|
219
|
+
example, you can specify local@<path> or service@<url>.
|
220
|
+
|
221
|
+
Returns
|
222
|
+
-------
|
223
|
+
str
|
224
|
+
The description of the metadata selected (equivalent to the result of
|
225
|
+
get_metadata()).
|
226
|
+
"""
|
227
|
+
...
|
228
|
+
|
229
|
+
def get_metadata() -> str:
|
230
|
+
"""
|
231
|
+
Returns the current Metadata provider.
|
232
|
+
|
233
|
+
If this is not set explicitly using `metadata`, the default value is
|
234
|
+
determined through the Metaflow configuration. You can use this call to
|
235
|
+
check that your configuration is set up properly.
|
236
|
+
|
237
|
+
If multiple configuration profiles are present, this call returns the one
|
238
|
+
selected through the `METAFLOW_PROFILE` environment variable.
|
239
|
+
|
240
|
+
Returns
|
241
|
+
-------
|
242
|
+
str
|
243
|
+
Information about the Metadata provider currently selected. This information typically
|
244
|
+
returns provider specific information (like URL for remote providers or local paths for
|
245
|
+
local providers).
|
246
|
+
"""
|
247
|
+
...
|
248
|
+
|
249
|
+
def default_metadata() -> str:
|
250
|
+
"""
|
251
|
+
Resets the Metadata provider to the default value, that is, to the value
|
252
|
+
that was used prior to any `metadata` calls.
|
253
|
+
|
254
|
+
Returns
|
255
|
+
-------
|
256
|
+
str
|
257
|
+
The result of get_metadata() after resetting the provider.
|
258
|
+
"""
|
259
|
+
...
|
260
|
+
|
261
|
+
def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
|
262
|
+
"""
|
263
|
+
Switch namespace to the one provided.
|
264
|
+
|
265
|
+
This call has a global effect. No objects outside this namespace
|
266
|
+
will be accessible. To access all objects regardless of namespaces,
|
267
|
+
pass None to this call.
|
268
|
+
|
269
|
+
Parameters
|
270
|
+
----------
|
271
|
+
ns : str, optional
|
272
|
+
Namespace to switch to or None to ignore namespaces.
|
273
|
+
|
274
|
+
Returns
|
275
|
+
-------
|
276
|
+
str, optional
|
277
|
+
Namespace set (result of get_namespace()).
|
278
|
+
"""
|
279
|
+
...
|
280
|
+
|
281
|
+
def get_namespace() -> typing.Optional[str]:
|
282
|
+
"""
|
283
|
+
Return the current namespace that is currently being used to filter objects.
|
284
|
+
|
285
|
+
The namespace is a tag associated with all objects in Metaflow.
|
286
|
+
|
287
|
+
Returns
|
288
|
+
-------
|
289
|
+
str, optional
|
290
|
+
The current namespace used to filter objects.
|
291
|
+
"""
|
292
|
+
...
|
293
|
+
|
294
|
+
def default_namespace() -> str:
|
295
|
+
"""
|
296
|
+
Resets the namespace used to filter objects to the default one, i.e. the one that was
|
297
|
+
used prior to any `namespace` calls.
|
298
|
+
|
299
|
+
Returns
|
300
|
+
-------
|
301
|
+
str
|
302
|
+
The result of get_namespace() after the namespace has been reset.
|
303
|
+
"""
|
304
|
+
...
|
305
|
+
|
306
|
+
class MetaflowObject(object, metaclass=type):
|
307
|
+
def __init__(self, pathspec: typing.Optional[str] = None, attempt: typing.Optional[int] = None, _object: typing.Optional["MetaflowObject"] = None, _parent: typing.Optional["MetaflowObject"] = None, _namespace_check: bool = True):
|
308
|
+
...
|
309
|
+
def __iter__(self) -> typing.Iterator["MetaflowObject"]:
|
310
|
+
"""
|
311
|
+
Iterate over all child objects of this object if any.
|
312
|
+
|
313
|
+
Note that only children present in the current namespace are returned if and
|
314
|
+
only if _namespace_check is set.
|
315
|
+
|
316
|
+
Yields
|
317
|
+
------
|
318
|
+
MetaflowObject
|
319
|
+
Children of this object
|
320
|
+
"""
|
321
|
+
...
|
322
|
+
def is_in_namespace(self) -> bool:
|
323
|
+
"""
|
324
|
+
Returns whether this object is in the current namespace.
|
325
|
+
|
326
|
+
If the current namespace is None, this will always return True.
|
327
|
+
|
328
|
+
Returns
|
329
|
+
-------
|
330
|
+
bool
|
331
|
+
Whether or not the object is in the current namespace
|
332
|
+
"""
|
333
|
+
...
|
334
|
+
def __str__(self):
|
335
|
+
...
|
336
|
+
def __repr__(self):
|
337
|
+
...
|
338
|
+
def __getitem__(self, id: str) -> MetaflowObject:
|
339
|
+
"""
|
340
|
+
Returns the child object named 'id'.
|
341
|
+
|
342
|
+
Parameters
|
343
|
+
----------
|
344
|
+
id : str
|
345
|
+
Name of the child object
|
346
|
+
|
347
|
+
Returns
|
348
|
+
-------
|
349
|
+
MetaflowObject
|
350
|
+
Child object
|
351
|
+
|
352
|
+
Raises
|
353
|
+
------
|
354
|
+
KeyError
|
355
|
+
If the name does not identify a valid child object
|
356
|
+
"""
|
357
|
+
...
|
358
|
+
def __contains__(self, id: str):
|
359
|
+
"""
|
360
|
+
Tests whether a child named 'id' exists.
|
361
|
+
|
362
|
+
Parameters
|
363
|
+
----------
|
364
|
+
id : str
|
365
|
+
Name of the child object
|
366
|
+
|
367
|
+
Returns
|
368
|
+
-------
|
369
|
+
bool
|
370
|
+
True if the child exists or False otherwise
|
371
|
+
"""
|
372
|
+
...
|
373
|
+
def __setstate__(self, state):
|
374
|
+
"""
|
375
|
+
This function is used during the unpickling operation.
|
376
|
+
More info here https://docs.python.org/3/library/pickle.html#object.__setstate__
|
377
|
+
"""
|
378
|
+
...
|
379
|
+
def __getstate__(self):
|
380
|
+
"""
|
381
|
+
This function is used during the pickling operation.
|
382
|
+
More info here https://docs.python.org/3/library/pickle.html#object.__getstate__
|
383
|
+
|
384
|
+
This function is not forward compatible i.e., if this object (or any of the objects deriving
|
385
|
+
from this object) are pickled (serialized) in a later version of Metaflow, it may not be possible
|
386
|
+
to unpickle (deserialize) them in a previous version of Metaflow.
|
387
|
+
"""
|
388
|
+
...
|
389
|
+
@property
|
390
|
+
def tags(self) -> typing.FrozenSet[str]:
|
391
|
+
"""
|
392
|
+
Tags associated with this object.
|
393
|
+
|
394
|
+
Tags can be user defined or system defined. This returns all tags associated
|
395
|
+
with the object.
|
396
|
+
|
397
|
+
Returns
|
398
|
+
-------
|
399
|
+
Set[str]
|
400
|
+
Tags associated with the object
|
401
|
+
"""
|
402
|
+
...
|
403
|
+
@property
|
404
|
+
def system_tags(self) -> typing.FrozenSet[str]:
|
405
|
+
"""
|
406
|
+
System defined tags associated with this object.
|
407
|
+
|
408
|
+
Returns
|
409
|
+
-------
|
410
|
+
Set[str]
|
411
|
+
System tags associated with the object
|
412
|
+
"""
|
413
|
+
...
|
414
|
+
@property
|
415
|
+
def user_tags(self) -> typing.FrozenSet[str]:
|
416
|
+
"""
|
417
|
+
User defined tags associated with this object.
|
418
|
+
|
419
|
+
Returns
|
420
|
+
-------
|
421
|
+
Set[str]
|
422
|
+
User tags associated with the object
|
423
|
+
"""
|
424
|
+
...
|
425
|
+
@property
|
426
|
+
def created_at(self) -> datetime.datetime:
|
427
|
+
"""
|
428
|
+
Creation time for this object.
|
429
|
+
|
430
|
+
This corresponds to the time the object's existence was first created which typically means
|
431
|
+
right before any code is run.
|
432
|
+
|
433
|
+
Returns
|
434
|
+
-------
|
435
|
+
datetime
|
436
|
+
Date time of this object's creation.
|
437
|
+
"""
|
438
|
+
...
|
439
|
+
@property
|
440
|
+
def origin_pathspec(self) -> typing.Optional[str]:
|
441
|
+
"""
|
442
|
+
The pathspec of the object from which the current object was cloned.
|
443
|
+
|
444
|
+
Returns:
|
445
|
+
str, optional
|
446
|
+
pathspec of the origin object from which current object was cloned.
|
447
|
+
"""
|
448
|
+
...
|
449
|
+
@property
|
450
|
+
def parent(self) -> typing.Optional["MetaflowObject"]:
|
451
|
+
"""
|
452
|
+
Returns the parent object of this object or None if none exists.
|
453
|
+
|
454
|
+
Returns
|
455
|
+
-------
|
456
|
+
MetaflowObject, optional
|
457
|
+
The parent of this object
|
458
|
+
"""
|
459
|
+
...
|
460
|
+
@property
|
461
|
+
def pathspec(self) -> str:
|
462
|
+
"""
|
463
|
+
Returns a string representation uniquely identifying this object.
|
464
|
+
|
465
|
+
The string is the same as the one you would pass into the constructor
|
466
|
+
to build this object except if you are looking for a specific attempt of
|
467
|
+
a task or a data artifact (in which case you need to add `attempt=<attempt>`
|
468
|
+
in the constructor).
|
469
|
+
|
470
|
+
Returns
|
471
|
+
-------
|
472
|
+
str
|
473
|
+
Unique representation of this object
|
474
|
+
"""
|
475
|
+
...
|
476
|
+
@property
|
477
|
+
def path_components(self) -> typing.List[str]:
|
478
|
+
"""
|
479
|
+
List of individual components of the pathspec.
|
480
|
+
|
481
|
+
Returns
|
482
|
+
-------
|
483
|
+
List[str]
|
484
|
+
Individual components of the pathspec
|
485
|
+
"""
|
486
|
+
...
|
487
|
+
...
|
488
|
+
|
489
|
+
class MetaflowCode(object, metaclass=type):
|
490
|
+
def __init__(self, flow_name: str, code_package: str):
|
491
|
+
...
|
492
|
+
@property
|
493
|
+
def path(self) -> str:
|
494
|
+
"""
|
495
|
+
Location (in the datastore provider) of the code package.
|
496
|
+
|
497
|
+
Returns
|
498
|
+
-------
|
499
|
+
str
|
500
|
+
Full path of the code package
|
501
|
+
"""
|
502
|
+
...
|
503
|
+
@property
|
504
|
+
def info(self) -> typing.Dict[str, str]:
|
505
|
+
"""
|
506
|
+
Metadata associated with the code package.
|
507
|
+
|
508
|
+
Returns
|
509
|
+
-------
|
510
|
+
Dict[str, str]
|
511
|
+
Dictionary of metadata. Keys and values are strings
|
512
|
+
"""
|
513
|
+
...
|
514
|
+
@property
|
515
|
+
def flowspec(self) -> str:
|
516
|
+
"""
|
517
|
+
Source code of the Python file containing the FlowSpec.
|
518
|
+
|
519
|
+
Returns
|
520
|
+
-------
|
521
|
+
str
|
522
|
+
Content of the Python file
|
523
|
+
"""
|
524
|
+
...
|
525
|
+
@property
|
526
|
+
def tarball(self) -> tarfile.TarFile:
|
527
|
+
"""
|
528
|
+
TarFile for this code package.
|
529
|
+
|
530
|
+
Returns
|
531
|
+
-------
|
532
|
+
TarFile
|
533
|
+
TarFile for everything in this code package
|
534
|
+
"""
|
535
|
+
...
|
536
|
+
def __str__(self):
|
537
|
+
...
|
538
|
+
...
|
539
|
+
|
540
|
+
class DataArtifact(MetaflowObject, metaclass=type):
|
541
|
+
@property
|
542
|
+
def data(self) -> typing.Any:
|
543
|
+
"""
|
544
|
+
Unpickled representation of the data contained in this artifact.
|
545
|
+
|
546
|
+
Returns
|
547
|
+
-------
|
548
|
+
object
|
549
|
+
Object contained in this artifact
|
550
|
+
"""
|
551
|
+
...
|
552
|
+
@property
|
553
|
+
def size(self) -> int:
|
554
|
+
"""
|
555
|
+
Returns the size (in bytes) of the pickled object representing this
|
556
|
+
DataArtifact
|
557
|
+
|
558
|
+
Returns
|
559
|
+
-------
|
560
|
+
int
|
561
|
+
size of the pickled representation of data artifact (in bytes)
|
562
|
+
"""
|
563
|
+
...
|
564
|
+
@property
|
565
|
+
def sha(self) -> str:
|
566
|
+
"""
|
567
|
+
Unique identifier for this artifact.
|
568
|
+
|
569
|
+
This is a unique hash of the artifact (historically SHA1 hash)
|
570
|
+
|
571
|
+
Returns
|
572
|
+
-------
|
573
|
+
str
|
574
|
+
Hash of this artifact
|
575
|
+
"""
|
576
|
+
...
|
577
|
+
@property
|
578
|
+
def finished_at(self) -> datetime.datetime:
|
579
|
+
"""
|
580
|
+
Creation time for this artifact.
|
581
|
+
|
582
|
+
Alias for created_at.
|
583
|
+
|
584
|
+
Returns
|
585
|
+
-------
|
586
|
+
datetime
|
587
|
+
Creation time
|
588
|
+
"""
|
589
|
+
...
|
590
|
+
def __getstate__(self):
|
591
|
+
...
|
592
|
+
def __setstate__(self, state):
|
593
|
+
...
|
594
|
+
...
|
595
|
+
|
596
|
+
class MetaflowData(object, metaclass=type):
|
597
|
+
def __init__(self, artifacts: typing.Iterable[DataArtifact]):
|
598
|
+
...
|
599
|
+
def __getattr__(self, name: str):
|
600
|
+
...
|
601
|
+
def __contains__(self, var):
|
602
|
+
...
|
603
|
+
def __str__(self):
|
604
|
+
...
|
605
|
+
def __repr__(self):
|
606
|
+
...
|
607
|
+
...
|
608
|
+
|
609
|
+
class Task(MetaflowObject, metaclass=type):
|
610
|
+
def __init__(self, *args, **kwargs):
|
611
|
+
...
|
612
|
+
@property
|
613
|
+
def metadata(self) -> typing.List[Metadata]:
|
614
|
+
"""
|
615
|
+
Metadata events produced by this task across all attempts of the task
|
616
|
+
*except* if you selected a specific task attempt.
|
617
|
+
|
618
|
+
Note that Metadata is different from tags.
|
619
|
+
|
620
|
+
Returns
|
621
|
+
-------
|
622
|
+
List[Metadata]
|
623
|
+
Metadata produced by this task
|
624
|
+
"""
|
625
|
+
...
|
626
|
+
@property
|
627
|
+
def metadata_dict(self) -> typing.Dict[str, str]:
|
628
|
+
"""
|
629
|
+
Dictionary mapping metadata names (keys) and their associated values.
|
630
|
+
|
631
|
+
Note that unlike the metadata() method, this call will only return the latest
|
632
|
+
metadata for a given name. For example, if a task executes multiple times (retries),
|
633
|
+
the same metadata name will be generated multiple times (one for each execution of the
|
634
|
+
task). The metadata() method returns all those metadata elements whereas this call will
|
635
|
+
return the metadata associated with the latest execution of the task.
|
636
|
+
|
637
|
+
Returns
|
638
|
+
-------
|
639
|
+
Dict[str, str]
|
640
|
+
Dictionary mapping metadata name with value
|
641
|
+
"""
|
642
|
+
...
|
643
|
+
@property
|
644
|
+
def index(self) -> typing.Optional[int]:
|
645
|
+
"""
|
646
|
+
Returns the index of the innermost foreach loop if this task is run inside at least
|
647
|
+
one foreach.
|
648
|
+
|
649
|
+
The index is what distinguishes the various tasks inside a given step.
|
650
|
+
This call returns None if this task was not run in a foreach loop.
|
651
|
+
|
652
|
+
Returns
|
653
|
+
-------
|
654
|
+
int, optional
|
655
|
+
Index in the innermost loop for this task
|
656
|
+
"""
|
657
|
+
...
|
658
|
+
@property
|
659
|
+
def data(self) -> MetaflowData:
|
660
|
+
"""
|
661
|
+
Returns a container of data artifacts produced by this task.
|
662
|
+
|
663
|
+
You can access data produced by this task as follows:
|
664
|
+
```
|
665
|
+
print(task.data.my_var)
|
666
|
+
```
|
667
|
+
|
668
|
+
Returns
|
669
|
+
-------
|
670
|
+
MetaflowData
|
671
|
+
Container of all artifacts produced by this task
|
672
|
+
"""
|
673
|
+
...
|
674
|
+
@property
|
675
|
+
def artifacts(self) -> typing.NamedTuple:
|
676
|
+
"""
|
677
|
+
Returns a container of DataArtifacts produced by this task.
|
678
|
+
|
679
|
+
You can access each DataArtifact by name like so:
|
680
|
+
```
|
681
|
+
print(task.artifacts.my_var)
|
682
|
+
```
|
683
|
+
This method differs from data() because it returns DataArtifact objects
|
684
|
+
(which contain additional metadata) as opposed to just the data.
|
685
|
+
|
686
|
+
Returns
|
687
|
+
-------
|
688
|
+
MetaflowArtifacts
|
689
|
+
Container of all DataArtifacts produced by this task
|
690
|
+
"""
|
691
|
+
...
|
692
|
+
@property
|
693
|
+
def successful(self) -> bool:
|
694
|
+
"""
|
695
|
+
Indicates whether or not the task completed successfully.
|
696
|
+
|
697
|
+
This information is always about the latest task to have completed (in case
|
698
|
+
of retries).
|
699
|
+
|
700
|
+
Returns
|
701
|
+
-------
|
702
|
+
bool
|
703
|
+
True if the task completed successfully and False otherwise
|
704
|
+
"""
|
705
|
+
...
|
706
|
+
@property
|
707
|
+
def finished(self) -> bool:
|
708
|
+
"""
|
709
|
+
Indicates whether or not the task completed.
|
710
|
+
|
711
|
+
This information is always about the latest task to have completed (in case
|
712
|
+
of retries).
|
713
|
+
|
714
|
+
Returns
|
715
|
+
-------
|
716
|
+
bool
|
717
|
+
True if the task completed and False otherwise
|
718
|
+
"""
|
719
|
+
...
|
720
|
+
@property
|
721
|
+
def exception(self) -> typing.Optional[typing.Any]:
|
722
|
+
"""
|
723
|
+
Returns the exception that caused the task to fail, if any.
|
724
|
+
|
725
|
+
This information is always about the latest task to have completed (in case
|
726
|
+
of retries). If successful() returns False and finished() returns True,
|
727
|
+
this method can help determine what went wrong.
|
728
|
+
|
729
|
+
Returns
|
730
|
+
-------
|
731
|
+
object
|
732
|
+
Exception raised by the task or None if not applicable
|
733
|
+
"""
|
734
|
+
...
|
735
|
+
@property
|
736
|
+
def finished_at(self) -> typing.Optional[datetime.datetime]:
|
737
|
+
"""
|
738
|
+
Returns the datetime object of when the task finished (successfully or not).
|
739
|
+
|
740
|
+
This information is always about the latest task to have completed (in case
|
741
|
+
of retries). This call will return None if the task is not finished.
|
742
|
+
|
743
|
+
Returns
|
744
|
+
-------
|
745
|
+
datetime
|
746
|
+
Datetime of when the task finished
|
747
|
+
"""
|
748
|
+
...
|
749
|
+
@property
|
750
|
+
def runtime_name(self) -> typing.Optional[str]:
|
751
|
+
"""
|
752
|
+
Returns the name of the runtime this task executed on.
|
753
|
+
|
754
|
+
|
755
|
+
Returns
|
756
|
+
-------
|
757
|
+
str
|
758
|
+
Name of the runtime this task executed on
|
759
|
+
"""
|
760
|
+
...
|
761
|
+
@property
|
762
|
+
def stdout(self) -> str:
|
763
|
+
"""
|
764
|
+
Returns the full standard out of this task.
|
765
|
+
|
766
|
+
If you specify a specific attempt for this task, it will return the
|
767
|
+
standard out for that attempt. If you do not specify an attempt,
|
768
|
+
this will return the current standard out for the latest *started*
|
769
|
+
attempt of the task. In both cases, multiple calls to this
|
770
|
+
method will return the most up-to-date log (so if an attempt is not
|
771
|
+
done, each call will fetch the latest log).
|
772
|
+
|
773
|
+
Returns
|
774
|
+
-------
|
775
|
+
str
|
776
|
+
Standard output of this task
|
777
|
+
"""
|
778
|
+
...
|
779
|
+
@property
|
780
|
+
def stdout_size(self) -> int:
|
781
|
+
"""
|
782
|
+
Returns the size of the stdout log of this task.
|
783
|
+
|
784
|
+
Similar to `stdout`, the size returned is the latest size of the log
|
785
|
+
(so for a running attempt, this value will increase as the task produces
|
786
|
+
more output).
|
787
|
+
|
788
|
+
Returns
|
789
|
+
-------
|
790
|
+
int
|
791
|
+
Size of the stdout log content (in bytes)
|
792
|
+
"""
|
793
|
+
...
|
794
|
+
@property
|
795
|
+
def stderr(self) -> str:
|
796
|
+
"""
|
797
|
+
Returns the full standard error of this task.
|
798
|
+
|
799
|
+
If you specify a specific attempt for this task, it will return the
|
800
|
+
standard error for that attempt. If you do not specify an attempt,
|
801
|
+
this will return the current standard error for the latest *started*
|
802
|
+
attempt. In both cases, multiple calls to this
|
803
|
+
method will return the most up-to-date log (so if an attempt is not
|
804
|
+
done, each call will fetch the latest log).
|
805
|
+
|
806
|
+
Returns
|
807
|
+
-------
|
808
|
+
str
|
809
|
+
Standard error of this task
|
810
|
+
"""
|
811
|
+
...
|
812
|
+
@property
|
813
|
+
def stderr_size(self) -> int:
|
814
|
+
"""
|
815
|
+
Returns the size of the stderr log of this task.
|
816
|
+
|
817
|
+
Similar to `stderr`, the size returned is the latest size of the log
|
818
|
+
(so for a running attempt, this value will increase as the task produces
|
819
|
+
more output).
|
820
|
+
|
821
|
+
Returns
|
822
|
+
-------
|
823
|
+
int
|
824
|
+
Size of the stderr log content (in bytes)
|
825
|
+
"""
|
826
|
+
...
|
827
|
+
@property
|
828
|
+
def current_attempt(self) -> int:
|
829
|
+
"""
|
830
|
+
Get the relevant attempt for this Task.
|
831
|
+
|
832
|
+
Returns the specific attempt used when
|
833
|
+
initializing the instance, or the latest *started* attempt for the Task.
|
834
|
+
|
835
|
+
Returns
|
836
|
+
-------
|
837
|
+
int
|
838
|
+
attempt id for this task object
|
839
|
+
"""
|
840
|
+
...
|
841
|
+
@property
|
842
|
+
def code(self) -> typing.Optional[MetaflowCode]:
|
843
|
+
"""
|
844
|
+
Returns the MetaflowCode object for this task, if present.
|
845
|
+
|
846
|
+
Not all tasks save their code so this call may return None in those cases.
|
847
|
+
|
848
|
+
Returns
|
849
|
+
-------
|
850
|
+
MetaflowCode
|
851
|
+
Code package for this task
|
852
|
+
"""
|
853
|
+
...
|
854
|
+
@property
|
855
|
+
def environment_info(self) -> typing.Dict[str, typing.Any]:
|
856
|
+
"""
|
857
|
+
Returns information about the environment that was used to execute this task. As an
|
858
|
+
example, if the Conda environment is selected, this will return information about the
|
859
|
+
dependencies that were used in the environment.
|
860
|
+
|
861
|
+
This environment information is only available for tasks that have a code package.
|
862
|
+
|
863
|
+
Returns
|
864
|
+
-------
|
865
|
+
Dict
|
866
|
+
Dictionary describing the environment
|
867
|
+
"""
|
868
|
+
...
|
869
|
+
def loglines(self, stream: str, as_unicode: bool = True, meta_dict: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Iterator[typing.Tuple[datetime.datetime, str]]:
|
870
|
+
"""
|
871
|
+
Return an iterator over (utc_timestamp, logline) tuples.
|
872
|
+
|
873
|
+
Parameters
|
874
|
+
----------
|
875
|
+
stream : str
|
876
|
+
Either 'stdout' or 'stderr'.
|
877
|
+
as_unicode : bool, default: True
|
878
|
+
If as_unicode=False, each logline is returned as a byte object. Otherwise,
|
879
|
+
it is returned as a (unicode) string.
|
880
|
+
|
881
|
+
Yields
|
882
|
+
------
|
883
|
+
Tuple[datetime, str]
|
884
|
+
Tuple of timestamp, logline pairs.
|
885
|
+
"""
|
886
|
+
...
|
887
|
+
def __iter__(self) -> typing.Iterator[DataArtifact]:
|
888
|
+
"""
|
889
|
+
Iterate over all children DataArtifact of this Task
|
890
|
+
|
891
|
+
Yields
|
892
|
+
------
|
893
|
+
DataArtifact
|
894
|
+
A DataArtifact in this Step
|
895
|
+
"""
|
896
|
+
...
|
897
|
+
def __getitem__(self, name: str) -> DataArtifact:
|
898
|
+
"""
|
899
|
+
Returns the DataArtifact object with the artifact name 'name'
|
900
|
+
|
901
|
+
Parameters
|
902
|
+
----------
|
903
|
+
name : str
|
904
|
+
Data artifact name
|
905
|
+
|
906
|
+
Returns
|
907
|
+
-------
|
908
|
+
DataArtifact
|
909
|
+
DataArtifact for this artifact name in this task
|
910
|
+
|
911
|
+
Raises
|
912
|
+
------
|
913
|
+
KeyError
|
914
|
+
If the name does not identify a valid DataArtifact object
|
915
|
+
"""
|
916
|
+
...
|
917
|
+
def __getstate__(self):
|
918
|
+
...
|
919
|
+
def __setstate__(self, state):
|
920
|
+
...
|
921
|
+
...
|
922
|
+
|
923
|
+
class Step(MetaflowObject, metaclass=type):
|
924
|
+
@property
|
925
|
+
def task(self) -> typing.Optional[Task]:
|
926
|
+
"""
|
927
|
+
Returns a Task object belonging to this step.
|
928
|
+
|
929
|
+
This is useful when the step only contains one task (a linear step for example).
|
930
|
+
|
931
|
+
Returns
|
932
|
+
-------
|
933
|
+
Task
|
934
|
+
A task in the step
|
935
|
+
"""
|
936
|
+
...
|
937
|
+
def tasks(self, *tags: str) -> typing.Iterable[Task]:
|
938
|
+
"""
|
939
|
+
[Legacy function - do not use]
|
940
|
+
|
941
|
+
Returns an iterator over all `Task` objects in the step. This is an alias
|
942
|
+
to iterating the object itself, i.e.
|
943
|
+
```
|
944
|
+
list(Step(...)) == list(Step(...).tasks())
|
945
|
+
```
|
946
|
+
|
947
|
+
Parameters
|
948
|
+
----------
|
949
|
+
tags : str
|
950
|
+
No op (legacy functionality)
|
951
|
+
|
952
|
+
Yields
|
953
|
+
------
|
954
|
+
Task
|
955
|
+
`Task` objects in this step.
|
956
|
+
"""
|
957
|
+
...
|
958
|
+
@property
|
959
|
+
def control_task(self) -> typing.Optional[Task]:
|
960
|
+
"""
|
961
|
+
[Unpublished API - use with caution!]
|
962
|
+
|
963
|
+
Returns a Control Task object belonging to this step.
|
964
|
+
This is useful when the step only contains one control task.
|
965
|
+
|
966
|
+
Returns
|
967
|
+
-------
|
968
|
+
Task
|
969
|
+
A control task in the step
|
970
|
+
"""
|
971
|
+
...
|
972
|
+
def control_tasks(self, *tags: str) -> typing.Iterator[Task]:
|
973
|
+
"""
|
974
|
+
[Unpublished API - use with caution!]
|
975
|
+
|
976
|
+
Returns an iterator over all the control tasks in the step.
|
977
|
+
An optional filter is available that allows you to filter on tags. The
|
978
|
+
control tasks returned if the filter is specified will contain all the
|
979
|
+
tags specified.
|
980
|
+
Parameters
|
981
|
+
----------
|
982
|
+
tags : str
|
983
|
+
Tags to match
|
984
|
+
|
985
|
+
Yields
|
986
|
+
------
|
987
|
+
Task
|
988
|
+
Control Task objects for this step
|
989
|
+
"""
|
990
|
+
...
|
991
|
+
def __iter__(self) -> typing.Iterator[Task]:
|
992
|
+
"""
|
993
|
+
Iterate over all children Task of this Step
|
994
|
+
|
995
|
+
Yields
|
996
|
+
------
|
997
|
+
Task
|
998
|
+
A Task in this Step
|
999
|
+
"""
|
1000
|
+
...
|
1001
|
+
def __getitem__(self, task_id: str) -> Task:
|
1002
|
+
"""
|
1003
|
+
Returns the Task object with the task ID 'task_id'
|
1004
|
+
|
1005
|
+
Parameters
|
1006
|
+
----------
|
1007
|
+
task_id : str
|
1008
|
+
Task ID
|
1009
|
+
|
1010
|
+
Returns
|
1011
|
+
-------
|
1012
|
+
Task
|
1013
|
+
Task for this task ID in this Step
|
1014
|
+
|
1015
|
+
Raises
|
1016
|
+
------
|
1017
|
+
KeyError
|
1018
|
+
If the task_id does not identify a valid Task object
|
1019
|
+
"""
|
1020
|
+
...
|
1021
|
+
def __getstate__(self):
|
1022
|
+
...
|
1023
|
+
def __setstate__(self, state):
|
1024
|
+
...
|
1025
|
+
@property
|
1026
|
+
def finished_at(self) -> typing.Optional[datetime.datetime]:
|
1027
|
+
"""
|
1028
|
+
Returns the datetime object of when the step finished (successfully or not).
|
1029
|
+
|
1030
|
+
A step is considered finished when all the tasks that belong to it have
|
1031
|
+
finished. This call will return None if the step has not finished
|
1032
|
+
|
1033
|
+
Returns
|
1034
|
+
-------
|
1035
|
+
datetime
|
1036
|
+
Datetime of when the step finished
|
1037
|
+
"""
|
1038
|
+
...
|
1039
|
+
@property
|
1040
|
+
def environment_info(self) -> typing.Optional[typing.Dict[str, typing.Any]]:
|
1041
|
+
"""
|
1042
|
+
Returns information about the environment that was used to execute this step. As an
|
1043
|
+
example, if the Conda environment is selected, this will return information about the
|
1044
|
+
dependencies that were used in the environment.
|
1045
|
+
|
1046
|
+
This environment information is only available for steps that have tasks
|
1047
|
+
for which the code package has been saved.
|
1048
|
+
|
1049
|
+
Returns
|
1050
|
+
-------
|
1051
|
+
Dict[str, Any], optional
|
1052
|
+
Dictionary describing the environment
|
1053
|
+
"""
|
1054
|
+
...
|
1055
|
+
...
|
1056
|
+
|
1057
|
+
class Run(MetaflowObject, metaclass=type):
|
1058
|
+
def steps(self, *tags: str) -> typing.Iterator[Step]:
|
1059
|
+
"""
|
1060
|
+
[Legacy function - do not use]
|
1061
|
+
|
1062
|
+
Returns an iterator over all `Step` objects in the step. This is an alias
|
1063
|
+
to iterating the object itself, i.e.
|
1064
|
+
```
|
1065
|
+
list(Run(...)) == list(Run(...).steps())
|
1066
|
+
```
|
1067
|
+
|
1068
|
+
Parameters
|
1069
|
+
----------
|
1070
|
+
tags : str
|
1071
|
+
No op (legacy functionality)
|
1072
|
+
|
1073
|
+
Yields
|
1074
|
+
------
|
1075
|
+
Step
|
1076
|
+
`Step` objects in this run.
|
1077
|
+
"""
|
1078
|
+
...
|
1079
|
+
@property
|
1080
|
+
def code(self) -> typing.Optional[MetaflowCode]:
|
1081
|
+
"""
|
1082
|
+
Returns the MetaflowCode object for this run, if present.
|
1083
|
+
Code is packed if atleast one `Step` runs remotely, else None is returned.
|
1084
|
+
|
1085
|
+
Returns
|
1086
|
+
-------
|
1087
|
+
MetaflowCode, optional
|
1088
|
+
Code package for this run
|
1089
|
+
"""
|
1090
|
+
...
|
1091
|
+
@property
|
1092
|
+
def data(self) -> typing.Optional[MetaflowData]:
|
1093
|
+
"""
|
1094
|
+
Returns a container of data artifacts produced by this run.
|
1095
|
+
|
1096
|
+
You can access data produced by this run as follows:
|
1097
|
+
```
|
1098
|
+
print(run.data.my_var)
|
1099
|
+
```
|
1100
|
+
This is a shorthand for `run['end'].task.data`. If the 'end' step has not yet
|
1101
|
+
executed, returns None.
|
1102
|
+
|
1103
|
+
Returns
|
1104
|
+
-------
|
1105
|
+
MetaflowData, optional
|
1106
|
+
Container of all artifacts produced by this task
|
1107
|
+
"""
|
1108
|
+
...
|
1109
|
+
@property
|
1110
|
+
def successful(self) -> bool:
|
1111
|
+
"""
|
1112
|
+
Indicates whether or not the run completed successfully.
|
1113
|
+
|
1114
|
+
A run is successful if its 'end' step is successful.
|
1115
|
+
|
1116
|
+
Returns
|
1117
|
+
-------
|
1118
|
+
bool
|
1119
|
+
True if the run completed successfully and False otherwise
|
1120
|
+
"""
|
1121
|
+
...
|
1122
|
+
@property
|
1123
|
+
def finished(self) -> bool:
|
1124
|
+
"""
|
1125
|
+
Indicates whether or not the run completed.
|
1126
|
+
|
1127
|
+
A run completed if its 'end' step completed.
|
1128
|
+
|
1129
|
+
Returns
|
1130
|
+
-------
|
1131
|
+
bool
|
1132
|
+
True if the run completed and False otherwise
|
1133
|
+
"""
|
1134
|
+
...
|
1135
|
+
@property
|
1136
|
+
def finished_at(self) -> typing.Optional[datetime.datetime]:
|
1137
|
+
"""
|
1138
|
+
Returns the datetime object of when the run finished (successfully or not).
|
1139
|
+
|
1140
|
+
The completion time of a run is the same as the completion time of its 'end' step.
|
1141
|
+
If the 'end' step has not completed, returns None.
|
1142
|
+
|
1143
|
+
Returns
|
1144
|
+
-------
|
1145
|
+
datetime, optional
|
1146
|
+
Datetime of when the run finished
|
1147
|
+
"""
|
1148
|
+
...
|
1149
|
+
@property
|
1150
|
+
def end_task(self) -> typing.Optional[Task]:
|
1151
|
+
"""
|
1152
|
+
Returns the Task corresponding to the 'end' step.
|
1153
|
+
|
1154
|
+
This returns None if the end step does not yet exist.
|
1155
|
+
|
1156
|
+
Returns
|
1157
|
+
-------
|
1158
|
+
Task, optional
|
1159
|
+
The 'end' task
|
1160
|
+
"""
|
1161
|
+
...
|
1162
|
+
def add_tag(self, tag: str):
|
1163
|
+
"""
|
1164
|
+
Add a tag to this `Run`.
|
1165
|
+
|
1166
|
+
Note that if the tag is already a system tag, it is not added as a user tag,
|
1167
|
+
and no error is thrown.
|
1168
|
+
|
1169
|
+
Parameters
|
1170
|
+
----------
|
1171
|
+
tag : str
|
1172
|
+
Tag to add.
|
1173
|
+
"""
|
1174
|
+
...
|
1175
|
+
def add_tags(self, tags: typing.Iterable[str]):
|
1176
|
+
"""
|
1177
|
+
Add one or more tags to this `Run`.
|
1178
|
+
|
1179
|
+
Note that if any tag is already a system tag, it is not added as a user tag
|
1180
|
+
and no error is thrown.
|
1181
|
+
|
1182
|
+
Parameters
|
1183
|
+
----------
|
1184
|
+
tags : Iterable[str]
|
1185
|
+
Tags to add.
|
1186
|
+
"""
|
1187
|
+
...
|
1188
|
+
def remove_tag(self, tag: str):
|
1189
|
+
"""
|
1190
|
+
Remove one tag from this `Run`.
|
1191
|
+
|
1192
|
+
Removing a system tag is an error. Removing a non-existent
|
1193
|
+
user tag is a no-op.
|
1194
|
+
|
1195
|
+
Parameters
|
1196
|
+
----------
|
1197
|
+
tag : str
|
1198
|
+
Tag to remove.
|
1199
|
+
"""
|
1200
|
+
...
|
1201
|
+
def remove_tags(self, tags: typing.Iterable[str]):
|
1202
|
+
"""
|
1203
|
+
Remove one or more tags to this `Run`.
|
1204
|
+
|
1205
|
+
Removing a system tag will result in an error. Removing a non-existent
|
1206
|
+
user tag is a no-op.
|
1207
|
+
|
1208
|
+
Parameters
|
1209
|
+
----------
|
1210
|
+
tags : Iterable[str]
|
1211
|
+
Tags to remove.
|
1212
|
+
"""
|
1213
|
+
...
|
1214
|
+
def replace_tag(self, tag_to_remove: str, tag_to_add: str):
|
1215
|
+
"""
|
1216
|
+
Remove a tag and add a tag atomically. Removal is done first.
|
1217
|
+
The rules for `Run.add_tag` and `Run.remove_tag` also apply here.
|
1218
|
+
|
1219
|
+
Parameters
|
1220
|
+
----------
|
1221
|
+
tag_to_remove : str
|
1222
|
+
Tag to remove.
|
1223
|
+
tag_to_add : str
|
1224
|
+
Tag to add.
|
1225
|
+
"""
|
1226
|
+
...
|
1227
|
+
def replace_tags(self, tags_to_remove: typing.Iterable[str], tags_to_add: typing.Iterable[str]):
|
1228
|
+
"""
|
1229
|
+
Remove and add tags atomically; the removal is done first.
|
1230
|
+
The rules for `Run.add_tag` and `Run.remove_tag` also apply here.
|
1231
|
+
|
1232
|
+
Parameters
|
1233
|
+
----------
|
1234
|
+
tags_to_remove : Iterable[str]
|
1235
|
+
Tags to remove.
|
1236
|
+
tags_to_add : Iterable[str]
|
1237
|
+
Tags to add.
|
1238
|
+
"""
|
1239
|
+
...
|
1240
|
+
def __iter__(self) -> typing.Iterator[Step]:
|
1241
|
+
"""
|
1242
|
+
Iterate over all children Step of this Run
|
1243
|
+
|
1244
|
+
Yields
|
1245
|
+
------
|
1246
|
+
Step
|
1247
|
+
A Step in this Run
|
1248
|
+
"""
|
1249
|
+
...
|
1250
|
+
def __getitem__(self, name: str) -> Step:
|
1251
|
+
"""
|
1252
|
+
Returns the Step object with the step name 'name'
|
1253
|
+
|
1254
|
+
Parameters
|
1255
|
+
----------
|
1256
|
+
name : str
|
1257
|
+
Step name
|
1258
|
+
|
1259
|
+
Returns
|
1260
|
+
-------
|
1261
|
+
Step
|
1262
|
+
Step for this step name in this Run
|
1263
|
+
|
1264
|
+
Raises
|
1265
|
+
------
|
1266
|
+
KeyError
|
1267
|
+
If the name does not identify a valid Step object
|
1268
|
+
"""
|
1269
|
+
...
|
1270
|
+
def __getstate__(self):
|
1271
|
+
...
|
1272
|
+
def __setstate__(self, state):
|
1273
|
+
...
|
1274
|
+
@property
|
1275
|
+
def trigger(self) -> typing.Optional[metaflow.events.Trigger]:
|
1276
|
+
"""
|
1277
|
+
Returns a container of events that triggered this run.
|
1278
|
+
|
1279
|
+
This returns None if the run was not triggered by any events.
|
1280
|
+
|
1281
|
+
Returns
|
1282
|
+
-------
|
1283
|
+
Trigger, optional
|
1284
|
+
Container of triggering events
|
1285
|
+
"""
|
1286
|
+
...
|
1287
|
+
...
|
1288
|
+
|
1289
|
+
class Flow(MetaflowObject, metaclass=type):
|
1290
|
+
def __init__(self, *args, **kwargs):
|
1291
|
+
...
|
1292
|
+
@property
|
1293
|
+
def latest_run(self) -> typing.Optional[Run]:
|
1294
|
+
"""
|
1295
|
+
Returns the latest run (either in progress or completed) of this flow.
|
1296
|
+
|
1297
|
+
Note that an in-progress run may be returned by this call. Use latest_successful_run
|
1298
|
+
to get an object representing a completed successful run.
|
1299
|
+
|
1300
|
+
Returns
|
1301
|
+
-------
|
1302
|
+
Run, optional
|
1303
|
+
Latest run of this flow
|
1304
|
+
"""
|
1305
|
+
...
|
1306
|
+
@property
|
1307
|
+
def latest_successful_run(self) -> typing.Optional[Run]:
|
1308
|
+
"""
|
1309
|
+
Returns the latest successful run of this flow.
|
1310
|
+
|
1311
|
+
Returns
|
1312
|
+
-------
|
1313
|
+
Run, optional
|
1314
|
+
Latest successful run of this flow
|
1315
|
+
"""
|
1316
|
+
...
|
1317
|
+
def runs(self, *tags: str) -> typing.Iterator[Run]:
|
1318
|
+
"""
|
1319
|
+
Returns an iterator over all `Run`s of this flow.
|
1320
|
+
|
1321
|
+
An optional filter is available that allows you to filter on tags.
|
1322
|
+
If multiple tags are specified, only runs that have all the
|
1323
|
+
specified tags are returned.
|
1324
|
+
|
1325
|
+
Parameters
|
1326
|
+
----------
|
1327
|
+
tags : str
|
1328
|
+
Tags to match.
|
1329
|
+
|
1330
|
+
Yields
|
1331
|
+
------
|
1332
|
+
Run
|
1333
|
+
`Run` objects in this flow.
|
1334
|
+
"""
|
1335
|
+
...
|
1336
|
+
def __iter__(self) -> typing.Iterator[Task]:
|
1337
|
+
"""
|
1338
|
+
Iterate over all children Run of this Flow.
|
1339
|
+
|
1340
|
+
Note that only runs in the current namespace are returned unless
|
1341
|
+
_namespace_check is False
|
1342
|
+
|
1343
|
+
Yields
|
1344
|
+
------
|
1345
|
+
Run
|
1346
|
+
A Run in this Flow
|
1347
|
+
"""
|
1348
|
+
...
|
1349
|
+
def __getitem__(self, run_id: str) -> Run:
|
1350
|
+
"""
|
1351
|
+
Returns the Run object with the run ID 'run_id'
|
1352
|
+
|
1353
|
+
Parameters
|
1354
|
+
----------
|
1355
|
+
run_id : str
|
1356
|
+
Run OD
|
1357
|
+
|
1358
|
+
Returns
|
1359
|
+
-------
|
1360
|
+
Run
|
1361
|
+
Run for this run ID in this Flow
|
1362
|
+
|
1363
|
+
Raises
|
1364
|
+
------
|
1365
|
+
KeyError
|
1366
|
+
If the run_id does not identify a valid Run object
|
1367
|
+
"""
|
1368
|
+
...
|
1369
|
+
def __getstate__(self):
|
1370
|
+
...
|
1371
|
+
def __setstate__(self, state):
|
1372
|
+
...
|
1373
|
+
...
|
1374
|
+
|
1375
|
+
class Metaflow(object, metaclass=type):
|
1376
|
+
def __init__(self):
|
1377
|
+
...
|
1378
|
+
@property
|
1379
|
+
def flows(self) -> typing.List[Flow]:
|
1380
|
+
"""
|
1381
|
+
Returns a list of all the flows present.
|
1382
|
+
|
1383
|
+
Only flows present in the set namespace are returned. A flow is present in a namespace if
|
1384
|
+
it has at least one run that is in the namespace.
|
1385
|
+
|
1386
|
+
Returns
|
1387
|
+
-------
|
1388
|
+
List[Flow]
|
1389
|
+
List of all flows present.
|
1390
|
+
"""
|
1391
|
+
...
|
1392
|
+
def __iter__(self) -> typing.Iterator[Flow]:
|
1393
|
+
"""
|
1394
|
+
Iterator over all flows present.
|
1395
|
+
|
1396
|
+
Only flows present in the set namespace are returned. A flow is present in a
|
1397
|
+
namespace if it has at least one run that is in the namespace.
|
1398
|
+
|
1399
|
+
Yields
|
1400
|
+
-------
|
1401
|
+
Flow
|
1402
|
+
A Flow present in the Metaflow universe.
|
1403
|
+
"""
|
1404
|
+
...
|
1405
|
+
def __str__(self) -> str:
|
1406
|
+
...
|
1407
|
+
def __getitem__(self, name: str) -> Flow:
|
1408
|
+
"""
|
1409
|
+
Returns a specific flow by name.
|
1410
|
+
|
1411
|
+
The flow will only be returned if it is present in the current namespace.
|
1412
|
+
|
1413
|
+
Parameters
|
1414
|
+
----------
|
1415
|
+
name : str
|
1416
|
+
Name of the Flow
|
1417
|
+
|
1418
|
+
Returns
|
1419
|
+
-------
|
1420
|
+
Flow
|
1421
|
+
Flow with the given name.
|
1422
|
+
"""
|
1423
|
+
...
|
1424
|
+
...
|
1425
|
+
|