ob-metaflow-stubs 6.0.3.103rc3__py2.py3-none-any.whl → 6.0.3.105__py2.py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- metaflow-stubs/__init__.pyi +836 -498
- metaflow-stubs/cards.pyi +211 -5
- metaflow-stubs/cli.pyi +23 -3
- metaflow-stubs/client/__init__.pyi +129 -4
- metaflow-stubs/client/core.pyi +227 -7
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +5 -2
- metaflow-stubs/events.pyi +21 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +71 -5
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +157 -4
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata/metadata.pyi +8 -2
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +39 -36
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +70 -4
- metaflow-stubs/plugins/__init__.pyi +13 -2
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +80 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +5 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +45 -3
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +45 -3
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +16 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +18 -5
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +98 -7
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +33 -5
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +50 -5
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +103 -3
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +15 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +21 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +49 -4
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +21 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +62 -4
- metaflow-stubs/plugins/cards/card_client.pyi +33 -2
- metaflow-stubs/plugins/cards/card_creator.pyi +5 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +8 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +52 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +28 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +42 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +28 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +183 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +5 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +36 -3
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +55 -2
- metaflow-stubs/plugins/cards/exception.pyi +8 -2
- metaflow-stubs/plugins/catch_decorator.pyi +20 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +63 -3
- metaflow-stubs/plugins/datatools/local.pyi +16 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +72 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +82 -5
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +10 -2
- metaflow-stubs/plugins/events_decorator.pyi +106 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +23 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +15 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +12 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +65 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +100 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +5 -2
- metaflow-stubs/plugins/logs_cli.pyi +4 -4
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +29 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +59 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +45 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +33 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +33 -2
- metaflow-stubs/plugins/retry_decorator.pyi +21 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +11 -2
- metaflow-stubs/plugins/storage_executor.pyi +6 -2
- metaflow-stubs/plugins/tag_cli.pyi +36 -5
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +9 -2
- metaflow-stubs/plugins/timeout_decorator.pyi +23 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +71 -132
- metaflow-stubs/runner/metaflow_runner.pyi +117 -9
- metaflow-stubs/runner/nbdeploy.pyi +66 -2
- metaflow-stubs/runner/nbrun.pyi +79 -2
- metaflow-stubs/runner/subprocess_manager.pyi +16 -4
- metaflow-stubs/runner/utils.pyi +32 -2
- metaflow-stubs/system/__init__.pyi +3 -3
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +3 -3
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- {ob_metaflow_stubs-6.0.3.103rc3.dist-info → ob_metaflow_stubs-6.0.3.105.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.105.dist-info/RECORD +140 -0
- ob_metaflow_stubs-6.0.3.103rc3.dist-info/RECORD +0 -140
- {ob_metaflow_stubs-6.0.3.103rc3.dist-info → ob_metaflow_stubs-6.0.3.105.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.103rc3.dist-info → ob_metaflow_stubs-6.0.3.105.dist-info}/top_level.txt +0 -0
metaflow-stubs/includefile.pyi
CHANGED
@@ -1,18 +1,18 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.641159 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.parameters
|
12
11
|
import typing
|
13
|
-
import metaflow._vendor.click.types
|
14
12
|
import io
|
13
|
+
import metaflow.parameters
|
15
14
|
import metaflow.plugins.datatools.s3.s3
|
15
|
+
import metaflow._vendor.click.types
|
16
16
|
|
17
17
|
class MetaflowException(Exception, metaclass=type):
|
18
18
|
def __init__(self, msg = "", lineno = None):
|
@@ -22,6 +22,18 @@ class MetaflowException(Exception, metaclass=type):
|
|
22
22
|
...
|
23
23
|
|
24
24
|
class DelayedEvaluationParameter(object, metaclass=type):
|
25
|
+
"""
|
26
|
+
This is a very simple wrapper to allow parameter "conversion" to be delayed until
|
27
|
+
the `_set_constants` function in FlowSpec. Typically, parameters are converted
|
28
|
+
by click when the command line option is processed. For some parameters, like
|
29
|
+
IncludeFile, this is too early as it would mean we would trigger the upload
|
30
|
+
of the file too early. If a parameter converts to a DelayedEvaluationParameter
|
31
|
+
object through the usual click mechanisms, `_set_constants` knows to invoke the
|
32
|
+
__call__ method on that DelayedEvaluationParameter; in that case, the __call__
|
33
|
+
method is invoked without any parameter. The return_str parameter will be used
|
34
|
+
by schedulers when they need to convert DelayedEvaluationParameters to a
|
35
|
+
string to store them
|
36
|
+
"""
|
25
37
|
def __init__(self, name, field, fun):
|
26
38
|
...
|
27
39
|
def __call__(self, return_str = False):
|
@@ -29,6 +41,14 @@ class DelayedEvaluationParameter(object, metaclass=type):
|
|
29
41
|
...
|
30
42
|
|
31
43
|
class DeployTimeField(object, metaclass=type):
|
44
|
+
"""
|
45
|
+
This a wrapper object for a user-defined function that is called
|
46
|
+
at deploy time to populate fields in a Parameter. The wrapper
|
47
|
+
is needed to make Click show the actual value returned by the
|
48
|
+
function instead of a function pointer in its help text. Also, this
|
49
|
+
object curries the context argument for the function, and pretty
|
50
|
+
prints any exceptions that occur during evaluation.
|
51
|
+
"""
|
32
52
|
def __init__(self, parameter_name, parameter_type, field, fun, return_str = True, print_representation = None):
|
33
53
|
...
|
34
54
|
def __call__(self, deploy_time = False):
|
@@ -43,6 +63,49 @@ class DeployTimeField(object, metaclass=type):
|
|
43
63
|
...
|
44
64
|
|
45
65
|
class Parameter(object, metaclass=type):
|
66
|
+
"""
|
67
|
+
Defines a parameter for a flow.
|
68
|
+
|
69
|
+
Parameters must be instantiated as class variables in flow classes, e.g.
|
70
|
+
```
|
71
|
+
class MyFlow(FlowSpec):
|
72
|
+
param = Parameter('myparam')
|
73
|
+
```
|
74
|
+
in this case, the parameter is specified on the command line as
|
75
|
+
```
|
76
|
+
python myflow.py run --myparam=5
|
77
|
+
```
|
78
|
+
and its value is accessible through a read-only artifact like this:
|
79
|
+
```
|
80
|
+
print(self.param == 5)
|
81
|
+
```
|
82
|
+
Note that the user-visible parameter name, `myparam` above, can be
|
83
|
+
different from the artifact name, `param` above.
|
84
|
+
|
85
|
+
The parameter value is converted to a Python type based on the `type`
|
86
|
+
argument or to match the type of `default`, if it is set.
|
87
|
+
|
88
|
+
Parameters
|
89
|
+
----------
|
90
|
+
name : str
|
91
|
+
User-visible parameter name.
|
92
|
+
default : str or float or int or bool or `JSONType` or a function.
|
93
|
+
Default value for the parameter. Use a special `JSONType` class to
|
94
|
+
indicate that the value must be a valid JSON object. A function
|
95
|
+
implies that the parameter corresponds to a *deploy-time parameter*.
|
96
|
+
The type of the default value is used as the parameter `type`.
|
97
|
+
type : Type, default None
|
98
|
+
If `default` is not specified, define the parameter type. Specify
|
99
|
+
one of `str`, `float`, `int`, `bool`, or `JSONType`. If None, defaults
|
100
|
+
to the type of `default` or `str` if none specified.
|
101
|
+
help : str, optional
|
102
|
+
Help text to show in `run --help`.
|
103
|
+
required : bool, default False
|
104
|
+
Require that the user specified a value for the parameter.
|
105
|
+
`required=True` implies that the `default` is not used.
|
106
|
+
show_default : bool, default True
|
107
|
+
If True, show the default value in the help text.
|
108
|
+
"""
|
46
109
|
def __init__(self, name: str, default: typing.Union[str, float, int, bool, typing.Dict[str, typing.Any], typing.Callable[[], typing.Union[str, float, int, bool, typing.Dict[str, typing.Any]]], None] = None, type: typing.Union[typing.Type[str], typing.Type[float], typing.Type[int], typing.Type[bool], metaflow.parameters.JSONTypeClass, None] = None, help: typing.Optional[str] = None, required: bool = False, show_default: bool = True, **kwargs: typing.Dict[str, typing.Any]):
|
47
110
|
...
|
48
111
|
def __repr__(self):
|
@@ -61,6 +124,9 @@ class Parameter(object, metaclass=type):
|
|
61
124
|
...
|
62
125
|
|
63
126
|
class ParameterContext(tuple, metaclass=type):
|
127
|
+
"""
|
128
|
+
ParameterContext(flow_name, user_name, parameter_name, logger, ds_type)
|
129
|
+
"""
|
64
130
|
@staticmethod
|
65
131
|
def __new__(_cls, flow_name: str, user_name: str, parameter_name: str, logger: typing.Callable[..., None], ds_type: str):
|
66
132
|
"""
|
@@ -82,6 +148,13 @@ class ParameterContext(tuple, metaclass=type):
|
|
82
148
|
...
|
83
149
|
|
84
150
|
class Local(object, metaclass=type):
|
151
|
+
"""
|
152
|
+
This class allows you to access the local filesystem in a way similar to the S3 datatools
|
153
|
+
client. It is a stripped down version for now and only implements the functionality needed
|
154
|
+
for this use case.
|
155
|
+
|
156
|
+
In the future, we may want to allow it to be used in a way similar to the S3() client.
|
157
|
+
"""
|
85
158
|
@classmethod
|
86
159
|
def get_root_from_config(cls, echo, create_on_absent = True):
|
87
160
|
...
|
@@ -104,6 +177,59 @@ class Local(object, metaclass=type):
|
|
104
177
|
...
|
105
178
|
|
106
179
|
class S3(object, metaclass=type):
|
180
|
+
"""
|
181
|
+
The Metaflow S3 client.
|
182
|
+
|
183
|
+
This object manages the connection to S3 and a temporary diretory that is used
|
184
|
+
to download objects. Note that in most cases when the data fits in memory, no local
|
185
|
+
disk IO is needed as operations are cached by the operating system, which makes
|
186
|
+
operations fast as long as there is enough memory available.
|
187
|
+
|
188
|
+
The easiest way is to use this object as a context manager:
|
189
|
+
```
|
190
|
+
with S3() as s3:
|
191
|
+
data = [obj.blob for obj in s3.get_many(urls)]
|
192
|
+
print(data)
|
193
|
+
```
|
194
|
+
The context manager takes care of creating and deleting a temporary directory
|
195
|
+
automatically. Without a context manager, you must call `.close()` to delete
|
196
|
+
the directory explicitly:
|
197
|
+
```
|
198
|
+
s3 = S3()
|
199
|
+
data = [obj.blob for obj in s3.get_many(urls)]
|
200
|
+
s3.close()
|
201
|
+
```
|
202
|
+
You can customize the location of the temporary directory with `tmproot`. It
|
203
|
+
defaults to the current working directory.
|
204
|
+
|
205
|
+
To make it easier to deal with object locations, the client can be initialized
|
206
|
+
with an S3 path prefix. There are three ways to handle locations:
|
207
|
+
|
208
|
+
1. Use a `metaflow.Run` object or `self`, e.g. `S3(run=self)` which
|
209
|
+
initializes the prefix with the global `DATATOOLS_S3ROOT` path, combined
|
210
|
+
with the current run ID. This mode makes it easy to version data based
|
211
|
+
on the run ID consistently. You can use the `bucket` and `prefix` to
|
212
|
+
override parts of `DATATOOLS_S3ROOT`.
|
213
|
+
|
214
|
+
2. Specify an S3 prefix explicitly with `s3root`,
|
215
|
+
e.g. `S3(s3root='s3://mybucket/some/path')`.
|
216
|
+
|
217
|
+
3. Specify nothing, i.e. `S3()`, in which case all operations require
|
218
|
+
a full S3 url prefixed with `s3://`.
|
219
|
+
|
220
|
+
Parameters
|
221
|
+
----------
|
222
|
+
tmproot : str, default: '.'
|
223
|
+
Where to store the temporary directory.
|
224
|
+
bucket : str, optional
|
225
|
+
Override the bucket from `DATATOOLS_S3ROOT` when `run` is specified.
|
226
|
+
prefix : str, optional
|
227
|
+
Override the path from `DATATOOLS_S3ROOT` when `run` is specified.
|
228
|
+
run : FlowSpec or Run, optional
|
229
|
+
Derive path prefix from the current or a past run ID, e.g. S3(run=self).
|
230
|
+
s3root : str, optional
|
231
|
+
If `run` is not specified, use this as the S3 prefix.
|
232
|
+
"""
|
107
233
|
@classmethod
|
108
234
|
def get_root_from_config(cls, echo, create_on_absent = True):
|
109
235
|
...
|
@@ -469,6 +595,33 @@ class FilePathClass(metaflow._vendor.click.types.ParamType, metaclass=type):
|
|
469
595
|
...
|
470
596
|
|
471
597
|
class IncludeFile(metaflow.parameters.Parameter, metaclass=type):
|
598
|
+
"""
|
599
|
+
Includes a local file as a parameter for the flow.
|
600
|
+
|
601
|
+
`IncludeFile` behaves like `Parameter` except that it reads its value from a file instead of
|
602
|
+
the command line. The user provides a path to a file on the command line. The file contents
|
603
|
+
are saved as a read-only artifact which is available in all steps of the flow.
|
604
|
+
|
605
|
+
Parameters
|
606
|
+
----------
|
607
|
+
name : str
|
608
|
+
User-visible parameter name.
|
609
|
+
default : Union[str, Callable[ParameterContext, str]]
|
610
|
+
Default path to a local file. A function
|
611
|
+
implies that the parameter corresponds to a *deploy-time parameter*.
|
612
|
+
is_text : bool, default True
|
613
|
+
Convert the file contents to a string using the provided `encoding`.
|
614
|
+
If False, the artifact is stored in `bytes`.
|
615
|
+
encoding : str, optional, default 'utf-8'
|
616
|
+
Use this encoding to decode the file contexts if `is_text=True`.
|
617
|
+
required : bool, default False
|
618
|
+
Require that the user specified a value for the parameter.
|
619
|
+
`required=True` implies that the `default` is not used.
|
620
|
+
help : str, optional
|
621
|
+
Help text to show in `run --help`.
|
622
|
+
show_default : bool, default True
|
623
|
+
If True, show the default value in the help text.
|
624
|
+
"""
|
472
625
|
def __init__(self, name: str, required: bool = False, is_text: bool = True, encoding: str = "utf-8", help: typing.Optional[str] = None, **kwargs: typing.Dict[str, str]):
|
473
626
|
...
|
474
627
|
def load_parameter(self, v):
|
metaflow-stubs/info_file.pyi
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.625444 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.678535 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -28,6 +28,9 @@ def validate_tag(tag):
|
|
28
28
|
...
|
29
29
|
|
30
30
|
class DataArtifact(tuple, metaclass=type):
|
31
|
+
"""
|
32
|
+
DataArtifact(name, ds_type, ds_root, url, type, sha)
|
33
|
+
"""
|
31
34
|
@staticmethod
|
32
35
|
def __new__(_cls, name, ds_type, ds_root, url, type, sha):
|
33
36
|
"""
|
@@ -47,6 +50,9 @@ class DataArtifact(tuple, metaclass=type):
|
|
47
50
|
...
|
48
51
|
|
49
52
|
class MetaDatum(tuple, metaclass=type):
|
53
|
+
"""
|
54
|
+
MetaDatum(field, value, type, tags)
|
55
|
+
"""
|
50
56
|
@staticmethod
|
51
57
|
def __new__(_cls, field, value, type, tags):
|
52
58
|
"""
|
metaflow-stubs/metadata/util.pyi
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.713890 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.627320 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,26 +1,29 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.725041 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import
|
12
|
-
import metaflow
|
11
|
+
import metaflow.metaflow_current
|
13
12
|
import typing
|
13
|
+
import metaflow
|
14
|
+
import metaflow.events
|
14
15
|
import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core
|
15
16
|
import metaflow.plugins.cards.component_serializer
|
16
|
-
import
|
17
|
-
import metaflow.metaflow_current
|
17
|
+
import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator
|
18
18
|
|
19
19
|
TYPE_CHECKING: bool
|
20
20
|
|
21
21
|
TEMPDIR: str
|
22
22
|
|
23
23
|
class Parallel(tuple, metaclass=type):
|
24
|
+
"""
|
25
|
+
Parallel(main_ip, num_nodes, node_index, control_task_id)
|
26
|
+
"""
|
24
27
|
@staticmethod
|
25
28
|
def __new__(_cls, main_ip, num_nodes, node_index, control_task_id):
|
26
29
|
"""
|
@@ -223,31 +226,21 @@ class Current(object, metaclass=type):
|
|
223
226
|
def graph(_):
|
224
227
|
...
|
225
228
|
@property
|
226
|
-
def
|
229
|
+
def card(self) -> "metaflow.plugins.cards.component_serializer.CardComponentCollector":
|
227
230
|
"""
|
228
|
-
(only in the presence of the @
|
231
|
+
(only in the presence of the @card decorator)
|
232
|
+
|
233
|
+
The `@card` decorator makes the cards available through the `current.card`
|
234
|
+
object. If multiple `@card` decorators are present, you can add an `ID` to
|
235
|
+
distinguish between them using `@card(id=ID)` as the decorator. You will then
|
236
|
+
be able to access that specific card using `current.card[ID].
|
229
237
|
|
238
|
+
Methods available are `append` and `extend`
|
230
239
|
|
231
240
|
Returns
|
232
241
|
-------
|
233
|
-
|
234
|
-
|
235
|
-
- main_ip : str
|
236
|
-
The IP address of the control task.
|
237
|
-
- num_nodes : int
|
238
|
-
The total number of tasks created by @parallel
|
239
|
-
- node_index : int
|
240
|
-
The index of the current task in all the @parallel tasks.
|
241
|
-
- control_task_id : Optional[str]
|
242
|
-
The task ID of the control task. Available to all tasks.
|
243
|
-
"""
|
244
|
-
...
|
245
|
-
@property
|
246
|
-
def is_parallel(self) -> bool:
|
247
|
-
"""
|
248
|
-
(only in the presence of the @parallel decorator)
|
249
|
-
|
250
|
-
True if the current step is a @parallel step.
|
242
|
+
CardComponentCollector
|
243
|
+
The or one of the cards attached to this step.
|
251
244
|
"""
|
252
245
|
...
|
253
246
|
@property
|
@@ -373,21 +366,31 @@ class Current(object, metaclass=type):
|
|
373
366
|
"""
|
374
367
|
...
|
375
368
|
@property
|
376
|
-
def
|
369
|
+
def parallel(self) -> "metaflow.metaflow_current.Parallel":
|
377
370
|
"""
|
378
|
-
(only in the presence of the @
|
379
|
-
|
380
|
-
The `@card` decorator makes the cards available through the `current.card`
|
381
|
-
object. If multiple `@card` decorators are present, you can add an `ID` to
|
382
|
-
distinguish between them using `@card(id=ID)` as the decorator. You will then
|
383
|
-
be able to access that specific card using `current.card[ID].
|
371
|
+
(only in the presence of the @parallel decorator)
|
384
372
|
|
385
|
-
Methods available are `append` and `extend`
|
386
373
|
|
387
374
|
Returns
|
388
375
|
-------
|
389
|
-
|
390
|
-
|
376
|
+
Parallel
|
377
|
+
`namedtuple` with the following fields:
|
378
|
+
- main_ip : str
|
379
|
+
The IP address of the control task.
|
380
|
+
- num_nodes : int
|
381
|
+
The total number of tasks created by @parallel
|
382
|
+
- node_index : int
|
383
|
+
The index of the current task in all the @parallel tasks.
|
384
|
+
- control_task_id : Optional[str]
|
385
|
+
The task ID of the control task. Available to all tasks.
|
386
|
+
"""
|
387
|
+
...
|
388
|
+
@property
|
389
|
+
def is_parallel(self) -> bool:
|
390
|
+
"""
|
391
|
+
(only in the presence of the @parallel decorator)
|
392
|
+
|
393
|
+
True if the current step is a @parallel step.
|
391
394
|
"""
|
392
395
|
...
|
393
396
|
@property
|
metaflow-stubs/mflog/mflog.pyi
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.678833 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.627727 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
metaflow-stubs/parameters.pyi
CHANGED
@@ -1,17 +1,17 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.628937 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import
|
11
|
+
import typing
|
12
12
|
import metaflow.exception
|
13
13
|
import metaflow.parameters
|
14
|
-
import
|
14
|
+
import metaflow._vendor.click.types
|
15
15
|
|
16
16
|
class ParameterFieldFailed(metaflow.exception.MetaflowException, metaclass=type):
|
17
17
|
def __init__(self, name, field):
|
@@ -31,6 +31,9 @@ class MetaflowException(Exception, metaclass=type):
|
|
31
31
|
...
|
32
32
|
|
33
33
|
class ParameterContext(tuple, metaclass=type):
|
34
|
+
"""
|
35
|
+
ParameterContext(flow_name, user_name, parameter_name, logger, ds_type)
|
36
|
+
"""
|
34
37
|
@staticmethod
|
35
38
|
def __new__(_cls, flow_name: str, user_name: str, parameter_name: str, logger: typing.Callable[..., None], ds_type: str):
|
36
39
|
"""
|
@@ -71,6 +74,14 @@ class JSONTypeClass(metaflow._vendor.click.types.ParamType, metaclass=type):
|
|
71
74
|
...
|
72
75
|
|
73
76
|
class DeployTimeField(object, metaclass=type):
|
77
|
+
"""
|
78
|
+
This a wrapper object for a user-defined function that is called
|
79
|
+
at deploy time to populate fields in a Parameter. The wrapper
|
80
|
+
is needed to make Click show the actual value returned by the
|
81
|
+
function instead of a function pointer in its help text. Also, this
|
82
|
+
object curries the context argument for the function, and pretty
|
83
|
+
prints any exceptions that occur during evaluation.
|
84
|
+
"""
|
74
85
|
def __init__(self, parameter_name, parameter_type, field, fun, return_str = True, print_representation = None):
|
75
86
|
...
|
76
87
|
def __call__(self, deploy_time = False):
|
@@ -91,6 +102,18 @@ def set_parameter_context(flow_name, echo, datastore):
|
|
91
102
|
...
|
92
103
|
|
93
104
|
class DelayedEvaluationParameter(object, metaclass=type):
|
105
|
+
"""
|
106
|
+
This is a very simple wrapper to allow parameter "conversion" to be delayed until
|
107
|
+
the `_set_constants` function in FlowSpec. Typically, parameters are converted
|
108
|
+
by click when the command line option is processed. For some parameters, like
|
109
|
+
IncludeFile, this is too early as it would mean we would trigger the upload
|
110
|
+
of the file too early. If a parameter converts to a DelayedEvaluationParameter
|
111
|
+
object through the usual click mechanisms, `_set_constants` knows to invoke the
|
112
|
+
__call__ method on that DelayedEvaluationParameter; in that case, the __call__
|
113
|
+
method is invoked without any parameter. The return_str parameter will be used
|
114
|
+
by schedulers when they need to convert DelayedEvaluationParameters to a
|
115
|
+
string to store them
|
116
|
+
"""
|
94
117
|
def __init__(self, name, field, fun):
|
95
118
|
...
|
96
119
|
def __call__(self, return_str = False):
|
@@ -98,6 +121,49 @@ class DelayedEvaluationParameter(object, metaclass=type):
|
|
98
121
|
...
|
99
122
|
|
100
123
|
class Parameter(object, metaclass=type):
|
124
|
+
"""
|
125
|
+
Defines a parameter for a flow.
|
126
|
+
|
127
|
+
Parameters must be instantiated as class variables in flow classes, e.g.
|
128
|
+
```
|
129
|
+
class MyFlow(FlowSpec):
|
130
|
+
param = Parameter('myparam')
|
131
|
+
```
|
132
|
+
in this case, the parameter is specified on the command line as
|
133
|
+
```
|
134
|
+
python myflow.py run --myparam=5
|
135
|
+
```
|
136
|
+
and its value is accessible through a read-only artifact like this:
|
137
|
+
```
|
138
|
+
print(self.param == 5)
|
139
|
+
```
|
140
|
+
Note that the user-visible parameter name, `myparam` above, can be
|
141
|
+
different from the artifact name, `param` above.
|
142
|
+
|
143
|
+
The parameter value is converted to a Python type based on the `type`
|
144
|
+
argument or to match the type of `default`, if it is set.
|
145
|
+
|
146
|
+
Parameters
|
147
|
+
----------
|
148
|
+
name : str
|
149
|
+
User-visible parameter name.
|
150
|
+
default : str or float or int or bool or `JSONType` or a function.
|
151
|
+
Default value for the parameter. Use a special `JSONType` class to
|
152
|
+
indicate that the value must be a valid JSON object. A function
|
153
|
+
implies that the parameter corresponds to a *deploy-time parameter*.
|
154
|
+
The type of the default value is used as the parameter `type`.
|
155
|
+
type : Type, default None
|
156
|
+
If `default` is not specified, define the parameter type. Specify
|
157
|
+
one of `str`, `float`, `int`, `bool`, or `JSONType`. If None, defaults
|
158
|
+
to the type of `default` or `str` if none specified.
|
159
|
+
help : str, optional
|
160
|
+
Help text to show in `run --help`.
|
161
|
+
required : bool, default False
|
162
|
+
Require that the user specified a value for the parameter.
|
163
|
+
`required=True` implies that the `default` is not used.
|
164
|
+
show_default : bool, default True
|
165
|
+
If True, show the default value in the help text.
|
166
|
+
"""
|
101
167
|
def __init__(self, name: str, default: typing.Union[str, float, int, bool, typing.Dict[str, typing.Any], typing.Callable[[], typing.Union[str, float, int, bool, typing.Dict[str, typing.Any]]], None] = None, type: typing.Union[typing.Type[str], typing.Type[float], typing.Type[int], typing.Type[bool], JSONTypeClass, None] = None, help: typing.Optional[str] = None, required: bool = False, show_default: bool = True, **kwargs: typing.Dict[str, typing.Any]):
|
102
168
|
...
|
103
169
|
def __repr__(self):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.642860 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -14,6 +14,10 @@ if typing.TYPE_CHECKING:
|
|
14
14
|
CLIS_DESC: list
|
15
15
|
|
16
16
|
class InternalTestUnboundedForeachInput(metaflow.unbounded_foreach.UnboundedForeachInput, metaclass=type):
|
17
|
+
"""
|
18
|
+
Test class that wraps around values (any iterator) and simulates an
|
19
|
+
unbounded-foreach instead of a bounded foreach.
|
20
|
+
"""
|
17
21
|
def __init__(self, iterable):
|
18
22
|
...
|
19
23
|
def __iter__(self):
|
@@ -194,6 +198,9 @@ class TestTimeoutCard(metaflow.plugins.cards.card_modules.card.MetaflowCard, met
|
|
194
198
|
...
|
195
199
|
|
196
200
|
class TestRefreshCard(metaflow.plugins.cards.card_modules.card.MetaflowCard, metaclass=type):
|
201
|
+
"""
|
202
|
+
This card takes no components and helps test the `current.card.refresh(data)` interface.
|
203
|
+
"""
|
197
204
|
def render(self, task) -> str:
|
198
205
|
...
|
199
206
|
def render_runtime(self, task, data):
|
@@ -203,6 +210,10 @@ class TestRefreshCard(metaflow.plugins.cards.card_modules.card.MetaflowCard, met
|
|
203
210
|
...
|
204
211
|
|
205
212
|
class TestRefreshComponentCard(metaflow.plugins.cards.card_modules.card.MetaflowCard, metaclass=type):
|
213
|
+
"""
|
214
|
+
This card takes components and helps test the `current.card.components["A"].update()`
|
215
|
+
interface
|
216
|
+
"""
|
206
217
|
def __init__(self, options = {}, components = [], graph = None):
|
207
218
|
...
|
208
219
|
def render(self, task) -> str:
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.672699 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.711434 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T10:13:10.710461 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|