ob-metaflow-stubs 6.0.3.104__py2.py3-none-any.whl → 6.0.3.106__py2.py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- metaflow-stubs/__init__.pyi +853 -515
- metaflow-stubs/cards.pyi +211 -5
- metaflow-stubs/cli.pyi +22 -2
- metaflow-stubs/client/__init__.pyi +128 -3
- metaflow-stubs/client/core.pyi +226 -6
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +5 -2
- metaflow-stubs/events.pyi +20 -2
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +70 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +156 -3
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata/metadata.pyi +8 -2
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +22 -19
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +68 -2
- metaflow-stubs/plugins/__init__.pyi +14 -3
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +80 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +5 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +44 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +44 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +16 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +17 -4
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +98 -7
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +32 -4
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +50 -5
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +102 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +15 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +21 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +49 -4
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +21 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +63 -5
- metaflow-stubs/plugins/cards/card_client.pyi +34 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +5 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +8 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +52 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +28 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +41 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +28 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +183 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +5 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +35 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +55 -2
- metaflow-stubs/plugins/cards/exception.pyi +8 -2
- metaflow-stubs/plugins/catch_decorator.pyi +19 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +63 -3
- metaflow-stubs/plugins/datatools/local.pyi +16 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +72 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +83 -6
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +10 -2
- metaflow-stubs/plugins/events_decorator.pyi +106 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +24 -3
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +15 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +11 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +65 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +99 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +5 -2
- metaflow-stubs/plugins/logs_cli.pyi +3 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +29 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +59 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +45 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +33 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +33 -2
- metaflow-stubs/plugins/retry_decorator.pyi +21 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +11 -2
- metaflow-stubs/plugins/storage_executor.pyi +6 -2
- metaflow-stubs/plugins/tag_cli.pyi +37 -6
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +10 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +23 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +70 -131
- metaflow-stubs/runner/metaflow_runner.pyi +118 -10
- metaflow-stubs/runner/nbdeploy.pyi +66 -2
- metaflow-stubs/runner/nbrun.pyi +79 -2
- metaflow-stubs/runner/subprocess_manager.pyi +16 -4
- metaflow-stubs/runner/utils.pyi +32 -2
- metaflow-stubs/system/__init__.pyi +3 -3
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.106.dist-info/RECORD +140 -0
- ob_metaflow_stubs-6.0.3.104.dist-info/RECORD +0 -140
- {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.933043 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -21,11 +21,115 @@ class MetaflowException(Exception, metaclass=type):
|
|
21
21
|
...
|
22
22
|
|
23
23
|
class TriggerDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
|
24
|
+
"""
|
25
|
+
Specifies the event(s) that this flow depends on.
|
26
|
+
|
27
|
+
```
|
28
|
+
@trigger(event='foo')
|
29
|
+
```
|
30
|
+
or
|
31
|
+
```
|
32
|
+
@trigger(events=['foo', 'bar'])
|
33
|
+
```
|
34
|
+
|
35
|
+
Additionally, you can specify the parameter mappings
|
36
|
+
to map event payload to Metaflow parameters for the flow.
|
37
|
+
```
|
38
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
39
|
+
```
|
40
|
+
or
|
41
|
+
```
|
42
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
43
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
44
|
+
```
|
45
|
+
|
46
|
+
'parameters' can also be a list of strings and tuples like so:
|
47
|
+
```
|
48
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
49
|
+
```
|
50
|
+
This is equivalent to:
|
51
|
+
```
|
52
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
53
|
+
```
|
54
|
+
|
55
|
+
Parameters
|
56
|
+
----------
|
57
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
58
|
+
Event dependency for this flow.
|
59
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
60
|
+
Events dependency for this flow.
|
61
|
+
options : Dict[str, Any], default {}
|
62
|
+
Backend-specific configuration for tuning eventing behavior.
|
63
|
+
|
64
|
+
MF Add To Current
|
65
|
+
-----------------
|
66
|
+
trigger -> metaflow.events.Trigger
|
67
|
+
Returns `Trigger` if the current run is triggered by an event
|
68
|
+
|
69
|
+
@@ Returns
|
70
|
+
-------
|
71
|
+
Trigger
|
72
|
+
`Trigger` if triggered by an event
|
73
|
+
"""
|
24
74
|
def flow_init(self, flow_name, graph, environment, flow_datastore, metadata, logger, echo, options):
|
25
75
|
...
|
26
76
|
...
|
27
77
|
|
28
78
|
class TriggerOnFinishDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
|
79
|
+
"""
|
80
|
+
Specifies the flow(s) that this flow depends on.
|
81
|
+
|
82
|
+
```
|
83
|
+
@trigger_on_finish(flow='FooFlow')
|
84
|
+
```
|
85
|
+
or
|
86
|
+
```
|
87
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
88
|
+
```
|
89
|
+
This decorator respects the @project decorator and triggers the flow
|
90
|
+
when upstream runs within the same namespace complete successfully
|
91
|
+
|
92
|
+
Additionally, you can specify project aware upstream flow dependencies
|
93
|
+
by specifying the fully qualified project_flow_name.
|
94
|
+
```
|
95
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
96
|
+
```
|
97
|
+
or
|
98
|
+
```
|
99
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
100
|
+
```
|
101
|
+
|
102
|
+
You can also specify just the project or project branch (other values will be
|
103
|
+
inferred from the current project or project branch):
|
104
|
+
```
|
105
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
106
|
+
```
|
107
|
+
|
108
|
+
Note that `branch` is typically one of:
|
109
|
+
- `prod`
|
110
|
+
- `user.bob`
|
111
|
+
- `test.my_experiment`
|
112
|
+
- `prod.staging`
|
113
|
+
|
114
|
+
Parameters
|
115
|
+
----------
|
116
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
117
|
+
Upstream flow dependency for this flow.
|
118
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
119
|
+
Upstream flow dependencies for this flow.
|
120
|
+
options : Dict[str, Any], default {}
|
121
|
+
Backend-specific configuration for tuning eventing behavior.
|
122
|
+
|
123
|
+
MF Add To Current
|
124
|
+
-----------------
|
125
|
+
trigger -> metaflow.events.Trigger
|
126
|
+
Returns `Trigger` if the current run is triggered by an event
|
127
|
+
|
128
|
+
@@ Returns
|
129
|
+
-------
|
130
|
+
Trigger
|
131
|
+
`Trigger` if triggered by an event
|
132
|
+
"""
|
29
133
|
def flow_init(self, flow_name, graph, environment, flow_datastore, metadata, logger, echo, options):
|
30
134
|
...
|
31
135
|
def get_top_level_options(self):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.930581 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,20 +1,41 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.964748 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.plugins.parallel_decorator
|
12
11
|
import metaflow.decorators
|
12
|
+
import metaflow.plugins.parallel_decorator
|
13
13
|
import metaflow.metaflow_current
|
14
14
|
|
15
15
|
current: metaflow.metaflow_current.Current
|
16
16
|
|
17
17
|
class ParallelDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
18
|
+
"""
|
19
|
+
MF Add To Current
|
20
|
+
-----------------
|
21
|
+
parallel -> metaflow.metaflow_current.Parallel
|
22
|
+
|
23
|
+
@@ Returns
|
24
|
+
-------
|
25
|
+
Parallel
|
26
|
+
`namedtuple` with the following fields:
|
27
|
+
- main_ip : str
|
28
|
+
The IP address of the control task.
|
29
|
+
- num_nodes : int
|
30
|
+
The total number of tasks created by @parallel
|
31
|
+
- node_index : int
|
32
|
+
The index of the current task in all the @parallel tasks.
|
33
|
+
- control_task_id : Optional[str]
|
34
|
+
The task ID of the control task. Available to all tasks.
|
35
|
+
|
36
|
+
is_parallel -> bool
|
37
|
+
True if the current step is a @parallel step.
|
38
|
+
"""
|
18
39
|
def __init__(self, attributes = None, statically_defined = False):
|
19
40
|
...
|
20
41
|
def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.928954 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.955820 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -9,8 +9,8 @@ from __future__ import annotations
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
11
|
import metaflow.plugins.secrets
|
12
|
-
import abc
|
13
12
|
import metaflow.exception
|
13
|
+
import abc
|
14
14
|
|
15
15
|
class MetaflowException(Exception, metaclass=type):
|
16
16
|
def __init__(self, msg = "", lineno = None):
|
@@ -34,15 +34,27 @@ def get_credentials(scopes, *args, **kwargs):
|
|
34
34
|
GCP_SECRET_MANAGER_PREFIX: None
|
35
35
|
|
36
36
|
class MetaflowGcpSecretsManagerBadResponse(metaflow.exception.MetaflowException, metaclass=type):
|
37
|
+
"""
|
38
|
+
Raised when the response from GCP Secrets Manager is not valid in some way
|
39
|
+
"""
|
37
40
|
...
|
38
41
|
|
39
42
|
class MetaflowGcpSecretsManagerDuplicateKey(metaflow.exception.MetaflowException, metaclass=type):
|
43
|
+
"""
|
44
|
+
Raised when the response from GCP Secrets Manager contains duplicate keys
|
45
|
+
"""
|
40
46
|
...
|
41
47
|
|
42
48
|
class MetaflowGcpSecretsManagerJSONParseError(metaflow.exception.MetaflowException, metaclass=type):
|
49
|
+
"""
|
50
|
+
Raised when the SecretString response from GCP Secrets Manager is not valid JSON
|
51
|
+
"""
|
43
52
|
...
|
44
53
|
|
45
54
|
class MetaflowGcpSecretsManagerNotJSONObject(metaflow.exception.MetaflowException, metaclass=type):
|
55
|
+
"""
|
56
|
+
Raised when the SecretString response from GCP Secrets Manager is not valid JSON dictionary
|
57
|
+
"""
|
46
58
|
...
|
47
59
|
|
48
60
|
class GcpSecretManagerSecretsProvider(metaflow.plugins.secrets.SecretsProvider, metaclass=abc.ABCMeta):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.955088 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.954905 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.955370 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.926734 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.932190 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.969325 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.966818 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -88,6 +88,15 @@ UBF_CONTROL: str
|
|
88
88
|
|
89
89
|
UBF_TASK: str
|
90
90
|
|
91
|
+
def init_config() -> typing.Dict[str, str]:
|
92
|
+
"""
|
93
|
+
OSS Metaflow reads the config file on every step initialization. This is because OSS assumes config files change
|
94
|
+
relatively infrequently. We want to avoid config values changing between flow steps. Our solution to prevent this
|
95
|
+
is to read a config once and cache it on an environment variable. Environment variables carry over between steps
|
96
|
+
because steps are executed in subprocesses (local) or environments which expect environment variables to be set.
|
97
|
+
"""
|
98
|
+
...
|
99
|
+
|
91
100
|
BASH_SAVE_LOGS: str
|
92
101
|
|
93
102
|
class KubernetesClient(object, metaclass=type):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.970487 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -86,6 +86,69 @@ def parse_kube_keyvalue_list(items: typing.List[str], requires_both: bool = True
|
|
86
86
|
...
|
87
87
|
|
88
88
|
class KubernetesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
89
|
+
"""
|
90
|
+
Specifies that this step should execute on Kubernetes.
|
91
|
+
|
92
|
+
Parameters
|
93
|
+
----------
|
94
|
+
cpu : int, default 1
|
95
|
+
Number of CPUs required for this step. If `@resources` is
|
96
|
+
also present, the maximum value from all decorators is used.
|
97
|
+
memory : int, default 4096
|
98
|
+
Memory size (in MB) required for this step. If
|
99
|
+
`@resources` is also present, the maximum value from all decorators is
|
100
|
+
used.
|
101
|
+
disk : int, default 10240
|
102
|
+
Disk size (in MB) required for this step. If
|
103
|
+
`@resources` is also present, the maximum value from all decorators is
|
104
|
+
used.
|
105
|
+
image : str, optional, default None
|
106
|
+
Docker image to use when launching on Kubernetes. If not specified, and
|
107
|
+
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
108
|
+
not, a default Docker image mapping to the current version of Python is used.
|
109
|
+
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
110
|
+
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
111
|
+
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
112
|
+
Kubernetes service account to use when launching pod in Kubernetes.
|
113
|
+
secrets : List[str], optional, default None
|
114
|
+
Kubernetes secrets to use when launching pod in Kubernetes. These
|
115
|
+
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
116
|
+
in Metaflow configuration.
|
117
|
+
node_selector: Union[Dict[str,str], str], optional, default None
|
118
|
+
Kubernetes node selector(s) to apply to the pod running the task.
|
119
|
+
Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
|
120
|
+
or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
|
121
|
+
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
122
|
+
Kubernetes namespace to use when launching pod in Kubernetes.
|
123
|
+
gpu : int, optional, default None
|
124
|
+
Number of GPUs required for this step. A value of zero implies that
|
125
|
+
the scheduled node should not have GPUs.
|
126
|
+
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
127
|
+
The vendor of the GPUs to be used for this step.
|
128
|
+
tolerations : List[str], default []
|
129
|
+
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
130
|
+
Kubernetes tolerations to use when launching pod in Kubernetes.
|
131
|
+
use_tmpfs : bool, default False
|
132
|
+
This enables an explicit tmpfs mount for this step.
|
133
|
+
tmpfs_tempdir : bool, default True
|
134
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
135
|
+
tmpfs_size : int, optional, default: None
|
136
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
137
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
138
|
+
memory allocated for this step.
|
139
|
+
tmpfs_path : str, optional, default /metaflow_temp
|
140
|
+
Path to tmpfs mount for this step.
|
141
|
+
persistent_volume_claims : Dict[str, str], optional, default None
|
142
|
+
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
143
|
+
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
144
|
+
shared_memory: int, optional
|
145
|
+
Shared memory size (in MiB) required for this step
|
146
|
+
port: int, optional
|
147
|
+
Port number to specify in the Kubernetes job object
|
148
|
+
compute_pool : str, optional, default None
|
149
|
+
Compute pool to be used for for this step.
|
150
|
+
If not specified, any accessible compute pool within the perimeter is used.
|
151
|
+
"""
|
89
152
|
def __init__(self, attributes = None, statically_defined = False):
|
90
153
|
...
|
91
154
|
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.884879 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.967778 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -22,6 +22,9 @@ class MetaflowException(Exception, metaclass=type):
|
|
22
22
|
...
|
23
23
|
|
24
24
|
class MetaDatum(tuple, metaclass=type):
|
25
|
+
"""
|
26
|
+
MetaDatum(field, value, type, tags)
|
27
|
+
"""
|
25
28
|
@staticmethod
|
26
29
|
def __new__(_cls, field, value, type, tags):
|
27
30
|
"""
|
@@ -76,6 +79,37 @@ KUBERNETES_SHARED_MEMORY: None
|
|
76
79
|
KUBERNETES_TOLERATIONS: str
|
77
80
|
|
78
81
|
class ResourcesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
82
|
+
"""
|
83
|
+
Specifies the resources needed when executing this step.
|
84
|
+
|
85
|
+
Use `@resources` to specify the resource requirements
|
86
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
87
|
+
|
88
|
+
You can choose the compute layer on the command line by executing e.g.
|
89
|
+
```
|
90
|
+
python myflow.py run --with batch
|
91
|
+
```
|
92
|
+
or
|
93
|
+
```
|
94
|
+
python myflow.py run --with kubernetes
|
95
|
+
```
|
96
|
+
which executes the flow on the desired system using the
|
97
|
+
requirements specified in `@resources`.
|
98
|
+
|
99
|
+
Parameters
|
100
|
+
----------
|
101
|
+
cpu : int, default 1
|
102
|
+
Number of CPUs required for this step.
|
103
|
+
gpu : int, default 0
|
104
|
+
Number of GPUs required for this step.
|
105
|
+
disk : int, optional, default None
|
106
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
107
|
+
memory : int, default 4096
|
108
|
+
Memory size (in MB) required for this step.
|
109
|
+
shared_memory : int, optional, default None
|
110
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
111
|
+
This parameter maps to the `--shm-size` option in Docker.
|
112
|
+
"""
|
79
113
|
...
|
80
114
|
|
81
115
|
def get_run_time_limit_for_task(step_decos):
|
@@ -151,6 +185,69 @@ MAX_MEMORY_PER_TASK: None
|
|
151
185
|
MAX_CPU_PER_TASK: None
|
152
186
|
|
153
187
|
class KubernetesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
188
|
+
"""
|
189
|
+
Specifies that this step should execute on Kubernetes.
|
190
|
+
|
191
|
+
Parameters
|
192
|
+
----------
|
193
|
+
cpu : int, default 1
|
194
|
+
Number of CPUs required for this step. If `@resources` is
|
195
|
+
also present, the maximum value from all decorators is used.
|
196
|
+
memory : int, default 4096
|
197
|
+
Memory size (in MB) required for this step. If
|
198
|
+
`@resources` is also present, the maximum value from all decorators is
|
199
|
+
used.
|
200
|
+
disk : int, default 10240
|
201
|
+
Disk size (in MB) required for this step. If
|
202
|
+
`@resources` is also present, the maximum value from all decorators is
|
203
|
+
used.
|
204
|
+
image : str, optional, default None
|
205
|
+
Docker image to use when launching on Kubernetes. If not specified, and
|
206
|
+
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
207
|
+
not, a default Docker image mapping to the current version of Python is used.
|
208
|
+
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
209
|
+
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
210
|
+
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
211
|
+
Kubernetes service account to use when launching pod in Kubernetes.
|
212
|
+
secrets : List[str], optional, default None
|
213
|
+
Kubernetes secrets to use when launching pod in Kubernetes. These
|
214
|
+
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
215
|
+
in Metaflow configuration.
|
216
|
+
node_selector: Union[Dict[str,str], str], optional, default None
|
217
|
+
Kubernetes node selector(s) to apply to the pod running the task.
|
218
|
+
Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
|
219
|
+
or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
|
220
|
+
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
221
|
+
Kubernetes namespace to use when launching pod in Kubernetes.
|
222
|
+
gpu : int, optional, default None
|
223
|
+
Number of GPUs required for this step. A value of zero implies that
|
224
|
+
the scheduled node should not have GPUs.
|
225
|
+
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
226
|
+
The vendor of the GPUs to be used for this step.
|
227
|
+
tolerations : List[str], default []
|
228
|
+
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
229
|
+
Kubernetes tolerations to use when launching pod in Kubernetes.
|
230
|
+
use_tmpfs : bool, default False
|
231
|
+
This enables an explicit tmpfs mount for this step.
|
232
|
+
tmpfs_tempdir : bool, default True
|
233
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
234
|
+
tmpfs_size : int, optional, default: None
|
235
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
236
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
237
|
+
memory allocated for this step.
|
238
|
+
tmpfs_path : str, optional, default /metaflow_temp
|
239
|
+
Path to tmpfs mount for this step.
|
240
|
+
persistent_volume_claims : Dict[str, str], optional, default None
|
241
|
+
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
242
|
+
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
243
|
+
shared_memory: int, optional
|
244
|
+
Shared memory size (in MiB) required for this step
|
245
|
+
port: int, optional
|
246
|
+
Port number to specify in the Kubernetes job object
|
247
|
+
compute_pool : str, optional, default None
|
248
|
+
Compute pool to be used for for this step.
|
249
|
+
If not specified, any accessible compute pool within the perimeter is used.
|
250
|
+
"""
|
154
251
|
def __init__(self, attributes = None, statically_defined = False):
|
155
252
|
...
|
156
253
|
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.969138 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -30,6 +30,9 @@ def k8s_retry(deadline_seconds = 60, max_backoff = 32):
|
|
30
30
|
...
|
31
31
|
|
32
32
|
class JobsetStatus(tuple, metaclass=type):
|
33
|
+
"""
|
34
|
+
JobsetStatus(control_pod_failed, control_exit_code, control_pod_status, control_started, control_completed, worker_pods_failed, workers_are_suspended, workers_have_started, all_jobs_are_suspended, jobset_finished, jobset_failed, status_unknown, jobset_was_terminated, some_jobs_are_running)
|
35
|
+
"""
|
33
36
|
@staticmethod
|
34
37
|
def __new__(_cls, control_pod_failed, control_exit_code, control_pod_status, control_started, control_completed, worker_pods_failed, workers_are_suspended, workers_have_started, all_jobs_are_suspended, jobset_finished, jobset_failed, status_unknown, jobset_was_terminated, some_jobs_are_running):
|
35
38
|
"""
|
@@ -1,15 +1,15 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.935357 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.exception
|
12
11
|
import metaflow.plugins.logs_cli
|
12
|
+
import metaflow.exception
|
13
13
|
import metaflow._vendor.click.core
|
14
14
|
|
15
15
|
LOGGER_TIMESTAMP: str
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.934962 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|