metaflow-stubs 2.12.22__py2.py3-none-any.whl → 2.12.23__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +1029 -638
- metaflow-stubs/cards.pyi +211 -5
- metaflow-stubs/cli.pyi +23 -3
- metaflow-stubs/client/__init__.pyi +128 -3
- metaflow-stubs/client/core.pyi +227 -7
- metaflow-stubs/client/filecache.pyi +3 -3
- metaflow-stubs/clone_util.pyi +5 -2
- metaflow-stubs/events.pyi +20 -2
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +71 -5
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +158 -5
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata/metadata.pyi +9 -3
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +26 -23
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +69 -3
- metaflow-stubs/plugins/__init__.pyi +14 -3
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow.pyi +15 -3
- metaflow-stubs/plugins/airflow/airflow_cli.pyi +66 -3
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +5 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +80 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +5 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +44 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +44 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +16 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +6 -4
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +97 -6
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +34 -6
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +49 -4
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +103 -3
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +15 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +21 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +69 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +5 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +49 -4
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +21 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +62 -4
- metaflow-stubs/plugins/cards/card_client.pyi +33 -2
- metaflow-stubs/plugins/cards/card_creator.pyi +5 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +8 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +52 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +28 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +41 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +28 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +184 -4
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +5 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +35 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +57 -4
- metaflow-stubs/plugins/cards/exception.pyi +8 -2
- metaflow-stubs/plugins/catch_decorator.pyi +20 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +62 -2
- metaflow-stubs/plugins/datatools/local.pyi +16 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +72 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +82 -5
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +10 -2
- metaflow-stubs/plugins/events_decorator.pyi +106 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +24 -3
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +15 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +65 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +100 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +5 -2
- metaflow-stubs/plugins/logs_cli.pyi +2 -2
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +29 -2
- metaflow-stubs/plugins/project_decorator.pyi +59 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +45 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +33 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +33 -2
- metaflow-stubs/plugins/retry_decorator.pyi +21 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +11 -2
- metaflow-stubs/plugins/storage_executor.pyi +6 -2
- metaflow-stubs/plugins/tag_cli.pyi +35 -4
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +10 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +23 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +70 -131
- metaflow-stubs/runner/metaflow_runner.pyi +117 -9
- metaflow-stubs/runner/nbdeploy.pyi +66 -2
- metaflow-stubs/runner/nbrun.pyi +79 -2
- metaflow-stubs/runner/subprocess_manager.pyi +16 -4
- metaflow-stubs/runner/utils.pyi +32 -2
- metaflow-stubs/system/__init__.pyi +4 -4
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +3 -3
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/version.pyi +2 -2
- {metaflow_stubs-2.12.22.dist-info → metaflow_stubs-2.12.23.dist-info}/METADATA +2 -2
- metaflow_stubs-2.12.23.dist-info/RECORD +152 -0
- metaflow_stubs-2.12.22.dist-info/RECORD +0 -152
- {metaflow_stubs-2.12.22.dist-info → metaflow_stubs-2.12.23.dist-info}/WHEEL +0 -0
- {metaflow_stubs-2.12.22.dist-info → metaflow_stubs-2.12.23.dist-info}/top_level.txt +0 -0
@@ -1,25 +1,28 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.031488 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.flowspec
|
12
11
|
import metaflow.events
|
13
|
-
import metaflow.decorators
|
14
|
-
import metaflow.metaflow_current
|
15
12
|
import metaflow
|
16
|
-
import metaflow.graph
|
17
13
|
import metaflow.datastore.inputs
|
14
|
+
import metaflow.metaflow_current
|
15
|
+
import metaflow.flowspec
|
16
|
+
import metaflow.graph
|
17
|
+
import metaflow.decorators
|
18
18
|
import typing
|
19
19
|
|
20
20
|
current: metaflow.metaflow_current.Current
|
21
21
|
|
22
22
|
class Trigger(object, metaclass=type):
|
23
|
+
"""
|
24
|
+
Defines a container of event triggers' metadata.
|
25
|
+
"""
|
23
26
|
def __init__(self, _meta = None):
|
24
27
|
...
|
25
28
|
@classmethod
|
@@ -93,6 +96,9 @@ class Trigger(object, metaclass=type):
|
|
93
96
|
...
|
94
97
|
|
95
98
|
class MetaDatum(tuple, metaclass=type):
|
99
|
+
"""
|
100
|
+
MetaDatum(field, value, type, tags)
|
101
|
+
"""
|
96
102
|
@staticmethod
|
97
103
|
def __new__(_cls, field, value, type, tags):
|
98
104
|
"""
|
@@ -114,6 +120,14 @@ class MetaDatum(tuple, metaclass=type):
|
|
114
120
|
ARGO_EVENTS_WEBHOOK_URL: None
|
115
121
|
|
116
122
|
class FlowSpec(object, metaclass=metaflow.flowspec._FlowSpecMeta):
|
123
|
+
"""
|
124
|
+
Main class from which all Flows should inherit.
|
125
|
+
|
126
|
+
Attributes
|
127
|
+
----------
|
128
|
+
index
|
129
|
+
input
|
130
|
+
"""
|
117
131
|
def __init__(self, use_cli = True):
|
118
132
|
"""
|
119
133
|
Construct a FlowSpec
|
@@ -335,6 +349,20 @@ class FlowSpec(object, metaclass=metaflow.flowspec._FlowSpecMeta):
|
|
335
349
|
...
|
336
350
|
|
337
351
|
class ArgoEvent(object, metaclass=type):
|
352
|
+
"""
|
353
|
+
ArgoEvent is a small event, a message, that can be published to Argo Workflows. The
|
354
|
+
event will eventually start all flows which have been previously deployed with `@trigger`
|
355
|
+
to wait for this particular named event.
|
356
|
+
|
357
|
+
Parameters
|
358
|
+
----------
|
359
|
+
name : str,
|
360
|
+
Name of the event
|
361
|
+
url : str, optional
|
362
|
+
Override the event endpoint from `ARGO_EVENTS_WEBHOOK_URL`.
|
363
|
+
payload : Dict, optional
|
364
|
+
A set of key-value pairs delivered in this event. Used to set parameters of triggered flows.
|
365
|
+
"""
|
338
366
|
def __init__(self, name, url = None, payload = None, access_token = None):
|
339
367
|
...
|
340
368
|
def add_to_payload(self, key, value):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.040397 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -9,7 +9,6 @@ from __future__ import annotations
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
11
|
import metaflow.runner.deployer
|
12
|
-
import metaflow.runner.subprocess_manager
|
13
12
|
import metaflow.graph
|
14
13
|
|
15
14
|
class ArgoWorkflows(object, metaclass=type):
|
@@ -55,6 +54,30 @@ class ArgoWorkflows(object, metaclass=type):
|
|
55
54
|
...
|
56
55
|
|
57
56
|
class DeployerImpl(object, metaclass=type):
|
57
|
+
"""
|
58
|
+
Base class for deployer implementations. Each implementation should define a TYPE
|
59
|
+
class variable that matches the name of the CLI group.
|
60
|
+
|
61
|
+
Parameters
|
62
|
+
----------
|
63
|
+
flow_file : str
|
64
|
+
Path to the flow file to deploy.
|
65
|
+
show_output : bool, default True
|
66
|
+
Show the 'stdout' and 'stderr' to the console by default.
|
67
|
+
profile : Optional[str], default None
|
68
|
+
Metaflow profile to use for the deployment. If not specified, the default
|
69
|
+
profile is used.
|
70
|
+
env : Optional[Dict], default None
|
71
|
+
Additional environment variables to set for the deployment.
|
72
|
+
cwd : Optional[str], default None
|
73
|
+
The directory to run the subprocess in; if not specified, the current
|
74
|
+
directory is used.
|
75
|
+
file_read_timeout : int, default 3600
|
76
|
+
The timeout until which we try to read the deployer attribute file.
|
77
|
+
**kwargs : Any
|
78
|
+
Additional arguments that you would pass to `python myflow.py` before
|
79
|
+
the deployment command.
|
80
|
+
"""
|
58
81
|
def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
|
59
82
|
...
|
60
83
|
def __enter__(self) -> metaflow.runner.deployer.DeployerImpl:
|
@@ -93,11 +116,25 @@ class DeployerImpl(object, metaclass=type):
|
|
93
116
|
...
|
94
117
|
|
95
118
|
class DeployedFlow(object, metaclass=type):
|
119
|
+
"""
|
120
|
+
DeployedFlow class represents a flow that has been deployed.
|
121
|
+
|
122
|
+
Parameters
|
123
|
+
----------
|
124
|
+
deployer : DeployerImpl
|
125
|
+
Instance of the deployer implementation.
|
126
|
+
"""
|
96
127
|
def __init__(self, deployer: metaflow.runner.deployer.DeployerImpl):
|
97
128
|
...
|
98
129
|
...
|
99
130
|
|
100
131
|
class TriggeredRun(object, metaclass=type):
|
132
|
+
"""
|
133
|
+
TriggeredRun class represents a run that has been triggered on a production orchestrator.
|
134
|
+
|
135
|
+
Only when the `start` task starts running, the `run` object corresponding to the run
|
136
|
+
becomes available.
|
137
|
+
"""
|
101
138
|
def __init__(self, deployer: metaflow.runner.deployer.DeployerImpl, content: str):
|
102
139
|
...
|
103
140
|
def wait_for_run(self, timeout = None):
|
@@ -158,7 +195,7 @@ def get_lower_level_group(api, top_level_kwargs: typing.Dict, _type: typing.Opti
|
|
158
195
|
"""
|
159
196
|
...
|
160
197
|
|
161
|
-
def handle_timeout(tfp_runner_attribute, command_obj:
|
198
|
+
def handle_timeout(tfp_runner_attribute, command_obj: "CommandManager", file_read_timeout: int):
|
162
199
|
"""
|
163
200
|
Handle the timeout for a running subprocess command that reads a file
|
164
201
|
and raises an error with appropriate logs if a TimeoutError occurs.
|
@@ -293,6 +330,14 @@ def trigger(instance: metaflow.runner.deployer.DeployedFlow, **kwargs):
|
|
293
330
|
...
|
294
331
|
|
295
332
|
class ArgoWorkflowsDeployer(metaflow.runner.deployer.DeployerImpl, metaclass=type):
|
333
|
+
"""
|
334
|
+
Deployer implementation for Argo Workflows.
|
335
|
+
|
336
|
+
Attributes
|
337
|
+
----------
|
338
|
+
TYPE : ClassVar[Optional[str]]
|
339
|
+
The type of the deployer, which is "argo-workflows".
|
340
|
+
"""
|
296
341
|
def __init__(self, deployer_kwargs, **kwargs):
|
297
342
|
"""
|
298
343
|
Initialize the ArgoWorkflowsDeployer.
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:39.991830 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.015090 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.014819 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.014401 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.076307 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.079218 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.073964 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,26 +1,60 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.078226 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.exception
|
12
11
|
import metaflow.metaflow_current
|
12
|
+
import metaflow.exception
|
13
13
|
import metaflow.decorators
|
14
14
|
|
15
15
|
current: metaflow.metaflow_current.Current
|
16
16
|
|
17
17
|
class ResourcesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
18
|
+
"""
|
19
|
+
Specifies the resources needed when executing this step.
|
20
|
+
|
21
|
+
Use `@resources` to specify the resource requirements
|
22
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
23
|
+
|
24
|
+
You can choose the compute layer on the command line by executing e.g.
|
25
|
+
```
|
26
|
+
python myflow.py run --with batch
|
27
|
+
```
|
28
|
+
or
|
29
|
+
```
|
30
|
+
python myflow.py run --with kubernetes
|
31
|
+
```
|
32
|
+
which executes the flow on the desired system using the
|
33
|
+
requirements specified in `@resources`.
|
34
|
+
|
35
|
+
Parameters
|
36
|
+
----------
|
37
|
+
cpu : int, default 1
|
38
|
+
Number of CPUs required for this step.
|
39
|
+
gpu : int, default 0
|
40
|
+
Number of GPUs required for this step.
|
41
|
+
disk : int, optional, default None
|
42
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
43
|
+
memory : int, default 4096
|
44
|
+
Memory size (in MB) required for this step.
|
45
|
+
shared_memory : int, optional, default None
|
46
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
47
|
+
This parameter maps to the `--shm-size` option in Docker.
|
48
|
+
"""
|
18
49
|
...
|
19
50
|
|
20
51
|
def get_run_time_limit_for_task(step_decos):
|
21
52
|
...
|
22
53
|
|
23
54
|
class MetaDatum(tuple, metaclass=type):
|
55
|
+
"""
|
56
|
+
MetaDatum(field, value, type, tags)
|
57
|
+
"""
|
24
58
|
@staticmethod
|
25
59
|
def __new__(_cls, field, value, type, tags):
|
26
60
|
"""
|
@@ -127,6 +161,72 @@ def get_ec2_instance_metadata():
|
|
127
161
|
...
|
128
162
|
|
129
163
|
class BatchDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
164
|
+
"""
|
165
|
+
Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
|
166
|
+
|
167
|
+
Parameters
|
168
|
+
----------
|
169
|
+
cpu : int, default 1
|
170
|
+
Number of CPUs required for this step. If `@resources` is
|
171
|
+
also present, the maximum value from all decorators is used.
|
172
|
+
gpu : int, default 0
|
173
|
+
Number of GPUs required for this step. If `@resources` is
|
174
|
+
also present, the maximum value from all decorators is used.
|
175
|
+
memory : int, default 4096
|
176
|
+
Memory size (in MB) required for this step. If
|
177
|
+
`@resources` is also present, the maximum value from all decorators is
|
178
|
+
used.
|
179
|
+
image : str, optional, default None
|
180
|
+
Docker image to use when launching on AWS Batch. If not specified, and
|
181
|
+
METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
|
182
|
+
not, a default Docker image mapping to the current version of Python is used.
|
183
|
+
queue : str, default METAFLOW_BATCH_JOB_QUEUE
|
184
|
+
AWS Batch Job Queue to submit the job to.
|
185
|
+
iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
|
186
|
+
AWS IAM role that AWS Batch container uses to access AWS cloud resources.
|
187
|
+
execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
|
188
|
+
AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
|
189
|
+
(https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
|
190
|
+
shared_memory : int, optional, default None
|
191
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
192
|
+
This parameter maps to the `--shm-size` option in Docker.
|
193
|
+
max_swap : int, optional, default None
|
194
|
+
The total amount of swap memory (in MiB) a container can use for this
|
195
|
+
step. This parameter is translated to the `--memory-swap` option in
|
196
|
+
Docker where the value is the sum of the container memory plus the
|
197
|
+
`max_swap` value.
|
198
|
+
swappiness : int, optional, default None
|
199
|
+
This allows you to tune memory swappiness behavior for this step.
|
200
|
+
A swappiness value of 0 causes swapping not to happen unless absolutely
|
201
|
+
necessary. A swappiness value of 100 causes pages to be swapped very
|
202
|
+
aggressively. Accepted values are whole numbers between 0 and 100.
|
203
|
+
use_tmpfs : bool, default False
|
204
|
+
This enables an explicit tmpfs mount for this step. Note that tmpfs is
|
205
|
+
not available on Fargate compute environments
|
206
|
+
tmpfs_tempdir : bool, default True
|
207
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
208
|
+
tmpfs_size : int, optional, default None
|
209
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
210
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
211
|
+
memory allocated for this step.
|
212
|
+
tmpfs_path : str, optional, default None
|
213
|
+
Path to tmpfs mount for this step. Defaults to /metaflow_temp.
|
214
|
+
inferentia : int, default 0
|
215
|
+
Number of Inferentia chips required for this step.
|
216
|
+
trainium : int, default None
|
217
|
+
Alias for inferentia. Use only one of the two.
|
218
|
+
efa : int, default 0
|
219
|
+
Number of elastic fabric adapter network devices to attach to container
|
220
|
+
ephemeral_storage : int, default None
|
221
|
+
The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
|
222
|
+
This is only relevant for Fargate compute environments
|
223
|
+
log_driver: str, optional, default None
|
224
|
+
The log driver to use for the Amazon ECS container.
|
225
|
+
log_options: List[str], optional, default None
|
226
|
+
List of strings containing options for the chosen log driver. The configurable values
|
227
|
+
depend on the `log driver` chosen. Validation of these options is not supported yet.
|
228
|
+
Example: [`awslogs-group:aws/batch/job`]
|
229
|
+
"""
|
130
230
|
def __init__(self, attributes = None, statically_defined = False):
|
131
231
|
...
|
132
232
|
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.015209 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.079735 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -9,8 +9,8 @@ from __future__ import annotations
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
11
|
import abc
|
12
|
-
import metaflow.plugins.secrets
|
13
12
|
import metaflow.exception
|
13
|
+
import metaflow.plugins.secrets
|
14
14
|
|
15
15
|
class MetaflowException(Exception, metaclass=type):
|
16
16
|
def __init__(self, msg = "", lineno = None):
|
@@ -31,15 +31,27 @@ class SecretsProvider(abc.ABC, metaclass=abc.ABCMeta):
|
|
31
31
|
...
|
32
32
|
|
33
33
|
class MetaflowAWSSecretsManagerBadResponse(metaflow.exception.MetaflowException, metaclass=type):
|
34
|
+
"""
|
35
|
+
Raised when the response from AWS Secrets Manager is not valid in some way
|
36
|
+
"""
|
34
37
|
...
|
35
38
|
|
36
39
|
class MetaflowAWSSecretsManagerDuplicateKey(metaflow.exception.MetaflowException, metaclass=type):
|
40
|
+
"""
|
41
|
+
Raised when the response from AWS Secrets Manager contains duplicate keys
|
42
|
+
"""
|
37
43
|
...
|
38
44
|
|
39
45
|
class MetaflowAWSSecretsManagerJSONParseError(metaflow.exception.MetaflowException, metaclass=type):
|
46
|
+
"""
|
47
|
+
Raised when the SecretString response from AWS Secrets Manager is not valid JSON
|
48
|
+
"""
|
40
49
|
...
|
41
50
|
|
42
51
|
class MetaflowAWSSecretsManagerNotJSONObject(metaflow.exception.MetaflowException, metaclass=type):
|
52
|
+
"""
|
53
|
+
Raised when the SecretString response from AWS Secrets Manager is not valid JSON object (dictionary)
|
54
|
+
"""
|
43
55
|
...
|
44
56
|
|
45
57
|
class AwsSecretsManagerSecretsProvider(metaflow.plugins.secrets.SecretsProvider, metaclass=abc.ABCMeta):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.014283 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.064747 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.065799 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.070575 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.065465 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -11,6 +11,25 @@ if typing.TYPE_CHECKING:
|
|
11
11
|
import metaflow.decorators
|
12
12
|
|
13
13
|
class ScheduleDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
|
14
|
+
"""
|
15
|
+
Specifies the times when the flow should be run when running on a
|
16
|
+
production scheduler.
|
17
|
+
|
18
|
+
Parameters
|
19
|
+
----------
|
20
|
+
hourly : bool, default False
|
21
|
+
Run the workflow hourly.
|
22
|
+
daily : bool, default True
|
23
|
+
Run the workflow daily.
|
24
|
+
weekly : bool, default False
|
25
|
+
Run the workflow weekly.
|
26
|
+
cron : str, optional, default None
|
27
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
28
|
+
specified by this expression.
|
29
|
+
timezone : str, optional, default None
|
30
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
31
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
32
|
+
"""
|
14
33
|
def flow_init(self, flow, graph, environment, flow_datastore, metadata, logger, echo, options):
|
15
34
|
...
|
16
35
|
...
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.068701 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,16 +1,16 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.072233 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.exception
|
12
11
|
import metaflow.metaflow_current
|
13
12
|
import metaflow.parameters
|
13
|
+
import metaflow.exception
|
14
14
|
import metaflow.decorators
|
15
15
|
|
16
16
|
JSONType: metaflow.parameters.JSONTypeClass
|
@@ -54,6 +54,72 @@ SFN_STATE_MACHINE_PREFIX: None
|
|
54
54
|
UI_URL: None
|
55
55
|
|
56
56
|
class BatchDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
57
|
+
"""
|
58
|
+
Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
|
59
|
+
|
60
|
+
Parameters
|
61
|
+
----------
|
62
|
+
cpu : int, default 1
|
63
|
+
Number of CPUs required for this step. If `@resources` is
|
64
|
+
also present, the maximum value from all decorators is used.
|
65
|
+
gpu : int, default 0
|
66
|
+
Number of GPUs required for this step. If `@resources` is
|
67
|
+
also present, the maximum value from all decorators is used.
|
68
|
+
memory : int, default 4096
|
69
|
+
Memory size (in MB) required for this step. If
|
70
|
+
`@resources` is also present, the maximum value from all decorators is
|
71
|
+
used.
|
72
|
+
image : str, optional, default None
|
73
|
+
Docker image to use when launching on AWS Batch. If not specified, and
|
74
|
+
METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
|
75
|
+
not, a default Docker image mapping to the current version of Python is used.
|
76
|
+
queue : str, default METAFLOW_BATCH_JOB_QUEUE
|
77
|
+
AWS Batch Job Queue to submit the job to.
|
78
|
+
iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
|
79
|
+
AWS IAM role that AWS Batch container uses to access AWS cloud resources.
|
80
|
+
execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
|
81
|
+
AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
|
82
|
+
(https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
|
83
|
+
shared_memory : int, optional, default None
|
84
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
85
|
+
This parameter maps to the `--shm-size` option in Docker.
|
86
|
+
max_swap : int, optional, default None
|
87
|
+
The total amount of swap memory (in MiB) a container can use for this
|
88
|
+
step. This parameter is translated to the `--memory-swap` option in
|
89
|
+
Docker where the value is the sum of the container memory plus the
|
90
|
+
`max_swap` value.
|
91
|
+
swappiness : int, optional, default None
|
92
|
+
This allows you to tune memory swappiness behavior for this step.
|
93
|
+
A swappiness value of 0 causes swapping not to happen unless absolutely
|
94
|
+
necessary. A swappiness value of 100 causes pages to be swapped very
|
95
|
+
aggressively. Accepted values are whole numbers between 0 and 100.
|
96
|
+
use_tmpfs : bool, default False
|
97
|
+
This enables an explicit tmpfs mount for this step. Note that tmpfs is
|
98
|
+
not available on Fargate compute environments
|
99
|
+
tmpfs_tempdir : bool, default True
|
100
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
101
|
+
tmpfs_size : int, optional, default None
|
102
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
103
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
104
|
+
memory allocated for this step.
|
105
|
+
tmpfs_path : str, optional, default None
|
106
|
+
Path to tmpfs mount for this step. Defaults to /metaflow_temp.
|
107
|
+
inferentia : int, default 0
|
108
|
+
Number of Inferentia chips required for this step.
|
109
|
+
trainium : int, default None
|
110
|
+
Alias for inferentia. Use only one of the two.
|
111
|
+
efa : int, default 0
|
112
|
+
Number of elastic fabric adapter network devices to attach to container
|
113
|
+
ephemeral_storage : int, default None
|
114
|
+
The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
|
115
|
+
This is only relevant for Fargate compute environments
|
116
|
+
log_driver: str, optional, default None
|
117
|
+
The log driver to use for the Amazon ECS container.
|
118
|
+
log_options: List[str], optional, default None
|
119
|
+
List of strings containing options for the chosen log driver. The configurable values
|
120
|
+
depend on the `log driver` chosen. Validation of these options is not supported yet.
|
121
|
+
Example: [`awslogs-group:aws/batch/job`]
|
122
|
+
"""
|
57
123
|
def __init__(self, attributes = None, statically_defined = False):
|
58
124
|
...
|
59
125
|
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.23 #
|
4
|
+
# Generated on 2024-10-01T14:32:40.066206 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|