ob-metaflow-stubs 6.0.3.104__py2.py3-none-any.whl → 6.0.3.106__py2.py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- metaflow-stubs/__init__.pyi +853 -515
- metaflow-stubs/cards.pyi +211 -5
- metaflow-stubs/cli.pyi +22 -2
- metaflow-stubs/client/__init__.pyi +128 -3
- metaflow-stubs/client/core.pyi +226 -6
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +5 -2
- metaflow-stubs/events.pyi +20 -2
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +70 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +156 -3
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata/metadata.pyi +8 -2
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +22 -19
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +68 -2
- metaflow-stubs/plugins/__init__.pyi +14 -3
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +80 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +5 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +44 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +44 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +16 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +17 -4
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +98 -7
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +32 -4
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +50 -5
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +102 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +15 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +21 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +49 -4
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +21 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +63 -5
- metaflow-stubs/plugins/cards/card_client.pyi +34 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +5 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +8 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +52 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +28 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +41 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +28 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +183 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +5 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +35 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +55 -2
- metaflow-stubs/plugins/cards/exception.pyi +8 -2
- metaflow-stubs/plugins/catch_decorator.pyi +19 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +63 -3
- metaflow-stubs/plugins/datatools/local.pyi +16 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +72 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +83 -6
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +10 -2
- metaflow-stubs/plugins/events_decorator.pyi +106 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +24 -3
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +15 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +11 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +65 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +99 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +5 -2
- metaflow-stubs/plugins/logs_cli.pyi +3 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +29 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +59 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +45 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +33 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +33 -2
- metaflow-stubs/plugins/retry_decorator.pyi +21 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +11 -2
- metaflow-stubs/plugins/storage_executor.pyi +6 -2
- metaflow-stubs/plugins/tag_cli.pyi +37 -6
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +10 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +23 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +70 -131
- metaflow-stubs/runner/metaflow_runner.pyi +118 -10
- metaflow-stubs/runner/nbdeploy.pyi +66 -2
- metaflow-stubs/runner/nbrun.pyi +79 -2
- metaflow-stubs/runner/subprocess_manager.pyi +16 -4
- metaflow-stubs/runner/utils.pyi +32 -2
- metaflow-stubs/system/__init__.pyi +3 -3
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.106.dist-info/RECORD +140 -0
- ob_metaflow_stubs-6.0.3.104.dist-info/RECORD +0 -140
- {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.981250 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -15,12 +15,46 @@ if typing.TYPE_CHECKING:
|
|
15
15
|
current: metaflow.metaflow_current.Current
|
16
16
|
|
17
17
|
class ResourcesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
18
|
+
"""
|
19
|
+
Specifies the resources needed when executing this step.
|
20
|
+
|
21
|
+
Use `@resources` to specify the resource requirements
|
22
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
23
|
+
|
24
|
+
You can choose the compute layer on the command line by executing e.g.
|
25
|
+
```
|
26
|
+
python myflow.py run --with batch
|
27
|
+
```
|
28
|
+
or
|
29
|
+
```
|
30
|
+
python myflow.py run --with kubernetes
|
31
|
+
```
|
32
|
+
which executes the flow on the desired system using the
|
33
|
+
requirements specified in `@resources`.
|
34
|
+
|
35
|
+
Parameters
|
36
|
+
----------
|
37
|
+
cpu : int, default 1
|
38
|
+
Number of CPUs required for this step.
|
39
|
+
gpu : int, default 0
|
40
|
+
Number of GPUs required for this step.
|
41
|
+
disk : int, optional, default None
|
42
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
43
|
+
memory : int, default 4096
|
44
|
+
Memory size (in MB) required for this step.
|
45
|
+
shared_memory : int, optional, default None
|
46
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
47
|
+
This parameter maps to the `--shm-size` option in Docker.
|
48
|
+
"""
|
18
49
|
...
|
19
50
|
|
20
51
|
def get_run_time_limit_for_task(step_decos):
|
21
52
|
...
|
22
53
|
|
23
54
|
class MetaDatum(tuple, metaclass=type):
|
55
|
+
"""
|
56
|
+
MetaDatum(field, value, type, tags)
|
57
|
+
"""
|
24
58
|
@staticmethod
|
25
59
|
def __new__(_cls, field, value, type, tags):
|
26
60
|
"""
|
@@ -127,6 +161,72 @@ def get_ec2_instance_metadata():
|
|
127
161
|
...
|
128
162
|
|
129
163
|
class BatchDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
164
|
+
"""
|
165
|
+
Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
|
166
|
+
|
167
|
+
Parameters
|
168
|
+
----------
|
169
|
+
cpu : int, default 1
|
170
|
+
Number of CPUs required for this step. If `@resources` is
|
171
|
+
also present, the maximum value from all decorators is used.
|
172
|
+
gpu : int, default 0
|
173
|
+
Number of GPUs required for this step. If `@resources` is
|
174
|
+
also present, the maximum value from all decorators is used.
|
175
|
+
memory : int, default 4096
|
176
|
+
Memory size (in MB) required for this step. If
|
177
|
+
`@resources` is also present, the maximum value from all decorators is
|
178
|
+
used.
|
179
|
+
image : str, optional, default None
|
180
|
+
Docker image to use when launching on AWS Batch. If not specified, and
|
181
|
+
METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
|
182
|
+
not, a default Docker image mapping to the current version of Python is used.
|
183
|
+
queue : str, default METAFLOW_BATCH_JOB_QUEUE
|
184
|
+
AWS Batch Job Queue to submit the job to.
|
185
|
+
iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
|
186
|
+
AWS IAM role that AWS Batch container uses to access AWS cloud resources.
|
187
|
+
execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
|
188
|
+
AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
|
189
|
+
(https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
|
190
|
+
shared_memory : int, optional, default None
|
191
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
192
|
+
This parameter maps to the `--shm-size` option in Docker.
|
193
|
+
max_swap : int, optional, default None
|
194
|
+
The total amount of swap memory (in MiB) a container can use for this
|
195
|
+
step. This parameter is translated to the `--memory-swap` option in
|
196
|
+
Docker where the value is the sum of the container memory plus the
|
197
|
+
`max_swap` value.
|
198
|
+
swappiness : int, optional, default None
|
199
|
+
This allows you to tune memory swappiness behavior for this step.
|
200
|
+
A swappiness value of 0 causes swapping not to happen unless absolutely
|
201
|
+
necessary. A swappiness value of 100 causes pages to be swapped very
|
202
|
+
aggressively. Accepted values are whole numbers between 0 and 100.
|
203
|
+
use_tmpfs : bool, default False
|
204
|
+
This enables an explicit tmpfs mount for this step. Note that tmpfs is
|
205
|
+
not available on Fargate compute environments
|
206
|
+
tmpfs_tempdir : bool, default True
|
207
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
208
|
+
tmpfs_size : int, optional, default None
|
209
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
210
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
211
|
+
memory allocated for this step.
|
212
|
+
tmpfs_path : str, optional, default None
|
213
|
+
Path to tmpfs mount for this step. Defaults to /metaflow_temp.
|
214
|
+
inferentia : int, default 0
|
215
|
+
Number of Inferentia chips required for this step.
|
216
|
+
trainium : int, default None
|
217
|
+
Alias for inferentia. Use only one of the two.
|
218
|
+
efa : int, default 0
|
219
|
+
Number of elastic fabric adapter network devices to attach to container
|
220
|
+
ephemeral_storage : int, default None
|
221
|
+
The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
|
222
|
+
This is only relevant for Fargate compute environments
|
223
|
+
log_driver: str, optional, default None
|
224
|
+
The log driver to use for the Amazon ECS container.
|
225
|
+
log_options: List[str], optional, default None
|
226
|
+
List of strings containing options for the chosen log driver. The configurable values
|
227
|
+
depend on the `log driver` chosen. Validation of these options is not supported yet.
|
228
|
+
Example: [`awslogs-group:aws/batch/job`]
|
229
|
+
"""
|
130
230
|
def __init__(self, attributes = None, statically_defined = False):
|
131
231
|
...
|
132
232
|
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.965688 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.981678 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -9,8 +9,8 @@ from __future__ import annotations
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
11
|
import metaflow.plugins.secrets
|
12
|
-
import abc
|
13
12
|
import metaflow.exception
|
13
|
+
import abc
|
14
14
|
|
15
15
|
class MetaflowException(Exception, metaclass=type):
|
16
16
|
def __init__(self, msg = "", lineno = None):
|
@@ -31,15 +31,27 @@ class SecretsProvider(abc.ABC, metaclass=abc.ABCMeta):
|
|
31
31
|
...
|
32
32
|
|
33
33
|
class MetaflowAWSSecretsManagerBadResponse(metaflow.exception.MetaflowException, metaclass=type):
|
34
|
+
"""
|
35
|
+
Raised when the response from AWS Secrets Manager is not valid in some way
|
36
|
+
"""
|
34
37
|
...
|
35
38
|
|
36
39
|
class MetaflowAWSSecretsManagerDuplicateKey(metaflow.exception.MetaflowException, metaclass=type):
|
40
|
+
"""
|
41
|
+
Raised when the response from AWS Secrets Manager contains duplicate keys
|
42
|
+
"""
|
37
43
|
...
|
38
44
|
|
39
45
|
class MetaflowAWSSecretsManagerJSONParseError(metaflow.exception.MetaflowException, metaclass=type):
|
46
|
+
"""
|
47
|
+
Raised when the SecretString response from AWS Secrets Manager is not valid JSON
|
48
|
+
"""
|
40
49
|
...
|
41
50
|
|
42
51
|
class MetaflowAWSSecretsManagerNotJSONObject(metaflow.exception.MetaflowException, metaclass=type):
|
52
|
+
"""
|
53
|
+
Raised when the SecretString response from AWS Secrets Manager is not valid JSON object (dictionary)
|
54
|
+
"""
|
43
55
|
...
|
44
56
|
|
45
57
|
class AwsSecretsManagerSecretsProvider(metaflow.plugins.secrets.SecretsProvider, metaclass=abc.ABCMeta):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.965247 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.974770 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.973935 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.974482 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -11,6 +11,25 @@ if typing.TYPE_CHECKING:
|
|
11
11
|
import metaflow.decorators
|
12
12
|
|
13
13
|
class ScheduleDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
|
14
|
+
"""
|
15
|
+
Specifies the times when the flow should be run when running on a
|
16
|
+
production scheduler.
|
17
|
+
|
18
|
+
Parameters
|
19
|
+
----------
|
20
|
+
hourly : bool, default False
|
21
|
+
Run the workflow hourly.
|
22
|
+
daily : bool, default True
|
23
|
+
Run the workflow daily.
|
24
|
+
weekly : bool, default False
|
25
|
+
Run the workflow weekly.
|
26
|
+
cron : str, optional, default None
|
27
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
28
|
+
specified by this expression.
|
29
|
+
timezone : str, optional, default None
|
30
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
31
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
32
|
+
"""
|
14
33
|
def flow_init(self, flow, graph, environment, flow_datastore, metadata, logger, echo, options):
|
15
34
|
...
|
16
35
|
...
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.977062 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.975069 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.978302 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -9,7 +9,6 @@ from __future__ import annotations
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
11
|
import metaflow.runner.deployer
|
12
|
-
import metaflow.runner.subprocess_manager
|
13
12
|
|
14
13
|
class StepFunctions(object, metaclass=type):
|
15
14
|
def __init__(self, name, graph, flow, code_package_sha, code_package_url, production_token, metadata, flow_datastore, environment, event_logger, monitor, tags = None, namespace = None, username = None, max_workers = None, workflow_timeout = None, is_project = False, use_distributed_map = False):
|
@@ -43,6 +42,30 @@ class StepFunctions(object, metaclass=type):
|
|
43
42
|
...
|
44
43
|
|
45
44
|
class DeployerImpl(object, metaclass=type):
|
45
|
+
"""
|
46
|
+
Base class for deployer implementations. Each implementation should define a TYPE
|
47
|
+
class variable that matches the name of the CLI group.
|
48
|
+
|
49
|
+
Parameters
|
50
|
+
----------
|
51
|
+
flow_file : str
|
52
|
+
Path to the flow file to deploy.
|
53
|
+
show_output : bool, default True
|
54
|
+
Show the 'stdout' and 'stderr' to the console by default.
|
55
|
+
profile : Optional[str], default None
|
56
|
+
Metaflow profile to use for the deployment. If not specified, the default
|
57
|
+
profile is used.
|
58
|
+
env : Optional[Dict], default None
|
59
|
+
Additional environment variables to set for the deployment.
|
60
|
+
cwd : Optional[str], default None
|
61
|
+
The directory to run the subprocess in; if not specified, the current
|
62
|
+
directory is used.
|
63
|
+
file_read_timeout : int, default 3600
|
64
|
+
The timeout until which we try to read the deployer attribute file.
|
65
|
+
**kwargs : Any
|
66
|
+
Additional arguments that you would pass to `python myflow.py` before
|
67
|
+
the deployment command.
|
68
|
+
"""
|
46
69
|
def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
|
47
70
|
...
|
48
71
|
def __enter__(self) -> metaflow.runner.deployer.DeployerImpl:
|
@@ -81,11 +104,25 @@ class DeployerImpl(object, metaclass=type):
|
|
81
104
|
...
|
82
105
|
|
83
106
|
class DeployedFlow(object, metaclass=type):
|
107
|
+
"""
|
108
|
+
DeployedFlow class represents a flow that has been deployed.
|
109
|
+
|
110
|
+
Parameters
|
111
|
+
----------
|
112
|
+
deployer : DeployerImpl
|
113
|
+
Instance of the deployer implementation.
|
114
|
+
"""
|
84
115
|
def __init__(self, deployer: metaflow.runner.deployer.DeployerImpl):
|
85
116
|
...
|
86
117
|
...
|
87
118
|
|
88
119
|
class TriggeredRun(object, metaclass=type):
|
120
|
+
"""
|
121
|
+
TriggeredRun class represents a run that has been triggered on a production orchestrator.
|
122
|
+
|
123
|
+
Only when the `start` task starts running, the `run` object corresponding to the run
|
124
|
+
becomes available.
|
125
|
+
"""
|
89
126
|
def __init__(self, deployer: metaflow.runner.deployer.DeployerImpl, content: str):
|
90
127
|
...
|
91
128
|
def wait_for_run(self, timeout = None):
|
@@ -146,7 +183,7 @@ def get_lower_level_group(api, top_level_kwargs: typing.Dict, _type: typing.Opti
|
|
146
183
|
"""
|
147
184
|
...
|
148
185
|
|
149
|
-
def handle_timeout(tfp_runner_attribute, command_obj:
|
186
|
+
def handle_timeout(tfp_runner_attribute, command_obj: "CommandManager", file_read_timeout: int):
|
150
187
|
"""
|
151
188
|
Handle the timeout for a running subprocess command that reads a file
|
152
189
|
and raises an error with appropriate logs if a TimeoutError occurs.
|
@@ -261,6 +298,14 @@ def trigger(instance: metaflow.runner.deployer.DeployedFlow, **kwargs):
|
|
261
298
|
...
|
262
299
|
|
263
300
|
class StepFunctionsDeployer(metaflow.runner.deployer.DeployerImpl, metaclass=type):
|
301
|
+
"""
|
302
|
+
Deployer implementation for AWS Step Functions.
|
303
|
+
|
304
|
+
Attributes
|
305
|
+
----------
|
306
|
+
TYPE : ClassVar[Optional[str]]
|
307
|
+
The type of the deployer, which is "step-functions".
|
308
|
+
"""
|
264
309
|
def __init__(self, deployer_kwargs, **kwargs):
|
265
310
|
"""
|
266
311
|
Initialize the StepFunctionsDeployer.
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.933175 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.972226 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.972438 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.973618 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -9,8 +9,8 @@ from __future__ import annotations
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
11
|
import metaflow.plugins.secrets
|
12
|
-
import abc
|
13
12
|
import metaflow.exception
|
13
|
+
import abc
|
14
14
|
|
15
15
|
class SecretsProvider(abc.ABC, metaclass=abc.ABCMeta):
|
16
16
|
def get_secret_as_dict(self, secret_id, options = {}, role = None) -> typing.Dict[str, str]:
|
@@ -34,21 +34,39 @@ def create_cacheable_azure_credential():
|
|
34
34
|
...
|
35
35
|
|
36
36
|
class MetaflowAzureKeyVaultBadVault(metaflow.exception.MetaflowException, metaclass=type):
|
37
|
+
"""
|
38
|
+
Raised when the secretid is fully qualified but does not have the right key vault domain
|
39
|
+
"""
|
37
40
|
...
|
38
41
|
|
39
42
|
class MetaflowAzureKeyVaultBadSecretType(metaflow.exception.MetaflowException, metaclass=type):
|
43
|
+
"""
|
44
|
+
Raised when the secret type is anything except secrets
|
45
|
+
"""
|
40
46
|
...
|
41
47
|
|
42
48
|
class MetaflowAzureKeyVaultBadSecretPath(metaflow.exception.MetaflowException, metaclass=type):
|
49
|
+
"""
|
50
|
+
Raised when the secret path does not match to expected length
|
51
|
+
"""
|
43
52
|
...
|
44
53
|
|
45
54
|
class MetaflowAzureKeyVaultBadSecretName(metaflow.exception.MetaflowException, metaclass=type):
|
55
|
+
"""
|
56
|
+
Raised when the secret name does not match expected pattern
|
57
|
+
"""
|
46
58
|
...
|
47
59
|
|
48
60
|
class MetaflowAzureKeyVaultBadSecretVersion(metaflow.exception.MetaflowException, metaclass=type):
|
61
|
+
"""
|
62
|
+
Raised when the secret version does not match expected pattern
|
63
|
+
"""
|
49
64
|
...
|
50
65
|
|
51
66
|
class MetaflowAzureKeyVaultBadSecret(metaflow.exception.MetaflowException, metaclass=type):
|
67
|
+
"""
|
68
|
+
Raised when the secret does not match supported patterns in Metaflow
|
69
|
+
"""
|
52
70
|
...
|
53
71
|
|
54
72
|
class AzureKeyVaultSecretsProvider(metaflow.plugins.secrets.SecretsProvider, metaclass=abc.ABCMeta):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.972851 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.973166 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.926221 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.929361 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,20 +1,69 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.960989 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.client.core
|
12
|
-
import datetime
|
13
|
-
import typing
|
14
11
|
import metaflow.parameters
|
15
12
|
import metaflow.exception
|
13
|
+
import datetime
|
14
|
+
import typing
|
15
|
+
import metaflow.client.core
|
16
16
|
|
17
17
|
class Task(metaflow.client.core.MetaflowObject, metaclass=type):
|
18
|
+
"""
|
19
|
+
A `Task` represents an execution of a `Step`.
|
20
|
+
|
21
|
+
It contains all `DataArtifact` objects produced by the task as
|
22
|
+
well as metadata related to execution.
|
23
|
+
|
24
|
+
Note that the `@retry` decorator may cause multiple attempts of
|
25
|
+
the task to be present. Usually you want the latest attempt, which
|
26
|
+
is what instantiating a `Task` object returns by default. If
|
27
|
+
you need to e.g. retrieve logs from a failed attempt, you can
|
28
|
+
explicitly get information about a specific attempt by using the
|
29
|
+
following syntax when creating a task:
|
30
|
+
|
31
|
+
`Task('flow/run/step/task', attempt=<attempt>)`
|
32
|
+
|
33
|
+
where `attempt=0` corresponds to the first attempt etc.
|
34
|
+
|
35
|
+
Attributes
|
36
|
+
----------
|
37
|
+
metadata : List[Metadata]
|
38
|
+
List of all metadata events associated with the task.
|
39
|
+
metadata_dict : Dict[str, str]
|
40
|
+
A condensed version of `metadata`: A dictionary where keys
|
41
|
+
are names of metadata events and values the latest corresponding event.
|
42
|
+
data : MetaflowData
|
43
|
+
Container of all data artifacts produced by this task. Note that this
|
44
|
+
call downloads all data locally, so it can be slower than accessing
|
45
|
+
artifacts individually. See `MetaflowData` for more information.
|
46
|
+
artifacts : MetaflowArtifacts
|
47
|
+
Container of `DataArtifact` objects produced by this task.
|
48
|
+
successful : bool
|
49
|
+
True if the task completed successfully.
|
50
|
+
finished : bool
|
51
|
+
True if the task completed.
|
52
|
+
exception : object
|
53
|
+
Exception raised by this task if there was one.
|
54
|
+
finished_at : datetime
|
55
|
+
Time this task finished.
|
56
|
+
runtime_name : str
|
57
|
+
Runtime this task was executed on.
|
58
|
+
stdout : str
|
59
|
+
Standard output for the task execution.
|
60
|
+
stderr : str
|
61
|
+
Standard error output for the task execution.
|
62
|
+
code : MetaflowCode
|
63
|
+
Code package for this task (if present). See `MetaflowCode`.
|
64
|
+
environment_info : Dict[str, str]
|
65
|
+
Information about the execution environment.
|
66
|
+
"""
|
18
67
|
def __init__(self, *args, **kwargs):
|
19
68
|
...
|
20
69
|
@property
|
@@ -409,6 +458,9 @@ class CardDatastore(object, metaclass=type):
|
|
409
458
|
NUM_SHORT_HASH_CHARS: int
|
410
459
|
|
411
460
|
class CardClassFoundException(metaflow.exception.MetaflowException, metaclass=type):
|
461
|
+
"""
|
462
|
+
This exception is raised with MetaflowCard class is not present for a particular card type.
|
463
|
+
"""
|
412
464
|
def __init__(self, card_name):
|
413
465
|
...
|
414
466
|
...
|
@@ -424,6 +476,9 @@ class UnrenderableCardException(metaflow.exception.MetaflowException, metaclass=
|
|
424
476
|
...
|
425
477
|
|
426
478
|
class CardNotPresentException(metaflow.exception.MetaflowException, metaclass=type):
|
479
|
+
"""
|
480
|
+
This exception is raised with a card is not present in the datastore.
|
481
|
+
"""
|
427
482
|
def __init__(self, pathspec, card_type = None, card_hash = None, card_id = None):
|
428
483
|
...
|
429
484
|
...
|
@@ -440,6 +495,9 @@ def resumed_info(task):
|
|
440
495
|
...
|
441
496
|
|
442
497
|
class CardRenderInfo(tuple, metaclass=type):
|
498
|
+
"""
|
499
|
+
CardRenderInfo(mode, is_implemented, data, timed_out, timeout_stack_trace)
|
500
|
+
"""
|
443
501
|
@staticmethod
|
444
502
|
def __new__(_cls, mode, is_implemented, data, timed_out, timeout_stack_trace):
|
445
503
|
"""
|