ob-metaflow-stubs 6.0.3.104__py2.py3-none-any.whl → 6.0.3.106__py2.py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- metaflow-stubs/__init__.pyi +853 -515
- metaflow-stubs/cards.pyi +211 -5
- metaflow-stubs/cli.pyi +22 -2
- metaflow-stubs/client/__init__.pyi +128 -3
- metaflow-stubs/client/core.pyi +226 -6
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +5 -2
- metaflow-stubs/events.pyi +20 -2
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +70 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +156 -3
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata/metadata.pyi +8 -2
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +22 -19
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +68 -2
- metaflow-stubs/plugins/__init__.pyi +14 -3
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +80 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +5 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +44 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +44 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +16 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +17 -4
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +98 -7
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +32 -4
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +50 -5
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +102 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +15 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +21 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +49 -4
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +21 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +63 -5
- metaflow-stubs/plugins/cards/card_client.pyi +34 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +5 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +8 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +52 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +28 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +41 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +28 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +183 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +5 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +35 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +55 -2
- metaflow-stubs/plugins/cards/exception.pyi +8 -2
- metaflow-stubs/plugins/catch_decorator.pyi +19 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +63 -3
- metaflow-stubs/plugins/datatools/local.pyi +16 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +72 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +83 -6
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +10 -2
- metaflow-stubs/plugins/events_decorator.pyi +106 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +24 -3
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +15 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +11 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +65 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +99 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +5 -2
- metaflow-stubs/plugins/logs_cli.pyi +3 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +29 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +59 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +45 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +33 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +33 -2
- metaflow-stubs/plugins/retry_decorator.pyi +21 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +11 -2
- metaflow-stubs/plugins/storage_executor.pyi +6 -2
- metaflow-stubs/plugins/tag_cli.pyi +37 -6
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +10 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +23 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +70 -131
- metaflow-stubs/runner/metaflow_runner.pyi +118 -10
- metaflow-stubs/runner/nbdeploy.pyi +66 -2
- metaflow-stubs/runner/nbrun.pyi +79 -2
- metaflow-stubs/runner/subprocess_manager.pyi +16 -4
- metaflow-stubs/runner/utils.pyi +32 -2
- metaflow-stubs/system/__init__.pyi +3 -3
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.106.dist-info/RECORD +140 -0
- ob_metaflow_stubs-6.0.3.104.dist-info/RECORD +0 -140
- {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.104.dist-info → ob_metaflow_stubs-6.0.3.106.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.931186 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -23,6 +23,9 @@ class MetaflowException(Exception, metaclass=type):
|
|
23
23
|
...
|
24
24
|
|
25
25
|
class MetaDatum(tuple, metaclass=type):
|
26
|
+
"""
|
27
|
+
MetaDatum(field, value, type, tags)
|
28
|
+
"""
|
26
29
|
@staticmethod
|
27
30
|
def __new__(_cls, field, value, type, tags):
|
28
31
|
"""
|
@@ -44,6 +47,9 @@ class MetaDatum(tuple, metaclass=type):
|
|
44
47
|
current: metaflow.metaflow_current.Current
|
45
48
|
|
46
49
|
class Parallel(tuple, metaclass=type):
|
50
|
+
"""
|
51
|
+
Parallel(main_ip, num_nodes, node_index, control_task_id)
|
52
|
+
"""
|
47
53
|
@staticmethod
|
48
54
|
def __new__(_cls, main_ip, num_nodes, node_index, control_task_id):
|
49
55
|
"""
|
@@ -63,6 +69,27 @@ class Parallel(tuple, metaclass=type):
|
|
63
69
|
...
|
64
70
|
|
65
71
|
class ParallelDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
72
|
+
"""
|
73
|
+
MF Add To Current
|
74
|
+
-----------------
|
75
|
+
parallel -> metaflow.metaflow_current.Parallel
|
76
|
+
|
77
|
+
@@ Returns
|
78
|
+
-------
|
79
|
+
Parallel
|
80
|
+
`namedtuple` with the following fields:
|
81
|
+
- main_ip : str
|
82
|
+
The IP address of the control task.
|
83
|
+
- num_nodes : int
|
84
|
+
The total number of tasks created by @parallel
|
85
|
+
- node_index : int
|
86
|
+
The index of the current task in all the @parallel tasks.
|
87
|
+
- control_task_id : Optional[str]
|
88
|
+
The task ID of the control task. Available to all tasks.
|
89
|
+
|
90
|
+
is_parallel -> bool
|
91
|
+
True if the current step is a @parallel step.
|
92
|
+
"""
|
66
93
|
def __init__(self, attributes = None, statically_defined = False):
|
67
94
|
...
|
68
95
|
def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.885187 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.932666 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -25,6 +25,63 @@ VALID_NAME_RE: str
|
|
25
25
|
VALID_NAME_LEN: int
|
26
26
|
|
27
27
|
class ProjectDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
|
28
|
+
"""
|
29
|
+
Specifies what flows belong to the same project.
|
30
|
+
|
31
|
+
A project-specific namespace is created for all flows that
|
32
|
+
use the same `@project(name)`.
|
33
|
+
|
34
|
+
Parameters
|
35
|
+
----------
|
36
|
+
name : str
|
37
|
+
Project name. Make sure that the name is unique amongst all
|
38
|
+
projects that use the same production scheduler. The name may
|
39
|
+
contain only lowercase alphanumeric characters and underscores.
|
40
|
+
|
41
|
+
MF Add To Current
|
42
|
+
-----------------
|
43
|
+
project_name -> str
|
44
|
+
The name of the project assigned to this flow, i.e. `X` in `@project(name=X)`.
|
45
|
+
|
46
|
+
@@ Returns
|
47
|
+
-------
|
48
|
+
str
|
49
|
+
Project name.
|
50
|
+
|
51
|
+
project_flow_name -> str
|
52
|
+
The flow name prefixed with the current project and branch. This name identifies
|
53
|
+
the deployment on a production scheduler.
|
54
|
+
|
55
|
+
@@ Returns
|
56
|
+
-------
|
57
|
+
str
|
58
|
+
Flow name prefixed with project information.
|
59
|
+
|
60
|
+
branch_name -> str
|
61
|
+
The current branch, i.e. `X` in `--branch=X` set during deployment or run.
|
62
|
+
|
63
|
+
@@ Returns
|
64
|
+
-------
|
65
|
+
str
|
66
|
+
Branch name.
|
67
|
+
|
68
|
+
is_user_branch -> bool
|
69
|
+
True if the flow is deployed without a specific `--branch` or a `--production`
|
70
|
+
flag.
|
71
|
+
|
72
|
+
@@ Returns
|
73
|
+
-------
|
74
|
+
bool
|
75
|
+
True if the deployment does not correspond to a specific branch.
|
76
|
+
|
77
|
+
is_production -> bool
|
78
|
+
True if the flow is deployed with the `--production` flag
|
79
|
+
|
80
|
+
@@ Returns
|
81
|
+
-------
|
82
|
+
bool
|
83
|
+
True if the flow is deployed with `--production`.
|
84
|
+
"""
|
28
85
|
def flow_init(self, flow, graph, environment, flow_datastore, metadata, logger, echo, options):
|
29
86
|
...
|
30
87
|
def get_top_level_options(self):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.929558 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.961633 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -13,6 +13,9 @@ if typing.TYPE_CHECKING:
|
|
13
13
|
EXT_PKG: str
|
14
14
|
|
15
15
|
class MetaDatum(tuple, metaclass=type):
|
16
|
+
"""
|
17
|
+
MetaDatum(field, value, type, tags)
|
18
|
+
"""
|
16
19
|
@staticmethod
|
17
20
|
def __new__(_cls, field, value, type, tags):
|
18
21
|
"""
|
@@ -34,6 +37,27 @@ class MetaDatum(tuple, metaclass=type):
|
|
34
37
|
INFO_FILE: str
|
35
38
|
|
36
39
|
class CondaStepDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
40
|
+
"""
|
41
|
+
Specifies the Conda environment for the step.
|
42
|
+
|
43
|
+
Information in this decorator will augment any
|
44
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
45
|
+
you can use `@conda_base` to set packages required by all
|
46
|
+
steps and use `@conda` to specify step-specific overrides.
|
47
|
+
|
48
|
+
Parameters
|
49
|
+
----------
|
50
|
+
packages : Dict[str, str], default {}
|
51
|
+
Packages to use for this step. The key is the name of the package
|
52
|
+
and the value is the version to use.
|
53
|
+
libraries : Dict[str, str], default {}
|
54
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
55
|
+
python : str, optional, default None
|
56
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
57
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
58
|
+
disabled : bool, default False
|
59
|
+
If set to True, disables @conda.
|
60
|
+
"""
|
37
61
|
def __init__(self, attributes = None, statically_defined = False):
|
38
62
|
...
|
39
63
|
def is_attribute_user_defined(self, name):
|
@@ -53,6 +77,25 @@ class CondaStepDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
|
53
77
|
...
|
54
78
|
|
55
79
|
class CondaFlowDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
|
80
|
+
"""
|
81
|
+
Specifies the Conda environment for all steps of the flow.
|
82
|
+
|
83
|
+
Use `@conda_base` to set common libraries required by all
|
84
|
+
steps and use `@conda` to specify step-specific additions.
|
85
|
+
|
86
|
+
Parameters
|
87
|
+
----------
|
88
|
+
packages : Dict[str, str], default {}
|
89
|
+
Packages to use for this flow. The key is the name of the package
|
90
|
+
and the value is the version to use.
|
91
|
+
libraries : Dict[str, str], default {}
|
92
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
93
|
+
python : str, optional, default None
|
94
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
95
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
96
|
+
disabled : bool, default False
|
97
|
+
If set to True, disables Conda.
|
98
|
+
"""
|
56
99
|
def __init__(self, attributes = None, statically_defined = False):
|
57
100
|
...
|
58
101
|
def is_attribute_user_defined(self, name):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.963058 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -10,8 +10,8 @@ import typing
|
|
10
10
|
if typing.TYPE_CHECKING:
|
11
11
|
import abc
|
12
12
|
import metaflow.exception
|
13
|
-
import metaflow.metaflow_environment
|
14
13
|
import io
|
14
|
+
import metaflow.metaflow_environment
|
15
15
|
|
16
16
|
class MetaflowException(Exception, metaclass=type):
|
17
17
|
def __init__(self, msg = "", lineno = None):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.961970 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -11,6 +11,23 @@ if typing.TYPE_CHECKING:
|
|
11
11
|
import metaflow.decorators
|
12
12
|
|
13
13
|
class PyPIStepDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
14
|
+
"""
|
15
|
+
Specifies the PyPI packages for the step.
|
16
|
+
|
17
|
+
Information in this decorator will augment any
|
18
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
19
|
+
you can use `@pypi_base` to set packages required by all
|
20
|
+
steps and use `@pypi` to specify step-specific overrides.
|
21
|
+
|
22
|
+
Parameters
|
23
|
+
----------
|
24
|
+
packages : Dict[str, str], default: {}
|
25
|
+
Packages to use for this step. The key is the name of the package
|
26
|
+
and the value is the version to use.
|
27
|
+
python : str, optional, default: None
|
28
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
29
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
30
|
+
"""
|
14
31
|
def __init__(self, attributes = None, statically_defined = False):
|
15
32
|
...
|
16
33
|
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
@@ -20,6 +37,20 @@ class PyPIStepDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
|
20
37
|
...
|
21
38
|
|
22
39
|
class PyPIFlowDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
|
40
|
+
"""
|
41
|
+
Specifies the PyPI packages for all steps of the flow.
|
42
|
+
|
43
|
+
Use `@pypi_base` to set common packages required by all
|
44
|
+
steps and use `@pypi` to specify step-specific overrides.
|
45
|
+
Parameters
|
46
|
+
----------
|
47
|
+
packages : Dict[str, str], default: {}
|
48
|
+
Packages to use for this flow. The key is the name of the package
|
49
|
+
and the value is the version to use.
|
50
|
+
python : str, optional, default: None
|
51
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
52
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
53
|
+
"""
|
23
54
|
def __init__(self, attributes = None, statically_defined = False):
|
24
55
|
...
|
25
56
|
def flow_init(self, flow, graph, environment, flow_datastore, metadata, logger, echo, options):
|
@@ -1,15 +1,15 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.963450 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.metaflow_environment
|
12
11
|
import metaflow.plugins.pypi.conda_environment
|
12
|
+
import metaflow.metaflow_environment
|
13
13
|
|
14
14
|
class CondaEnvironment(metaflow.metaflow_environment.MetaflowEnvironment, metaclass=type):
|
15
15
|
def __init__(self, flow):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.962261 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.931326 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -11,5 +11,36 @@ if typing.TYPE_CHECKING:
|
|
11
11
|
import metaflow.decorators
|
12
12
|
|
13
13
|
class ResourcesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
14
|
+
"""
|
15
|
+
Specifies the resources needed when executing this step.
|
16
|
+
|
17
|
+
Use `@resources` to specify the resource requirements
|
18
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
19
|
+
|
20
|
+
You can choose the compute layer on the command line by executing e.g.
|
21
|
+
```
|
22
|
+
python myflow.py run --with batch
|
23
|
+
```
|
24
|
+
or
|
25
|
+
```
|
26
|
+
python myflow.py run --with kubernetes
|
27
|
+
```
|
28
|
+
which executes the flow on the desired system using the
|
29
|
+
requirements specified in `@resources`.
|
30
|
+
|
31
|
+
Parameters
|
32
|
+
----------
|
33
|
+
cpu : int, default 1
|
34
|
+
Number of CPUs required for this step.
|
35
|
+
gpu : int, default 0
|
36
|
+
Number of GPUs required for this step.
|
37
|
+
disk : int, optional, default None
|
38
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
39
|
+
memory : int, default 4096
|
40
|
+
Memory size (in MB) required for this step.
|
41
|
+
shared_memory : int, optional, default None
|
42
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
43
|
+
This parameter maps to the `--shm-size` option in Docker.
|
44
|
+
"""
|
14
45
|
...
|
15
46
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.931588 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -20,6 +20,25 @@ class MetaflowException(Exception, metaclass=type):
|
|
20
20
|
MAX_ATTEMPTS: int
|
21
21
|
|
22
22
|
class RetryDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
23
|
+
"""
|
24
|
+
Specifies the number of times the task corresponding
|
25
|
+
to a step needs to be retried.
|
26
|
+
|
27
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
28
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
29
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
30
|
+
|
31
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
32
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
33
|
+
ensuring that the flow execution can continue.
|
34
|
+
|
35
|
+
Parameters
|
36
|
+
----------
|
37
|
+
times : int, default 3
|
38
|
+
Number of times to retry this task.
|
39
|
+
minutes_between_retries : int, default 2
|
40
|
+
Number of minutes between retries.
|
41
|
+
"""
|
23
42
|
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
24
43
|
...
|
25
44
|
def step_task_retry_count(self):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.930296 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.964305 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.964074 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -69,6 +69,15 @@ def get_secrets_backend_provider(secrets_backend_type):
|
|
69
69
|
...
|
70
70
|
|
71
71
|
class SecretsDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
72
|
+
"""
|
73
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
74
|
+
the execution of a step.
|
75
|
+
|
76
|
+
Parameters
|
77
|
+
----------
|
78
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
79
|
+
List of secret specs, defining how the secrets are to be retrieved
|
80
|
+
"""
|
72
81
|
def task_pre_step(self, step_name, task_datastore, metadata, run_id, task_id, flow, graph, retry_count, max_user_code_retries, ubf_context, inputs):
|
73
82
|
...
|
74
83
|
...
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.929271 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -15,6 +15,10 @@ class MetaflowException(Exception, metaclass=type):
|
|
15
15
|
...
|
16
16
|
|
17
17
|
class StorageExecutor(object, metaclass=type):
|
18
|
+
"""
|
19
|
+
Thin wrapper around a ProcessPoolExecutor, or a ThreadPoolExecutor where
|
20
|
+
the former may be unsafe.
|
21
|
+
"""
|
18
22
|
def __init__(self, use_processes = False):
|
19
23
|
...
|
20
24
|
def warm_up(self):
|
@@ -1,18 +1,18 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.936527 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.client.core
|
12
|
-
import metaflow.events
|
13
|
-
import metaflow.metaflow_current
|
14
|
-
import datetime
|
15
11
|
import metaflow.exception
|
12
|
+
import datetime
|
13
|
+
import metaflow.metaflow_current
|
14
|
+
import metaflow.events
|
15
|
+
import metaflow.client.core
|
16
16
|
|
17
17
|
def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
|
18
18
|
"""
|
@@ -35,6 +35,17 @@ def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
|
|
35
35
|
...
|
36
36
|
|
37
37
|
class Flow(metaflow.client.core.MetaflowObject, metaclass=type):
|
38
|
+
"""
|
39
|
+
A Flow represents all existing flows with a certain name, in other words,
|
40
|
+
classes derived from `FlowSpec`. A container of `Run` objects.
|
41
|
+
|
42
|
+
Attributes
|
43
|
+
----------
|
44
|
+
latest_run : Run
|
45
|
+
Latest `Run` (in progress or completed, successfully or not) of this flow.
|
46
|
+
latest_successful_run : Run
|
47
|
+
Latest successfully completed `Run` of this flow.
|
48
|
+
"""
|
38
49
|
def __init__(self, *args, **kwargs):
|
39
50
|
...
|
40
51
|
@property
|
@@ -121,6 +132,26 @@ class Flow(metaflow.client.core.MetaflowObject, metaclass=type):
|
|
121
132
|
...
|
122
133
|
|
123
134
|
class Run(metaflow.client.core.MetaflowObject, metaclass=type):
|
135
|
+
"""
|
136
|
+
A `Run` represents an execution of a `Flow`. It is a container of `Step`s.
|
137
|
+
|
138
|
+
Attributes
|
139
|
+
----------
|
140
|
+
data : MetaflowData
|
141
|
+
a shortcut to run['end'].task.data, i.e. data produced by this run.
|
142
|
+
successful : bool
|
143
|
+
True if the run completed successfully.
|
144
|
+
finished : bool
|
145
|
+
True if the run completed.
|
146
|
+
finished_at : datetime
|
147
|
+
Time this run finished.
|
148
|
+
code : MetaflowCode
|
149
|
+
Code package for this run (if present). See `MetaflowCode`.
|
150
|
+
trigger : MetaflowTrigger
|
151
|
+
Information about event(s) that triggered this run (if present). See `MetaflowTrigger`.
|
152
|
+
end_task : Task
|
153
|
+
`Task` for the end step (if it is present already).
|
154
|
+
"""
|
124
155
|
def steps(self, *tags: str) -> typing.Iterator[metaflow.client.core.Step]:
|
125
156
|
"""
|
126
157
|
[Legacy function - do not use]
|
@@ -1,15 +1,15 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.927364 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.decorators
|
12
11
|
import metaflow.unbounded_foreach
|
12
|
+
import metaflow.decorators
|
13
13
|
|
14
14
|
class MetaflowException(Exception, metaclass=type):
|
15
15
|
def __init__(self, msg = "", lineno = None):
|
@@ -25,6 +25,9 @@ UBF_TASK: str
|
|
25
25
|
CONTROL_TASK_TAG: str
|
26
26
|
|
27
27
|
class MetaDatum(tuple, metaclass=type):
|
28
|
+
"""
|
29
|
+
MetaDatum(field, value, type, tags)
|
30
|
+
"""
|
28
31
|
@staticmethod
|
29
32
|
def __new__(_cls, field, value, type, tags):
|
30
33
|
"""
|
@@ -44,6 +47,10 @@ class MetaDatum(tuple, metaclass=type):
|
|
44
47
|
...
|
45
48
|
|
46
49
|
class InternalTestUnboundedForeachInput(metaflow.unbounded_foreach.UnboundedForeachInput, metaclass=type):
|
50
|
+
"""
|
51
|
+
Test class that wraps around values (any iterator) and simulates an
|
52
|
+
unbounded-foreach instead of a bounded foreach.
|
53
|
+
"""
|
47
54
|
def __init__(self, iterable):
|
48
55
|
...
|
49
56
|
def __iter__(self):
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.931992 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -26,6 +26,27 @@ class TimeoutException(metaflow.exception.MetaflowException, metaclass=type):
|
|
26
26
|
...
|
27
27
|
|
28
28
|
class TimeoutDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
29
|
+
"""
|
30
|
+
Specifies a timeout for your step.
|
31
|
+
|
32
|
+
This decorator is useful if this step may hang indefinitely.
|
33
|
+
|
34
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
35
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
36
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
37
|
+
|
38
|
+
Note that all the values specified in parameters are added together so if you specify
|
39
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
40
|
+
|
41
|
+
Parameters
|
42
|
+
----------
|
43
|
+
seconds : int, default 0
|
44
|
+
Number of seconds to wait prior to timing out.
|
45
|
+
minutes : int, default 0
|
46
|
+
Number of minutes to wait prior to timing out.
|
47
|
+
hours : int, default 0
|
48
|
+
Number of hours to wait prior to timing out.
|
49
|
+
"""
|
29
50
|
def __init__(self, *args, **kwargs):
|
30
51
|
...
|
31
52
|
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
metaflow-stubs/procpoll.pyi
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.912561 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.24.1+obcheckpoint(0.0.11);ob(v1) #
|
4
|
+
# Generated on 2024-10-04T19:10:58.885415 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|