ob-metaflow-stubs 6.0.3.123__py2.py3-none-any.whl → 6.0.3.126__py2.py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- metaflow-stubs/__init__.pyi +253 -248
- metaflow-stubs/cards.pyi +2 -2
- metaflow-stubs/cli.pyi +9 -21
- metaflow-stubs/{plugins/package_cli.pyi → cli_components/__init__.pyi} +3 -3
- metaflow-stubs/cli_components/utils.pyi +35 -0
- metaflow-stubs/client/__init__.pyi +2 -2
- metaflow-stubs/client/core.pyi +4 -4
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +10 -3
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +18 -11
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata_provider/__init__.pyi +2 -2
- metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
- metaflow-stubs/metadata_provider/metadata.pyi +2 -2
- metaflow-stubs/metadata_provider/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +4 -2
- metaflow-stubs/metaflow_current.pyi +58 -58
- metaflow-stubs/mf_extensions/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
- metaflow-stubs/{mflog → mf_extensions/outerbounds/plugins}/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +92 -0
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +3 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +33 -16
- metaflow-stubs/plugins/__init__.pyi +16 -15
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -5
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -3
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +31 -3
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +5 -5
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -3
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +3 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +5 -5
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -3
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +2 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +2 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +5 -5
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -5
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +6 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/snowflake/__init__.pyi +14 -0
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +30 -30
- metaflow-stubs/runner/deployer_impl.pyi +3 -3
- metaflow-stubs/runner/metaflow_runner.pyi +4 -4
- metaflow-stubs/runner/nbdeploy.pyi +4 -2
- metaflow-stubs/runner/nbrun.pyi +3 -3
- metaflow-stubs/runner/subprocess_manager.pyi +5 -2
- metaflow-stubs/runner/utils.pyi +8 -8
- metaflow-stubs/system/__init__.pyi +2 -2
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/{plugins/aws/step_functions/production_token.pyi → user_configs/__init__.pyi} +5 -11
- metaflow-stubs/user_configs/config_decorators.pyi +253 -0
- metaflow-stubs/user_configs/config_options.pyi +82 -0
- metaflow-stubs/user_configs/config_parameters.pyi +217 -0
- {ob_metaflow_stubs-6.0.3.123.dist-info → ob_metaflow_stubs-6.0.3.126.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.126.dist-info/RECORD +173 -0
- metaflow-stubs/clone_util.pyi +0 -14
- metaflow-stubs/mflog/mflog.pyi +0 -69
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +0 -103
- metaflow-stubs/plugins/cards/card_cli.pyi +0 -146
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +0 -33
- metaflow-stubs/plugins/logs_cli.pyi +0 -46
- metaflow-stubs/plugins/tag_cli.pyi +0 -19
- metaflow-stubs/procpoll.pyi +0 -53
- ob_metaflow_stubs-6.0.3.123.dist-info/RECORD +0 -174
- {ob_metaflow_stubs-6.0.3.123.dist-info → ob_metaflow_stubs-6.0.3.126.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.123.dist-info → ob_metaflow_stubs-6.0.3.126.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
######################################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-12-
|
3
|
+
# MF version: 2.12.39.1+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-12-17T01:10:55.898283 #
|
5
5
|
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -22,6 +22,7 @@ from .multicore_utils import parallel_map as parallel_map
|
|
22
22
|
from . import metaflow_current as metaflow_current
|
23
23
|
from .metaflow_current import current as current
|
24
24
|
from . import parameters as parameters
|
25
|
+
from . import user_configs as user_configs
|
25
26
|
from . import tagging_util as tagging_util
|
26
27
|
from . import metadata_provider as metadata_provider
|
27
28
|
from . import flowspec as flowspec
|
@@ -29,9 +30,13 @@ from .flowspec import FlowSpec as FlowSpec
|
|
29
30
|
from .parameters import Parameter as Parameter
|
30
31
|
from .parameters import JSONTypeClass as JSONTypeClass
|
31
32
|
from .parameters import JSONType as JSONType
|
33
|
+
from .user_configs.config_parameters import Config as Config
|
34
|
+
from .user_configs.config_parameters import config_expr as config_expr
|
35
|
+
from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
|
36
|
+
from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
|
37
|
+
from . import events as events
|
32
38
|
from . import tuple_util as tuple_util
|
33
39
|
from . import cards as cards
|
34
|
-
from . import events as events
|
35
40
|
from . import runner as runner
|
36
41
|
from . import plugins as plugins
|
37
42
|
from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
|
@@ -58,10 +63,10 @@ from .runner.nbdeploy import NBDeployer as NBDeployer
|
|
58
63
|
from .mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.final_api import Checkpoint as Checkpoint
|
59
64
|
from .mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures import load_model as load_model
|
60
65
|
from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import get_aws_client as get_aws_client
|
66
|
+
from .mf_extensions.outerbounds.plugins.snowflake.snowflake import Snowflake as Snowflake
|
67
|
+
from . import cli_components as cli_components
|
61
68
|
from . import system as system
|
62
69
|
from . import pylint_wrapper as pylint_wrapper
|
63
|
-
from . import procpoll as procpoll
|
64
|
-
from . import clone_util as clone_util
|
65
70
|
from . import cli as cli
|
66
71
|
from . import profilers as profilers
|
67
72
|
|
@@ -140,33 +145,6 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
140
145
|
"""
|
141
146
|
...
|
142
147
|
|
143
|
-
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
144
|
-
"""
|
145
|
-
Specifies that this step should execute on Kubernetes.
|
146
|
-
"""
|
147
|
-
...
|
148
|
-
|
149
|
-
@typing.overload
|
150
|
-
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
151
|
-
"""
|
152
|
-
Enables checkpointing for a step.
|
153
|
-
"""
|
154
|
-
...
|
155
|
-
|
156
|
-
@typing.overload
|
157
|
-
def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
158
|
-
...
|
159
|
-
|
160
|
-
@typing.overload
|
161
|
-
def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
162
|
-
...
|
163
|
-
|
164
|
-
def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
|
165
|
-
"""
|
166
|
-
Enables checkpointing for a step.
|
167
|
-
"""
|
168
|
-
...
|
169
|
-
|
170
148
|
@typing.overload
|
171
149
|
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
172
150
|
"""
|
@@ -199,85 +177,135 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
199
177
|
...
|
200
178
|
|
201
179
|
@typing.overload
|
202
|
-
def
|
180
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
203
181
|
"""
|
204
|
-
|
182
|
+
Decorator prototype for all step decorators. This function gets specialized
|
183
|
+
and imported for all decorators types by _import_plugin_decorators().
|
205
184
|
"""
|
206
185
|
...
|
207
186
|
|
208
187
|
@typing.overload
|
209
|
-
def
|
188
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
189
|
+
...
|
190
|
+
|
191
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
192
|
+
"""
|
193
|
+
Decorator prototype for all step decorators. This function gets specialized
|
194
|
+
and imported for all decorators types by _import_plugin_decorators().
|
195
|
+
"""
|
210
196
|
...
|
211
197
|
|
212
198
|
@typing.overload
|
213
|
-
def
|
199
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
200
|
+
"""
|
201
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
202
|
+
the execution of a step.
|
203
|
+
"""
|
214
204
|
...
|
215
205
|
|
216
|
-
|
206
|
+
@typing.overload
|
207
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
208
|
+
...
|
209
|
+
|
210
|
+
@typing.overload
|
211
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
212
|
+
...
|
213
|
+
|
214
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
217
215
|
"""
|
218
|
-
Specifies
|
216
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
217
|
+
the execution of a step.
|
219
218
|
"""
|
220
219
|
...
|
221
220
|
|
222
221
|
@typing.overload
|
223
|
-
def
|
222
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
224
223
|
"""
|
225
|
-
Specifies
|
224
|
+
Specifies a timeout for your step.
|
226
225
|
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
226
|
+
This decorator is useful if this step may hang indefinitely.
|
227
|
+
|
228
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
229
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
230
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
231
|
+
|
232
|
+
Note that all the values specified in parameters are added together so if you specify
|
233
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
231
234
|
"""
|
232
235
|
...
|
233
236
|
|
234
237
|
@typing.overload
|
235
|
-
def
|
238
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
236
239
|
...
|
237
240
|
|
238
241
|
@typing.overload
|
239
|
-
def
|
242
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
240
243
|
...
|
241
244
|
|
242
|
-
def
|
245
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
243
246
|
"""
|
244
|
-
Specifies
|
247
|
+
Specifies a timeout for your step.
|
245
248
|
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
249
|
+
This decorator is useful if this step may hang indefinitely.
|
250
|
+
|
251
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
252
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
253
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
254
|
+
|
255
|
+
Note that all the values specified in parameters are added together so if you specify
|
256
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
250
257
|
"""
|
251
258
|
...
|
252
259
|
|
253
|
-
|
260
|
+
@typing.overload
|
261
|
+
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
254
262
|
"""
|
255
|
-
|
263
|
+
Enables checkpointing for a step.
|
256
264
|
"""
|
257
265
|
...
|
258
266
|
|
259
267
|
@typing.overload
|
260
|
-
def
|
268
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
269
|
+
...
|
270
|
+
|
271
|
+
@typing.overload
|
272
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
273
|
+
...
|
274
|
+
|
275
|
+
def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
|
261
276
|
"""
|
262
|
-
|
277
|
+
Enables checkpointing for a step.
|
278
|
+
"""
|
279
|
+
...
|
280
|
+
|
281
|
+
@typing.overload
|
282
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
283
|
+
"""
|
284
|
+
Specifies that the step will success under all circumstances.
|
263
285
|
|
264
|
-
|
286
|
+
The decorator will create an optional artifact, specified by `var`, which
|
287
|
+
contains the exception raised. You can use it to detect the presence
|
288
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
289
|
+
are missing.
|
265
290
|
"""
|
266
291
|
...
|
267
292
|
|
268
293
|
@typing.overload
|
269
|
-
def
|
294
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
270
295
|
...
|
271
296
|
|
272
297
|
@typing.overload
|
273
|
-
def
|
298
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
274
299
|
...
|
275
300
|
|
276
|
-
def
|
301
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
277
302
|
"""
|
278
|
-
|
303
|
+
Specifies that the step will success under all circumstances.
|
279
304
|
|
280
|
-
|
305
|
+
The decorator will create an optional artifact, specified by `var`, which
|
306
|
+
contains the exception raised. You can use it to detect the presence
|
307
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
308
|
+
are missing.
|
281
309
|
"""
|
282
310
|
...
|
283
311
|
|
@@ -330,115 +358,105 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
330
358
|
"""
|
331
359
|
...
|
332
360
|
|
361
|
+
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
362
|
+
"""
|
363
|
+
Specifies that this step should execute on Kubernetes.
|
364
|
+
"""
|
365
|
+
...
|
366
|
+
|
333
367
|
@typing.overload
|
334
|
-
def
|
368
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
335
369
|
"""
|
336
|
-
|
337
|
-
|
370
|
+
Specifies the PyPI packages for the step.
|
371
|
+
|
372
|
+
Information in this decorator will augment any
|
373
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
374
|
+
you can use `@pypi_base` to set packages required by all
|
375
|
+
steps and use `@pypi` to specify step-specific overrides.
|
338
376
|
"""
|
339
377
|
...
|
340
378
|
|
341
379
|
@typing.overload
|
342
|
-
def
|
380
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
343
381
|
...
|
344
382
|
|
345
|
-
|
383
|
+
@typing.overload
|
384
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
385
|
+
...
|
386
|
+
|
387
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
346
388
|
"""
|
347
|
-
|
348
|
-
|
389
|
+
Specifies the PyPI packages for the step.
|
390
|
+
|
391
|
+
Information in this decorator will augment any
|
392
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
393
|
+
you can use `@pypi_base` to set packages required by all
|
394
|
+
steps and use `@pypi` to specify step-specific overrides.
|
395
|
+
"""
|
396
|
+
...
|
397
|
+
|
398
|
+
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
399
|
+
"""
|
400
|
+
Decorator that helps cache, version and store models/datasets from huggingface hub.
|
349
401
|
"""
|
350
402
|
...
|
351
403
|
|
352
404
|
@typing.overload
|
353
|
-
def
|
405
|
+
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
354
406
|
"""
|
355
|
-
|
356
|
-
|
357
|
-
The decorator will create an optional artifact, specified by `var`, which
|
358
|
-
contains the exception raised. You can use it to detect the presence
|
359
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
360
|
-
are missing.
|
407
|
+
Enables loading / saving of models within a step.
|
361
408
|
"""
|
362
409
|
...
|
363
410
|
|
364
411
|
@typing.overload
|
365
|
-
def
|
412
|
+
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
366
413
|
...
|
367
414
|
|
368
415
|
@typing.overload
|
369
|
-
def
|
416
|
+
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
370
417
|
...
|
371
418
|
|
372
|
-
def
|
419
|
+
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
373
420
|
"""
|
374
|
-
|
375
|
-
|
376
|
-
The decorator will create an optional artifact, specified by `var`, which
|
377
|
-
contains the exception raised. You can use it to detect the presence
|
378
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
379
|
-
are missing.
|
421
|
+
Enables loading / saving of models within a step.
|
380
422
|
"""
|
381
423
|
...
|
382
424
|
|
383
425
|
@typing.overload
|
384
|
-
def
|
426
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
385
427
|
"""
|
386
|
-
Specifies
|
387
|
-
|
388
|
-
This decorator is useful if this step may hang indefinitely.
|
389
|
-
|
390
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
391
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
392
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
393
|
-
|
394
|
-
Note that all the values specified in parameters are added together so if you specify
|
395
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
428
|
+
Specifies environment variables to be set prior to the execution of a step.
|
396
429
|
"""
|
397
430
|
...
|
398
431
|
|
399
432
|
@typing.overload
|
400
|
-
def
|
433
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
401
434
|
...
|
402
435
|
|
403
436
|
@typing.overload
|
404
|
-
def
|
437
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
405
438
|
...
|
406
439
|
|
407
|
-
def
|
440
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
408
441
|
"""
|
409
|
-
Specifies
|
410
|
-
|
411
|
-
This decorator is useful if this step may hang indefinitely.
|
412
|
-
|
413
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
414
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
415
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
416
|
-
|
417
|
-
Note that all the values specified in parameters are added together so if you specify
|
418
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
442
|
+
Specifies environment variables to be set prior to the execution of a step.
|
419
443
|
"""
|
420
444
|
...
|
421
445
|
|
422
446
|
@typing.overload
|
423
|
-
def
|
447
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
424
448
|
"""
|
425
|
-
|
426
|
-
the execution of a step.
|
449
|
+
Internal decorator to support Fast bakery
|
427
450
|
"""
|
428
451
|
...
|
429
452
|
|
430
453
|
@typing.overload
|
431
|
-
def
|
432
|
-
...
|
433
|
-
|
434
|
-
@typing.overload
|
435
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
454
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
436
455
|
...
|
437
456
|
|
438
|
-
def
|
457
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
439
458
|
"""
|
440
|
-
|
441
|
-
the execution of a step.
|
459
|
+
Internal decorator to support Fast bakery
|
442
460
|
"""
|
443
461
|
...
|
444
462
|
|
@@ -482,40 +500,60 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
482
500
|
...
|
483
501
|
|
484
502
|
@typing.overload
|
485
|
-
def
|
503
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
486
504
|
"""
|
487
|
-
|
505
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
506
|
+
|
507
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
488
508
|
"""
|
489
509
|
...
|
490
510
|
|
491
511
|
@typing.overload
|
492
|
-
def
|
512
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
493
513
|
...
|
494
514
|
|
495
|
-
|
515
|
+
@typing.overload
|
516
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
517
|
+
...
|
518
|
+
|
519
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
496
520
|
"""
|
497
|
-
|
521
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
522
|
+
|
523
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
498
524
|
"""
|
499
525
|
...
|
500
526
|
|
501
|
-
|
502
|
-
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
527
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
503
528
|
"""
|
504
|
-
|
529
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
530
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
531
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
532
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
533
|
+
starts only after all sensors finish.
|
505
534
|
"""
|
506
535
|
...
|
507
536
|
|
508
537
|
@typing.overload
|
509
|
-
def
|
538
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
539
|
+
"""
|
540
|
+
Specifies the PyPI packages for all steps of the flow.
|
541
|
+
|
542
|
+
Use `@pypi_base` to set common packages required by all
|
543
|
+
steps and use `@pypi` to specify step-specific overrides.
|
544
|
+
"""
|
510
545
|
...
|
511
546
|
|
512
547
|
@typing.overload
|
513
|
-
def
|
548
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
514
549
|
...
|
515
550
|
|
516
|
-
def
|
551
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
517
552
|
"""
|
518
|
-
|
553
|
+
Specifies the PyPI packages for all steps of the flow.
|
554
|
+
|
555
|
+
Use `@pypi_base` to set common packages required by all
|
556
|
+
steps and use `@pypi` to specify step-specific overrides.
|
519
557
|
"""
|
520
558
|
...
|
521
559
|
|
@@ -544,115 +582,53 @@ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[Fl
|
|
544
582
|
...
|
545
583
|
|
546
584
|
@typing.overload
|
547
|
-
def
|
585
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
548
586
|
"""
|
549
|
-
Specifies the
|
550
|
-
|
551
|
-
```
|
552
|
-
@trigger(event='foo')
|
553
|
-
```
|
554
|
-
or
|
555
|
-
```
|
556
|
-
@trigger(events=['foo', 'bar'])
|
557
|
-
```
|
558
|
-
|
559
|
-
Additionally, you can specify the parameter mappings
|
560
|
-
to map event payload to Metaflow parameters for the flow.
|
561
|
-
```
|
562
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
563
|
-
```
|
564
|
-
or
|
565
|
-
```
|
566
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
567
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
568
|
-
```
|
569
|
-
|
570
|
-
'parameters' can also be a list of strings and tuples like so:
|
571
|
-
```
|
572
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
573
|
-
```
|
574
|
-
This is equivalent to:
|
575
|
-
```
|
576
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
577
|
-
```
|
587
|
+
Specifies the times when the flow should be run when running on a
|
588
|
+
production scheduler.
|
578
589
|
"""
|
579
590
|
...
|
580
591
|
|
581
592
|
@typing.overload
|
582
|
-
def
|
583
|
-
...
|
584
|
-
|
585
|
-
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
586
|
-
"""
|
587
|
-
Specifies the event(s) that this flow depends on.
|
588
|
-
|
589
|
-
```
|
590
|
-
@trigger(event='foo')
|
591
|
-
```
|
592
|
-
or
|
593
|
-
```
|
594
|
-
@trigger(events=['foo', 'bar'])
|
595
|
-
```
|
596
|
-
|
597
|
-
Additionally, you can specify the parameter mappings
|
598
|
-
to map event payload to Metaflow parameters for the flow.
|
599
|
-
```
|
600
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
601
|
-
```
|
602
|
-
or
|
603
|
-
```
|
604
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
605
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
606
|
-
```
|
607
|
-
|
608
|
-
'parameters' can also be a list of strings and tuples like so:
|
609
|
-
```
|
610
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
611
|
-
```
|
612
|
-
This is equivalent to:
|
613
|
-
```
|
614
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
615
|
-
```
|
616
|
-
"""
|
593
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
617
594
|
...
|
618
595
|
|
619
|
-
def
|
596
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
620
597
|
"""
|
621
|
-
|
622
|
-
|
623
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
624
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
625
|
-
starts only after all sensors finish.
|
598
|
+
Specifies the times when the flow should be run when running on a
|
599
|
+
production scheduler.
|
626
600
|
"""
|
627
601
|
...
|
628
602
|
|
629
603
|
@typing.overload
|
630
|
-
def
|
604
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
631
605
|
"""
|
632
|
-
Specifies the
|
606
|
+
Specifies the Conda environment for all steps of the flow.
|
633
607
|
|
634
|
-
Use `@
|
635
|
-
steps and use `@
|
608
|
+
Use `@conda_base` to set common libraries required by all
|
609
|
+
steps and use `@conda` to specify step-specific additions.
|
636
610
|
"""
|
637
611
|
...
|
638
612
|
|
639
613
|
@typing.overload
|
640
|
-
def
|
614
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
641
615
|
...
|
642
616
|
|
643
|
-
def
|
617
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
644
618
|
"""
|
645
|
-
Specifies the
|
619
|
+
Specifies the Conda environment for all steps of the flow.
|
646
620
|
|
647
|
-
Use `@
|
648
|
-
steps and use `@
|
621
|
+
Use `@conda_base` to set common libraries required by all
|
622
|
+
steps and use `@conda` to specify step-specific additions.
|
649
623
|
"""
|
650
624
|
...
|
651
625
|
|
652
|
-
def
|
626
|
+
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
653
627
|
"""
|
654
|
-
|
655
|
-
|
628
|
+
Specifies what flows belong to the same project.
|
629
|
+
|
630
|
+
A project-specific namespace is created for all flows that
|
631
|
+
use the same `@project(name)`.
|
656
632
|
"""
|
657
633
|
...
|
658
634
|
|
@@ -737,54 +713,83 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
737
713
|
"""
|
738
714
|
...
|
739
715
|
|
740
|
-
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
741
|
-
"""
|
742
|
-
Specifies what flows belong to the same project.
|
743
|
-
|
744
|
-
A project-specific namespace is created for all flows that
|
745
|
-
use the same `@project(name)`.
|
746
|
-
"""
|
747
|
-
...
|
748
|
-
|
749
716
|
@typing.overload
|
750
|
-
def
|
717
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
751
718
|
"""
|
752
|
-
Specifies the
|
719
|
+
Specifies the event(s) that this flow depends on.
|
753
720
|
|
754
|
-
|
755
|
-
|
721
|
+
```
|
722
|
+
@trigger(event='foo')
|
723
|
+
```
|
724
|
+
or
|
725
|
+
```
|
726
|
+
@trigger(events=['foo', 'bar'])
|
727
|
+
```
|
728
|
+
|
729
|
+
Additionally, you can specify the parameter mappings
|
730
|
+
to map event payload to Metaflow parameters for the flow.
|
731
|
+
```
|
732
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
733
|
+
```
|
734
|
+
or
|
735
|
+
```
|
736
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
737
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
738
|
+
```
|
739
|
+
|
740
|
+
'parameters' can also be a list of strings and tuples like so:
|
741
|
+
```
|
742
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
743
|
+
```
|
744
|
+
This is equivalent to:
|
745
|
+
```
|
746
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
747
|
+
```
|
756
748
|
"""
|
757
749
|
...
|
758
750
|
|
759
751
|
@typing.overload
|
760
|
-
def
|
752
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
761
753
|
...
|
762
754
|
|
763
|
-
def
|
755
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
764
756
|
"""
|
765
|
-
Specifies the
|
757
|
+
Specifies the event(s) that this flow depends on.
|
766
758
|
|
767
|
-
|
768
|
-
|
769
|
-
|
770
|
-
|
771
|
-
|
772
|
-
@
|
773
|
-
|
774
|
-
|
775
|
-
|
776
|
-
|
759
|
+
```
|
760
|
+
@trigger(event='foo')
|
761
|
+
```
|
762
|
+
or
|
763
|
+
```
|
764
|
+
@trigger(events=['foo', 'bar'])
|
765
|
+
```
|
766
|
+
|
767
|
+
Additionally, you can specify the parameter mappings
|
768
|
+
to map event payload to Metaflow parameters for the flow.
|
769
|
+
```
|
770
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
771
|
+
```
|
772
|
+
or
|
773
|
+
```
|
774
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
775
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
776
|
+
```
|
777
|
+
|
778
|
+
'parameters' can also be a list of strings and tuples like so:
|
779
|
+
```
|
780
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
781
|
+
```
|
782
|
+
This is equivalent to:
|
783
|
+
```
|
784
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
785
|
+
```
|
777
786
|
"""
|
778
787
|
...
|
779
788
|
|
780
|
-
|
781
|
-
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
782
|
-
...
|
783
|
-
|
784
|
-
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
789
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
785
790
|
"""
|
786
|
-
|
787
|
-
|
791
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
792
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
788
793
|
"""
|
789
794
|
...
|
790
795
|
|