ob-metaflow-stubs 6.0.3.165__py2.py3-none-any.whl → 6.0.3.167__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +670 -670
- metaflow-stubs/cards.pyi +2 -2
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/cli_components/__init__.pyi +2 -2
- metaflow-stubs/cli_components/utils.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +2 -2
- metaflow-stubs/client/core.pyi +4 -4
- metaflow-stubs/client/filecache.pyi +3 -3
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +4 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +2 -2
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata_provider/__init__.pyi +2 -2
- metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
- metaflow-stubs/metadata_provider/metadata.pyi +2 -2
- metaflow-stubs/metadata_provider/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +4 -2
- metaflow-stubs/metaflow_current.pyi +80 -80
- metaflow-stubs/metaflow_git.pyi +2 -2
- metaflow-stubs/mf_extensions/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +9 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +3 -3
- metaflow-stubs/plugins/__init__.pyi +13 -13
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +6 -6
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +2 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +2 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
- metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
- metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/plugins/uv/__init__.pyi +2 -2
- metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +6 -6
- metaflow-stubs/runner/deployer_impl.pyi +3 -3
- metaflow-stubs/runner/metaflow_runner.pyi +3 -3
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +3 -3
- metaflow-stubs/system/__init__.pyi +2 -2
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/user_configs/__init__.pyi +2 -2
- metaflow-stubs/user_configs/config_decorators.pyi +5 -5
- metaflow-stubs/user_configs/config_options.pyi +2 -2
- metaflow-stubs/user_configs/config_parameters.pyi +5 -5
- {ob_metaflow_stubs-6.0.3.165.dist-info → ob_metaflow_stubs-6.0.3.167.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.167.dist-info/RECORD +206 -0
- ob_metaflow_stubs-6.0.3.165.dist-info/RECORD +0 -206
- {ob_metaflow_stubs-6.0.3.165.dist-info → ob_metaflow_stubs-6.0.3.167.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.165.dist-info → ob_metaflow_stubs-6.0.3.167.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
######################################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.15.11.
|
4
|
-
# Generated on 2025-05-
|
3
|
+
# MF version: 2.15.11.2+obcheckpoint(0.2.1);ob(v1) #
|
4
|
+
# Generated on 2025-05-13T17:00:07.187688 #
|
5
5
|
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import typing
|
12
11
|
import datetime
|
12
|
+
import typing
|
13
13
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
14
14
|
StepFlag = typing.NewType("StepFlag", bool)
|
15
15
|
|
@@ -35,18 +35,18 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
|
|
35
35
|
from .user_configs.config_parameters import config_expr as config_expr
|
36
36
|
from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
|
37
37
|
from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
|
38
|
-
from . import
|
38
|
+
from . import events as events
|
39
39
|
from . import metaflow_git as metaflow_git
|
40
40
|
from . import tuple_util as tuple_util
|
41
|
-
from . import
|
41
|
+
from . import cards as cards
|
42
42
|
from . import runner as runner
|
43
43
|
from . import plugins as plugins
|
44
44
|
from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
|
45
45
|
from . import includefile as includefile
|
46
46
|
from .includefile import IncludeFile as IncludeFile
|
47
|
-
from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
|
48
|
-
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
49
47
|
from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
|
48
|
+
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
49
|
+
from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
|
50
50
|
from . import client as client
|
51
51
|
from .client.core import namespace as namespace
|
52
52
|
from .client.core import get_namespace as get_namespace
|
@@ -154,423 +154,423 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
154
154
|
...
|
155
155
|
|
156
156
|
@typing.overload
|
157
|
-
def
|
157
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
158
158
|
"""
|
159
|
-
Specifies
|
159
|
+
Specifies a timeout for your step.
|
160
160
|
|
161
|
-
|
162
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
161
|
+
This decorator is useful if this step may hang indefinitely.
|
163
162
|
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
python myflow.py run --with kubernetes
|
171
|
-
```
|
172
|
-
which executes the flow on the desired system using the
|
173
|
-
requirements specified in `@resources`.
|
163
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
164
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
165
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
166
|
+
|
167
|
+
Note that all the values specified in parameters are added together so if you specify
|
168
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
174
169
|
|
175
170
|
|
176
171
|
Parameters
|
177
172
|
----------
|
178
|
-
|
179
|
-
Number of
|
180
|
-
|
181
|
-
Number of
|
182
|
-
|
183
|
-
|
184
|
-
memory : int, default 4096
|
185
|
-
Memory size (in MB) required for this step.
|
186
|
-
shared_memory : int, optional, default None
|
187
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
188
|
-
This parameter maps to the `--shm-size` option in Docker.
|
173
|
+
seconds : int, default 0
|
174
|
+
Number of seconds to wait prior to timing out.
|
175
|
+
minutes : int, default 0
|
176
|
+
Number of minutes to wait prior to timing out.
|
177
|
+
hours : int, default 0
|
178
|
+
Number of hours to wait prior to timing out.
|
189
179
|
"""
|
190
180
|
...
|
191
181
|
|
192
182
|
@typing.overload
|
193
|
-
def
|
183
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
194
184
|
...
|
195
185
|
|
196
186
|
@typing.overload
|
197
|
-
def
|
187
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
198
188
|
...
|
199
189
|
|
200
|
-
def
|
190
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
201
191
|
"""
|
202
|
-
Specifies
|
192
|
+
Specifies a timeout for your step.
|
203
193
|
|
204
|
-
|
205
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
194
|
+
This decorator is useful if this step may hang indefinitely.
|
206
195
|
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
python myflow.py run --with kubernetes
|
214
|
-
```
|
215
|
-
which executes the flow on the desired system using the
|
216
|
-
requirements specified in `@resources`.
|
196
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
197
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
198
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
199
|
+
|
200
|
+
Note that all the values specified in parameters are added together so if you specify
|
201
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
217
202
|
|
218
203
|
|
219
204
|
Parameters
|
220
205
|
----------
|
221
|
-
|
222
|
-
Number of
|
223
|
-
|
224
|
-
Number of
|
225
|
-
|
226
|
-
|
227
|
-
memory : int, default 4096
|
228
|
-
Memory size (in MB) required for this step.
|
229
|
-
shared_memory : int, optional, default None
|
230
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
231
|
-
This parameter maps to the `--shm-size` option in Docker.
|
206
|
+
seconds : int, default 0
|
207
|
+
Number of seconds to wait prior to timing out.
|
208
|
+
minutes : int, default 0
|
209
|
+
Number of minutes to wait prior to timing out.
|
210
|
+
hours : int, default 0
|
211
|
+
Number of hours to wait prior to timing out.
|
232
212
|
"""
|
233
213
|
...
|
234
214
|
|
235
215
|
@typing.overload
|
236
|
-
def
|
216
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
237
217
|
"""
|
238
|
-
Specifies
|
239
|
-
the execution of a step.
|
218
|
+
Specifies environment variables to be set prior to the execution of a step.
|
240
219
|
|
241
220
|
|
242
221
|
Parameters
|
243
222
|
----------
|
244
|
-
|
245
|
-
|
223
|
+
vars : Dict[str, str], default {}
|
224
|
+
Dictionary of environment variables to set.
|
246
225
|
"""
|
247
226
|
...
|
248
227
|
|
249
228
|
@typing.overload
|
250
|
-
def
|
229
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
251
230
|
...
|
252
231
|
|
253
232
|
@typing.overload
|
254
|
-
def
|
255
|
-
...
|
256
|
-
|
257
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
258
|
-
"""
|
259
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
260
|
-
the execution of a step.
|
261
|
-
|
262
|
-
|
263
|
-
Parameters
|
264
|
-
----------
|
265
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
266
|
-
List of secret specs, defining how the secrets are to be retrieved
|
267
|
-
"""
|
233
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
268
234
|
...
|
269
235
|
|
270
|
-
def
|
236
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
271
237
|
"""
|
272
|
-
Specifies
|
238
|
+
Specifies environment variables to be set prior to the execution of a step.
|
273
239
|
|
274
240
|
|
275
241
|
Parameters
|
276
242
|
----------
|
277
|
-
|
278
|
-
|
279
|
-
gpu_type : str
|
280
|
-
Type of Nvidia GPU to use.
|
281
|
-
queue_timeout : int
|
282
|
-
Time to keep the job in NVCF's queue.
|
243
|
+
vars : Dict[str, str], default {}
|
244
|
+
Dictionary of environment variables to set.
|
283
245
|
"""
|
284
246
|
...
|
285
247
|
|
286
248
|
@typing.overload
|
287
|
-
def
|
249
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
288
250
|
"""
|
289
|
-
Specifies
|
251
|
+
Specifies the PyPI packages for the step.
|
290
252
|
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
253
|
+
Information in this decorator will augment any
|
254
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
255
|
+
you can use `@pypi_base` to set packages required by all
|
256
|
+
steps and use `@pypi` to specify step-specific overrides.
|
295
257
|
|
296
258
|
|
297
259
|
Parameters
|
298
260
|
----------
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
261
|
+
packages : Dict[str, str], default: {}
|
262
|
+
Packages to use for this step. The key is the name of the package
|
263
|
+
and the value is the version to use.
|
264
|
+
python : str, optional, default: None
|
265
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
266
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
305
267
|
"""
|
306
268
|
...
|
307
269
|
|
308
270
|
@typing.overload
|
309
|
-
def
|
271
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
310
272
|
...
|
311
273
|
|
312
274
|
@typing.overload
|
313
|
-
def
|
275
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
314
276
|
...
|
315
277
|
|
316
|
-
def
|
278
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
317
279
|
"""
|
318
|
-
Specifies
|
280
|
+
Specifies the PyPI packages for the step.
|
319
281
|
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
282
|
+
Information in this decorator will augment any
|
283
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
284
|
+
you can use `@pypi_base` to set packages required by all
|
285
|
+
steps and use `@pypi` to specify step-specific overrides.
|
324
286
|
|
325
287
|
|
326
288
|
Parameters
|
327
289
|
----------
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
290
|
+
packages : Dict[str, str], default: {}
|
291
|
+
Packages to use for this step. The key is the name of the package
|
292
|
+
and the value is the version to use.
|
293
|
+
python : str, optional, default: None
|
294
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
295
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
334
296
|
"""
|
335
297
|
...
|
336
298
|
|
337
|
-
def
|
299
|
+
def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
338
300
|
"""
|
339
|
-
|
340
|
-
|
341
|
-
User code call
|
342
|
-
-----------
|
343
|
-
@nim(
|
344
|
-
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
345
|
-
backend='managed'
|
346
|
-
)
|
347
|
-
|
348
|
-
Valid backend options
|
349
|
-
---------------------
|
350
|
-
- 'managed': Outerbounds selects a compute provider based on the model.
|
351
|
-
|
352
|
-
Valid model options
|
353
|
-
----------------
|
354
|
-
- 'meta/llama3-8b-instruct': 8B parameter model
|
355
|
-
- 'meta/llama3-70b-instruct': 70B parameter model
|
356
|
-
- any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
301
|
+
Specifies that this step is used to deploy an instance of the app.
|
302
|
+
Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
|
357
303
|
|
358
304
|
|
359
305
|
Parameters
|
360
306
|
----------
|
361
|
-
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
queue_timeout : int
|
366
|
-
Time to keep the job in NVCF's queue.
|
307
|
+
app_port : int
|
308
|
+
Number of GPUs to use.
|
309
|
+
app_name : str
|
310
|
+
Name of the app to deploy.
|
367
311
|
"""
|
368
312
|
...
|
369
313
|
|
370
314
|
@typing.overload
|
371
|
-
def
|
315
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
372
316
|
"""
|
373
|
-
|
317
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
374
318
|
|
375
|
-
|
376
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
377
|
-
you can use `@conda_base` to set packages required by all
|
378
|
-
steps and use `@conda` to specify step-specific overrides.
|
319
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
379
320
|
|
380
321
|
|
381
322
|
Parameters
|
382
323
|
----------
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
disabled : bool, default False
|
392
|
-
If set to True, disables @conda.
|
324
|
+
type : str, default 'default'
|
325
|
+
Card type.
|
326
|
+
id : str, optional, default None
|
327
|
+
If multiple cards are present, use this id to identify this card.
|
328
|
+
options : Dict[str, Any], default {}
|
329
|
+
Options passed to the card. The contents depend on the card type.
|
330
|
+
timeout : int, default 45
|
331
|
+
Interrupt reporting if it takes more than this many seconds.
|
393
332
|
"""
|
394
333
|
...
|
395
334
|
|
396
335
|
@typing.overload
|
397
|
-
def
|
336
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
398
337
|
...
|
399
338
|
|
400
339
|
@typing.overload
|
401
|
-
def
|
340
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
402
341
|
...
|
403
342
|
|
404
|
-
def
|
343
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
405
344
|
"""
|
406
|
-
|
345
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
407
346
|
|
408
|
-
|
409
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
410
|
-
you can use `@conda_base` to set packages required by all
|
411
|
-
steps and use `@conda` to specify step-specific overrides.
|
347
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
412
348
|
|
413
349
|
|
414
350
|
Parameters
|
415
351
|
----------
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
disabled : bool, default False
|
425
|
-
If set to True, disables @conda.
|
352
|
+
type : str, default 'default'
|
353
|
+
Card type.
|
354
|
+
id : str, optional, default None
|
355
|
+
If multiple cards are present, use this id to identify this card.
|
356
|
+
options : Dict[str, Any], default {}
|
357
|
+
Options passed to the card. The contents depend on the card type.
|
358
|
+
timeout : int, default 45
|
359
|
+
Interrupt reporting if it takes more than this many seconds.
|
426
360
|
"""
|
427
361
|
...
|
428
362
|
|
429
363
|
@typing.overload
|
430
|
-
def
|
364
|
+
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
431
365
|
"""
|
432
|
-
|
366
|
+
Enables loading / saving of models within a step.
|
433
367
|
|
434
|
-
Information in this decorator will augment any
|
435
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
436
|
-
you can use `@pypi_base` to set packages required by all
|
437
|
-
steps and use `@pypi` to specify step-specific overrides.
|
438
368
|
|
439
369
|
|
440
370
|
Parameters
|
441
371
|
----------
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
372
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
373
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
374
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
375
|
+
- `current.checkpoint`
|
376
|
+
- `current.model`
|
377
|
+
- `current.huggingface_hub`
|
378
|
+
|
379
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
380
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
381
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
382
|
+
|
383
|
+
temp_dir_root : str, default: None
|
384
|
+
The root directory under which `current.model.loaded` will store loaded models
|
448
385
|
"""
|
449
386
|
...
|
450
387
|
|
451
388
|
@typing.overload
|
452
|
-
def
|
389
|
+
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
453
390
|
...
|
454
391
|
|
455
392
|
@typing.overload
|
456
|
-
def
|
393
|
+
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
457
394
|
...
|
458
395
|
|
459
|
-
def
|
396
|
+
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
460
397
|
"""
|
461
|
-
|
398
|
+
Enables loading / saving of models within a step.
|
462
399
|
|
463
|
-
Information in this decorator will augment any
|
464
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
465
|
-
you can use `@pypi_base` to set packages required by all
|
466
|
-
steps and use `@pypi` to specify step-specific overrides.
|
467
400
|
|
468
401
|
|
469
402
|
Parameters
|
470
403
|
----------
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
|
475
|
-
|
476
|
-
|
477
|
-
"""
|
478
|
-
...
|
479
|
-
|
480
|
-
@typing.overload
|
481
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
482
|
-
"""
|
483
|
-
Specifies environment variables to be set prior to the execution of a step.
|
404
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
405
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
406
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
407
|
+
- `current.checkpoint`
|
408
|
+
- `current.model`
|
409
|
+
- `current.huggingface_hub`
|
484
410
|
|
411
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
412
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
413
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
485
414
|
|
486
|
-
|
487
|
-
|
488
|
-
vars : Dict[str, str], default {}
|
489
|
-
Dictionary of environment variables to set.
|
415
|
+
temp_dir_root : str, default: None
|
416
|
+
The root directory under which `current.model.loaded` will store loaded models
|
490
417
|
"""
|
491
418
|
...
|
492
419
|
|
493
420
|
@typing.overload
|
494
|
-
def
|
421
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
422
|
+
"""
|
423
|
+
Internal decorator to support Fast bakery
|
424
|
+
"""
|
495
425
|
...
|
496
426
|
|
497
427
|
@typing.overload
|
498
|
-
def
|
428
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
499
429
|
...
|
500
430
|
|
501
|
-
def
|
431
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
502
432
|
"""
|
503
|
-
|
504
|
-
|
505
|
-
|
506
|
-
Parameters
|
507
|
-
----------
|
508
|
-
vars : Dict[str, str], default {}
|
509
|
-
Dictionary of environment variables to set.
|
433
|
+
Internal decorator to support Fast bakery
|
510
434
|
"""
|
511
435
|
...
|
512
436
|
|
513
|
-
def
|
437
|
+
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
514
438
|
"""
|
515
|
-
Specifies that this step
|
516
|
-
Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
|
439
|
+
Specifies that this step should execute on Kubernetes.
|
517
440
|
|
518
441
|
|
519
442
|
Parameters
|
520
443
|
----------
|
521
|
-
|
522
|
-
Number of
|
523
|
-
|
524
|
-
|
444
|
+
cpu : int, default 1
|
445
|
+
Number of CPUs required for this step. If `@resources` is
|
446
|
+
also present, the maximum value from all decorators is used.
|
447
|
+
memory : int, default 4096
|
448
|
+
Memory size (in MB) required for this step. If
|
449
|
+
`@resources` is also present, the maximum value from all decorators is
|
450
|
+
used.
|
451
|
+
disk : int, default 10240
|
452
|
+
Disk size (in MB) required for this step. If
|
453
|
+
`@resources` is also present, the maximum value from all decorators is
|
454
|
+
used.
|
455
|
+
image : str, optional, default None
|
456
|
+
Docker image to use when launching on Kubernetes. If not specified, and
|
457
|
+
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
458
|
+
not, a default Docker image mapping to the current version of Python is used.
|
459
|
+
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
460
|
+
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
461
|
+
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
462
|
+
Kubernetes service account to use when launching pod in Kubernetes.
|
463
|
+
secrets : List[str], optional, default None
|
464
|
+
Kubernetes secrets to use when launching pod in Kubernetes. These
|
465
|
+
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
466
|
+
in Metaflow configuration.
|
467
|
+
node_selector: Union[Dict[str,str], str], optional, default None
|
468
|
+
Kubernetes node selector(s) to apply to the pod running the task.
|
469
|
+
Can be passed in as a comma separated string of values e.g.
|
470
|
+
'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
|
471
|
+
{'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
|
472
|
+
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
473
|
+
Kubernetes namespace to use when launching pod in Kubernetes.
|
474
|
+
gpu : int, optional, default None
|
475
|
+
Number of GPUs required for this step. A value of zero implies that
|
476
|
+
the scheduled node should not have GPUs.
|
477
|
+
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
478
|
+
The vendor of the GPUs to be used for this step.
|
479
|
+
tolerations : List[str], default []
|
480
|
+
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
481
|
+
Kubernetes tolerations to use when launching pod in Kubernetes.
|
482
|
+
labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
|
483
|
+
Kubernetes labels to use when launching pod in Kubernetes.
|
484
|
+
annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
|
485
|
+
Kubernetes annotations to use when launching pod in Kubernetes.
|
486
|
+
use_tmpfs : bool, default False
|
487
|
+
This enables an explicit tmpfs mount for this step.
|
488
|
+
tmpfs_tempdir : bool, default True
|
489
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
490
|
+
tmpfs_size : int, optional, default: None
|
491
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
492
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
493
|
+
memory allocated for this step.
|
494
|
+
tmpfs_path : str, optional, default /metaflow_temp
|
495
|
+
Path to tmpfs mount for this step.
|
496
|
+
persistent_volume_claims : Dict[str, str], optional, default None
|
497
|
+
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
498
|
+
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
499
|
+
shared_memory: int, optional
|
500
|
+
Shared memory size (in MiB) required for this step
|
501
|
+
port: int, optional
|
502
|
+
Port number to specify in the Kubernetes job object
|
503
|
+
compute_pool : str, optional, default None
|
504
|
+
Compute pool to be used for for this step.
|
505
|
+
If not specified, any accessible compute pool within the perimeter is used.
|
506
|
+
hostname_resolution_timeout: int, default 10 * 60
|
507
|
+
Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
|
508
|
+
Only applicable when @parallel is used.
|
509
|
+
qos: str, default: Burstable
|
510
|
+
Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
|
511
|
+
|
512
|
+
security_context: Dict[str, Any], optional, default None
|
513
|
+
Container security context. Applies to the task container. Allows the following keys:
|
514
|
+
- privileged: bool, optional, default None
|
515
|
+
- allow_privilege_escalation: bool, optional, default None
|
516
|
+
- run_as_user: int, optional, default None
|
517
|
+
- run_as_group: int, optional, default None
|
518
|
+
- run_as_non_root: bool, optional, default None
|
525
519
|
"""
|
526
520
|
...
|
527
521
|
|
528
522
|
@typing.overload
|
529
|
-
def
|
523
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
530
524
|
"""
|
531
|
-
|
525
|
+
Specifies the number of times the task corresponding
|
526
|
+
to a step needs to be retried.
|
532
527
|
|
533
|
-
|
528
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
529
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
530
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
531
|
+
|
532
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
533
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
534
|
+
ensuring that the flow execution can continue.
|
534
535
|
|
535
536
|
|
536
537
|
Parameters
|
537
538
|
----------
|
538
|
-
|
539
|
-
|
540
|
-
|
541
|
-
|
542
|
-
options : Dict[str, Any], default {}
|
543
|
-
Options passed to the card. The contents depend on the card type.
|
544
|
-
timeout : int, default 45
|
545
|
-
Interrupt reporting if it takes more than this many seconds.
|
539
|
+
times : int, default 3
|
540
|
+
Number of times to retry this task.
|
541
|
+
minutes_between_retries : int, default 2
|
542
|
+
Number of minutes between retries.
|
546
543
|
"""
|
547
544
|
...
|
548
545
|
|
549
546
|
@typing.overload
|
550
|
-
def
|
547
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
551
548
|
...
|
552
549
|
|
553
550
|
@typing.overload
|
554
|
-
def
|
551
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
555
552
|
...
|
556
553
|
|
557
|
-
def
|
554
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
558
555
|
"""
|
559
|
-
|
556
|
+
Specifies the number of times the task corresponding
|
557
|
+
to a step needs to be retried.
|
560
558
|
|
561
|
-
|
559
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
560
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
561
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
562
|
+
|
563
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
564
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
565
|
+
ensuring that the flow execution can continue.
|
562
566
|
|
563
567
|
|
564
568
|
Parameters
|
565
569
|
----------
|
566
|
-
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
options : Dict[str, Any], default {}
|
571
|
-
Options passed to the card. The contents depend on the card type.
|
572
|
-
timeout : int, default 45
|
573
|
-
Interrupt reporting if it takes more than this many seconds.
|
570
|
+
times : int, default 3
|
571
|
+
Number of times to retry this task.
|
572
|
+
minutes_between_retries : int, default 2
|
573
|
+
Number of minutes between retries.
|
574
574
|
"""
|
575
575
|
...
|
576
576
|
|
@@ -593,17 +593,50 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
593
593
|
"""
|
594
594
|
...
|
595
595
|
|
596
|
-
def
|
596
|
+
def ollama(*, models: "list[Ollama]", backend: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
597
597
|
"""
|
598
|
-
|
599
|
-
|
598
|
+
This decorator is used to run Ollama APIs as Metaflow task sidecars.
|
600
599
|
|
601
|
-
|
602
|
-
|
603
|
-
|
604
|
-
|
600
|
+
User code call
|
601
|
+
-----------
|
602
|
+
@ollama(
|
603
|
+
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
604
|
+
backend='local'
|
605
|
+
)
|
605
606
|
|
606
|
-
|
607
|
+
Valid backend options
|
608
|
+
---------------------
|
609
|
+
- 'local': Run as a separate process on the local task machine.
|
610
|
+
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
611
|
+
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
612
|
+
|
613
|
+
Valid model options
|
614
|
+
----------------
|
615
|
+
- 'llama3.2'
|
616
|
+
- 'llama3.3'
|
617
|
+
- any model here https://ollama.com/search
|
618
|
+
|
619
|
+
|
620
|
+
Parameters
|
621
|
+
----------
|
622
|
+
models: list[Ollama]
|
623
|
+
List of Ollama containers running models in sidecars.
|
624
|
+
backend: str
|
625
|
+
Determines where and how to run the Ollama process.
|
626
|
+
"""
|
627
|
+
...
|
628
|
+
|
629
|
+
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
630
|
+
"""
|
631
|
+
Decorator that helps cache, version and store models/datasets from huggingface hub.
|
632
|
+
|
633
|
+
|
634
|
+
Parameters
|
635
|
+
----------
|
636
|
+
temp_dir_root : str, optional
|
637
|
+
The root directory that will hold the temporary directory where objects will be downloaded.
|
638
|
+
|
639
|
+
load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
|
607
640
|
The list of repos (models/datasets) to load.
|
608
641
|
|
609
642
|
Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
|
@@ -619,249 +652,289 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.
|
|
619
652
|
...
|
620
653
|
|
621
654
|
@typing.overload
|
622
|
-
def
|
655
|
+
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
623
656
|
"""
|
624
|
-
Specifies the
|
625
|
-
to a step needs to be retried.
|
657
|
+
Specifies the resources needed when executing this step.
|
626
658
|
|
627
|
-
|
628
|
-
|
629
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
659
|
+
Use `@resources` to specify the resource requirements
|
660
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
630
661
|
|
631
|
-
|
632
|
-
|
633
|
-
|
662
|
+
You can choose the compute layer on the command line by executing e.g.
|
663
|
+
```
|
664
|
+
python myflow.py run --with batch
|
665
|
+
```
|
666
|
+
or
|
667
|
+
```
|
668
|
+
python myflow.py run --with kubernetes
|
669
|
+
```
|
670
|
+
which executes the flow on the desired system using the
|
671
|
+
requirements specified in `@resources`.
|
634
672
|
|
635
673
|
|
636
674
|
Parameters
|
637
675
|
----------
|
638
|
-
|
639
|
-
Number of
|
640
|
-
|
641
|
-
Number of
|
676
|
+
cpu : int, default 1
|
677
|
+
Number of CPUs required for this step.
|
678
|
+
gpu : int, optional, default None
|
679
|
+
Number of GPUs required for this step.
|
680
|
+
disk : int, optional, default None
|
681
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
682
|
+
memory : int, default 4096
|
683
|
+
Memory size (in MB) required for this step.
|
684
|
+
shared_memory : int, optional, default None
|
685
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
686
|
+
This parameter maps to the `--shm-size` option in Docker.
|
642
687
|
"""
|
643
688
|
...
|
644
689
|
|
645
690
|
@typing.overload
|
646
|
-
def
|
691
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
647
692
|
...
|
648
693
|
|
649
694
|
@typing.overload
|
650
|
-
def
|
695
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
651
696
|
...
|
652
697
|
|
653
|
-
def
|
698
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
654
699
|
"""
|
655
|
-
Specifies the
|
656
|
-
to a step needs to be retried.
|
700
|
+
Specifies the resources needed when executing this step.
|
657
701
|
|
658
|
-
|
659
|
-
|
660
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
702
|
+
Use `@resources` to specify the resource requirements
|
703
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
661
704
|
|
662
|
-
|
663
|
-
|
664
|
-
|
705
|
+
You can choose the compute layer on the command line by executing e.g.
|
706
|
+
```
|
707
|
+
python myflow.py run --with batch
|
708
|
+
```
|
709
|
+
or
|
710
|
+
```
|
711
|
+
python myflow.py run --with kubernetes
|
712
|
+
```
|
713
|
+
which executes the flow on the desired system using the
|
714
|
+
requirements specified in `@resources`.
|
665
715
|
|
666
716
|
|
667
717
|
Parameters
|
668
718
|
----------
|
669
|
-
|
670
|
-
Number of
|
671
|
-
|
672
|
-
Number of
|
719
|
+
cpu : int, default 1
|
720
|
+
Number of CPUs required for this step.
|
721
|
+
gpu : int, optional, default None
|
722
|
+
Number of GPUs required for this step.
|
723
|
+
disk : int, optional, default None
|
724
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
725
|
+
memory : int, default 4096
|
726
|
+
Memory size (in MB) required for this step.
|
727
|
+
shared_memory : int, optional, default None
|
728
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
729
|
+
This parameter maps to the `--shm-size` option in Docker.
|
673
730
|
"""
|
674
731
|
...
|
675
732
|
|
676
733
|
@typing.overload
|
677
|
-
def
|
734
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
678
735
|
"""
|
679
|
-
|
736
|
+
Specifies the Conda environment for the step.
|
680
737
|
|
738
|
+
Information in this decorator will augment any
|
739
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
740
|
+
you can use `@conda_base` to set packages required by all
|
741
|
+
steps and use `@conda` to specify step-specific overrides.
|
681
742
|
|
682
743
|
|
683
744
|
Parameters
|
684
745
|
----------
|
685
|
-
|
686
|
-
|
687
|
-
|
688
|
-
|
689
|
-
|
690
|
-
|
691
|
-
|
692
|
-
|
693
|
-
|
694
|
-
If
|
695
|
-
|
696
|
-
temp_dir_root : str, default: None
|
697
|
-
The root directory under which `current.model.loaded` will store loaded models
|
746
|
+
packages : Dict[str, str], default {}
|
747
|
+
Packages to use for this step. The key is the name of the package
|
748
|
+
and the value is the version to use.
|
749
|
+
libraries : Dict[str, str], default {}
|
750
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
751
|
+
python : str, optional, default None
|
752
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
753
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
754
|
+
disabled : bool, default False
|
755
|
+
If set to True, disables @conda.
|
698
756
|
"""
|
699
757
|
...
|
700
758
|
|
701
759
|
@typing.overload
|
702
|
-
def
|
760
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
703
761
|
...
|
704
762
|
|
705
763
|
@typing.overload
|
706
|
-
def
|
764
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
707
765
|
...
|
708
766
|
|
709
|
-
def
|
767
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
710
768
|
"""
|
711
|
-
|
769
|
+
Specifies the Conda environment for the step.
|
712
770
|
|
771
|
+
Information in this decorator will augment any
|
772
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
773
|
+
you can use `@conda_base` to set packages required by all
|
774
|
+
steps and use `@conda` to specify step-specific overrides.
|
713
775
|
|
714
776
|
|
715
777
|
Parameters
|
716
778
|
----------
|
717
|
-
|
718
|
-
|
719
|
-
|
720
|
-
|
721
|
-
|
722
|
-
|
723
|
-
|
724
|
-
|
725
|
-
|
726
|
-
If
|
727
|
-
|
728
|
-
temp_dir_root : str, default: None
|
729
|
-
The root directory under which `current.model.loaded` will store loaded models
|
779
|
+
packages : Dict[str, str], default {}
|
780
|
+
Packages to use for this step. The key is the name of the package
|
781
|
+
and the value is the version to use.
|
782
|
+
libraries : Dict[str, str], default {}
|
783
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
784
|
+
python : str, optional, default None
|
785
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
786
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
787
|
+
disabled : bool, default False
|
788
|
+
If set to True, disables @conda.
|
730
789
|
"""
|
731
790
|
...
|
732
791
|
|
733
792
|
@typing.overload
|
734
|
-
def
|
793
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
735
794
|
"""
|
736
|
-
|
795
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
796
|
+
the execution of a step.
|
797
|
+
|
798
|
+
|
799
|
+
Parameters
|
800
|
+
----------
|
801
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
802
|
+
List of secret specs, defining how the secrets are to be retrieved
|
737
803
|
"""
|
738
804
|
...
|
739
805
|
|
740
806
|
@typing.overload
|
741
|
-
def
|
807
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
742
808
|
...
|
743
809
|
|
744
|
-
|
745
|
-
|
746
|
-
Internal decorator to support Fast bakery
|
747
|
-
"""
|
810
|
+
@typing.overload
|
811
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
748
812
|
...
|
749
813
|
|
750
|
-
def
|
814
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
751
815
|
"""
|
752
|
-
|
753
|
-
|
754
|
-
User code call
|
755
|
-
-----------
|
756
|
-
@ollama(
|
757
|
-
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
758
|
-
backend='local'
|
759
|
-
)
|
816
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
817
|
+
the execution of a step.
|
760
818
|
|
761
|
-
Valid backend options
|
762
|
-
---------------------
|
763
|
-
- 'local': Run as a separate process on the local task machine.
|
764
|
-
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
765
|
-
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
766
819
|
|
767
|
-
|
768
|
-
|
769
|
-
|
770
|
-
|
771
|
-
|
820
|
+
Parameters
|
821
|
+
----------
|
822
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
823
|
+
List of secret specs, defining how the secrets are to be retrieved
|
824
|
+
"""
|
825
|
+
...
|
826
|
+
|
827
|
+
def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
828
|
+
"""
|
829
|
+
Specifies that this step should execute on DGX cloud.
|
772
830
|
|
773
831
|
|
774
832
|
Parameters
|
775
833
|
----------
|
776
|
-
|
777
|
-
|
778
|
-
|
779
|
-
|
834
|
+
gpu : int
|
835
|
+
Number of GPUs to use.
|
836
|
+
gpu_type : str
|
837
|
+
Type of Nvidia GPU to use.
|
838
|
+
queue_timeout : int
|
839
|
+
Time to keep the job in NVCF's queue.
|
780
840
|
"""
|
781
841
|
...
|
782
842
|
|
783
|
-
def
|
843
|
+
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
784
844
|
"""
|
785
|
-
Specifies that this step should execute on
|
845
|
+
Specifies that this step should execute on DGX cloud.
|
786
846
|
|
787
847
|
|
788
848
|
Parameters
|
789
849
|
----------
|
790
|
-
|
791
|
-
Number of
|
792
|
-
|
793
|
-
|
794
|
-
|
795
|
-
|
796
|
-
|
797
|
-
|
798
|
-
|
799
|
-
|
800
|
-
|
801
|
-
image : str, optional, default None
|
802
|
-
Docker image to use when launching on Kubernetes. If not specified, and
|
803
|
-
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
804
|
-
not, a default Docker image mapping to the current version of Python is used.
|
805
|
-
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
806
|
-
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
807
|
-
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
808
|
-
Kubernetes service account to use when launching pod in Kubernetes.
|
809
|
-
secrets : List[str], optional, default None
|
810
|
-
Kubernetes secrets to use when launching pod in Kubernetes. These
|
811
|
-
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
812
|
-
in Metaflow configuration.
|
813
|
-
node_selector: Union[Dict[str,str], str], optional, default None
|
814
|
-
Kubernetes node selector(s) to apply to the pod running the task.
|
815
|
-
Can be passed in as a comma separated string of values e.g.
|
816
|
-
'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
|
817
|
-
{'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
|
818
|
-
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
819
|
-
Kubernetes namespace to use when launching pod in Kubernetes.
|
820
|
-
gpu : int, optional, default None
|
821
|
-
Number of GPUs required for this step. A value of zero implies that
|
822
|
-
the scheduled node should not have GPUs.
|
823
|
-
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
824
|
-
The vendor of the GPUs to be used for this step.
|
825
|
-
tolerations : List[str], default []
|
826
|
-
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
827
|
-
Kubernetes tolerations to use when launching pod in Kubernetes.
|
828
|
-
labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
|
829
|
-
Kubernetes labels to use when launching pod in Kubernetes.
|
830
|
-
annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
|
831
|
-
Kubernetes annotations to use when launching pod in Kubernetes.
|
832
|
-
use_tmpfs : bool, default False
|
833
|
-
This enables an explicit tmpfs mount for this step.
|
834
|
-
tmpfs_tempdir : bool, default True
|
835
|
-
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
836
|
-
tmpfs_size : int, optional, default: None
|
837
|
-
The value for the size (in MiB) of the tmpfs mount for this step.
|
838
|
-
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
839
|
-
memory allocated for this step.
|
840
|
-
tmpfs_path : str, optional, default /metaflow_temp
|
841
|
-
Path to tmpfs mount for this step.
|
842
|
-
persistent_volume_claims : Dict[str, str], optional, default None
|
843
|
-
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
844
|
-
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
845
|
-
shared_memory: int, optional
|
846
|
-
Shared memory size (in MiB) required for this step
|
847
|
-
port: int, optional
|
848
|
-
Port number to specify in the Kubernetes job object
|
849
|
-
compute_pool : str, optional, default None
|
850
|
-
Compute pool to be used for for this step.
|
851
|
-
If not specified, any accessible compute pool within the perimeter is used.
|
852
|
-
hostname_resolution_timeout: int, default 10 * 60
|
853
|
-
Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
|
854
|
-
Only applicable when @parallel is used.
|
855
|
-
qos: str, default: Burstable
|
856
|
-
Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
|
850
|
+
gpu : int
|
851
|
+
Number of GPUs to use.
|
852
|
+
gpu_type : str
|
853
|
+
Type of Nvidia GPU to use.
|
854
|
+
"""
|
855
|
+
...
|
856
|
+
|
857
|
+
@typing.overload
|
858
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
859
|
+
"""
|
860
|
+
Specifies that the step will success under all circumstances.
|
857
861
|
|
858
|
-
|
859
|
-
|
860
|
-
|
861
|
-
|
862
|
-
|
863
|
-
|
864
|
-
|
862
|
+
The decorator will create an optional artifact, specified by `var`, which
|
863
|
+
contains the exception raised. You can use it to detect the presence
|
864
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
865
|
+
are missing.
|
866
|
+
|
867
|
+
|
868
|
+
Parameters
|
869
|
+
----------
|
870
|
+
var : str, optional, default None
|
871
|
+
Name of the artifact in which to store the caught exception.
|
872
|
+
If not specified, the exception is not stored.
|
873
|
+
print_exception : bool, default True
|
874
|
+
Determines whether or not the exception is printed to
|
875
|
+
stdout when caught.
|
876
|
+
"""
|
877
|
+
...
|
878
|
+
|
879
|
+
@typing.overload
|
880
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
881
|
+
...
|
882
|
+
|
883
|
+
@typing.overload
|
884
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
885
|
+
...
|
886
|
+
|
887
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
888
|
+
"""
|
889
|
+
Specifies that the step will success under all circumstances.
|
890
|
+
|
891
|
+
The decorator will create an optional artifact, specified by `var`, which
|
892
|
+
contains the exception raised. You can use it to detect the presence
|
893
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
894
|
+
are missing.
|
895
|
+
|
896
|
+
|
897
|
+
Parameters
|
898
|
+
----------
|
899
|
+
var : str, optional, default None
|
900
|
+
Name of the artifact in which to store the caught exception.
|
901
|
+
If not specified, the exception is not stored.
|
902
|
+
print_exception : bool, default True
|
903
|
+
Determines whether or not the exception is printed to
|
904
|
+
stdout when caught.
|
905
|
+
"""
|
906
|
+
...
|
907
|
+
|
908
|
+
def nim(*, models: "list[NIM]", backend: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
909
|
+
"""
|
910
|
+
This decorator is used to run NIM containers in Metaflow tasks as sidecars.
|
911
|
+
|
912
|
+
User code call
|
913
|
+
-----------
|
914
|
+
@nim(
|
915
|
+
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
916
|
+
backend='managed'
|
917
|
+
)
|
918
|
+
|
919
|
+
Valid backend options
|
920
|
+
---------------------
|
921
|
+
- 'managed': Outerbounds selects a compute provider based on the model.
|
922
|
+
|
923
|
+
Valid model options
|
924
|
+
----------------
|
925
|
+
- 'meta/llama3-8b-instruct': 8B parameter model
|
926
|
+
- 'meta/llama3-70b-instruct': 70B parameter model
|
927
|
+
- any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
928
|
+
|
929
|
+
|
930
|
+
Parameters
|
931
|
+
----------
|
932
|
+
models: list[NIM]
|
933
|
+
List of NIM containers running models in sidecars.
|
934
|
+
backend: str
|
935
|
+
Compute provider to run the NIM container.
|
936
|
+
queue_timeout : int
|
937
|
+
Time to keep the job in NVCF's queue.
|
865
938
|
"""
|
866
939
|
...
|
867
940
|
|
@@ -922,162 +995,96 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
|
|
922
995
|
"""
|
923
996
|
...
|
924
997
|
|
925
|
-
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
926
|
-
"""
|
927
|
-
Specifies that this step should execute on DGX cloud.
|
928
|
-
|
929
|
-
|
930
|
-
Parameters
|
931
|
-
----------
|
932
|
-
gpu : int
|
933
|
-
Number of GPUs to use.
|
934
|
-
gpu_type : str
|
935
|
-
Type of Nvidia GPU to use.
|
936
|
-
"""
|
937
|
-
...
|
938
|
-
|
939
998
|
@typing.overload
|
940
|
-
def
|
999
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
941
1000
|
"""
|
942
|
-
Specifies
|
1001
|
+
Specifies the event(s) that this flow depends on.
|
943
1002
|
|
944
|
-
|
1003
|
+
```
|
1004
|
+
@trigger(event='foo')
|
1005
|
+
```
|
1006
|
+
or
|
1007
|
+
```
|
1008
|
+
@trigger(events=['foo', 'bar'])
|
1009
|
+
```
|
945
1010
|
|
946
|
-
|
947
|
-
|
948
|
-
|
1011
|
+
Additionally, you can specify the parameter mappings
|
1012
|
+
to map event payload to Metaflow parameters for the flow.
|
1013
|
+
```
|
1014
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1015
|
+
```
|
1016
|
+
or
|
1017
|
+
```
|
1018
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1019
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1020
|
+
```
|
949
1021
|
|
950
|
-
|
951
|
-
|
1022
|
+
'parameters' can also be a list of strings and tuples like so:
|
1023
|
+
```
|
1024
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1025
|
+
```
|
1026
|
+
This is equivalent to:
|
1027
|
+
```
|
1028
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1029
|
+
```
|
952
1030
|
|
953
1031
|
|
954
1032
|
Parameters
|
955
1033
|
----------
|
956
|
-
|
957
|
-
|
958
|
-
|
959
|
-
|
960
|
-
|
961
|
-
|
1034
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1035
|
+
Event dependency for this flow.
|
1036
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1037
|
+
Events dependency for this flow.
|
1038
|
+
options : Dict[str, Any], default {}
|
1039
|
+
Backend-specific configuration for tuning eventing behavior.
|
962
1040
|
"""
|
963
1041
|
...
|
964
1042
|
|
965
1043
|
@typing.overload
|
966
|
-
def
|
967
|
-
...
|
968
|
-
|
969
|
-
@typing.overload
|
970
|
-
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
971
|
-
...
|
972
|
-
|
973
|
-
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
974
|
-
"""
|
975
|
-
Specifies a timeout for your step.
|
976
|
-
|
977
|
-
This decorator is useful if this step may hang indefinitely.
|
978
|
-
|
979
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
980
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
981
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
982
|
-
|
983
|
-
Note that all the values specified in parameters are added together so if you specify
|
984
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
985
|
-
|
986
|
-
|
987
|
-
Parameters
|
988
|
-
----------
|
989
|
-
seconds : int, default 0
|
990
|
-
Number of seconds to wait prior to timing out.
|
991
|
-
minutes : int, default 0
|
992
|
-
Number of minutes to wait prior to timing out.
|
993
|
-
hours : int, default 0
|
994
|
-
Number of hours to wait prior to timing out.
|
995
|
-
"""
|
996
|
-
...
|
997
|
-
|
998
|
-
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
999
|
-
"""
|
1000
|
-
Specifies what flows belong to the same project.
|
1001
|
-
|
1002
|
-
A project-specific namespace is created for all flows that
|
1003
|
-
use the same `@project(name)`.
|
1004
|
-
|
1005
|
-
|
1006
|
-
Parameters
|
1007
|
-
----------
|
1008
|
-
name : str
|
1009
|
-
Project name. Make sure that the name is unique amongst all
|
1010
|
-
projects that use the same production scheduler. The name may
|
1011
|
-
contain only lowercase alphanumeric characters and underscores.
|
1012
|
-
|
1013
|
-
branch : Optional[str], default None
|
1014
|
-
The branch to use. If not specified, the branch is set to
|
1015
|
-
`user.<username>` unless `production` is set to `True`. This can
|
1016
|
-
also be set on the command line using `--branch` as a top-level option.
|
1017
|
-
It is an error to specify `branch` in the decorator and on the command line.
|
1018
|
-
|
1019
|
-
production : bool, default False
|
1020
|
-
Whether or not the branch is the production branch. This can also be set on the
|
1021
|
-
command line using `--production` as a top-level option. It is an error to specify
|
1022
|
-
`production` in the decorator and on the command line.
|
1023
|
-
The project branch name will be:
|
1024
|
-
- if `branch` is specified:
|
1025
|
-
- if `production` is True: `prod.<branch>`
|
1026
|
-
- if `production` is False: `test.<branch>`
|
1027
|
-
- if `branch` is not specified:
|
1028
|
-
- if `production` is True: `prod`
|
1029
|
-
- if `production` is False: `user.<username>`
|
1030
|
-
"""
|
1044
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1031
1045
|
...
|
1032
1046
|
|
1033
|
-
|
1034
|
-
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1047
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1035
1048
|
"""
|
1036
|
-
Specifies the
|
1037
|
-
|
1038
|
-
Use `@conda_base` to set common libraries required by all
|
1039
|
-
steps and use `@conda` to specify step-specific additions.
|
1040
|
-
|
1049
|
+
Specifies the event(s) that this flow depends on.
|
1041
1050
|
|
1042
|
-
|
1043
|
-
|
1044
|
-
|
1045
|
-
|
1046
|
-
|
1047
|
-
|
1048
|
-
|
1049
|
-
python : str, optional, default None
|
1050
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1051
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1052
|
-
disabled : bool, default False
|
1053
|
-
If set to True, disables Conda.
|
1054
|
-
"""
|
1055
|
-
...
|
1056
|
-
|
1057
|
-
@typing.overload
|
1058
|
-
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1059
|
-
...
|
1060
|
-
|
1061
|
-
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1062
|
-
"""
|
1063
|
-
Specifies the Conda environment for all steps of the flow.
|
1051
|
+
```
|
1052
|
+
@trigger(event='foo')
|
1053
|
+
```
|
1054
|
+
or
|
1055
|
+
```
|
1056
|
+
@trigger(events=['foo', 'bar'])
|
1057
|
+
```
|
1064
1058
|
|
1065
|
-
|
1066
|
-
|
1059
|
+
Additionally, you can specify the parameter mappings
|
1060
|
+
to map event payload to Metaflow parameters for the flow.
|
1061
|
+
```
|
1062
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1063
|
+
```
|
1064
|
+
or
|
1065
|
+
```
|
1066
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1067
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1068
|
+
```
|
1069
|
+
|
1070
|
+
'parameters' can also be a list of strings and tuples like so:
|
1071
|
+
```
|
1072
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1073
|
+
```
|
1074
|
+
This is equivalent to:
|
1075
|
+
```
|
1076
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1077
|
+
```
|
1067
1078
|
|
1068
1079
|
|
1069
1080
|
Parameters
|
1070
1081
|
----------
|
1071
|
-
|
1072
|
-
|
1073
|
-
|
1074
|
-
|
1075
|
-
|
1076
|
-
|
1077
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1078
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1079
|
-
disabled : bool, default False
|
1080
|
-
If set to True, disables Conda.
|
1082
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1083
|
+
Event dependency for this flow.
|
1084
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1085
|
+
Events dependency for this flow.
|
1086
|
+
options : Dict[str, Any], default {}
|
1087
|
+
Backend-specific configuration for tuning eventing behavior.
|
1081
1088
|
"""
|
1082
1089
|
...
|
1083
1090
|
|
@@ -1246,47 +1253,6 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
|
|
1246
1253
|
"""
|
1247
1254
|
...
|
1248
1255
|
|
1249
|
-
@typing.overload
|
1250
|
-
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1251
|
-
"""
|
1252
|
-
Specifies the PyPI packages for all steps of the flow.
|
1253
|
-
|
1254
|
-
Use `@pypi_base` to set common packages required by all
|
1255
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1256
|
-
|
1257
|
-
Parameters
|
1258
|
-
----------
|
1259
|
-
packages : Dict[str, str], default: {}
|
1260
|
-
Packages to use for this flow. The key is the name of the package
|
1261
|
-
and the value is the version to use.
|
1262
|
-
python : str, optional, default: None
|
1263
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1264
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1265
|
-
"""
|
1266
|
-
...
|
1267
|
-
|
1268
|
-
@typing.overload
|
1269
|
-
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1270
|
-
...
|
1271
|
-
|
1272
|
-
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1273
|
-
"""
|
1274
|
-
Specifies the PyPI packages for all steps of the flow.
|
1275
|
-
|
1276
|
-
Use `@pypi_base` to set common packages required by all
|
1277
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1278
|
-
|
1279
|
-
Parameters
|
1280
|
-
----------
|
1281
|
-
packages : Dict[str, str], default: {}
|
1282
|
-
Packages to use for this flow. The key is the name of the package
|
1283
|
-
and the value is the version to use.
|
1284
|
-
python : str, optional, default: None
|
1285
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1286
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1287
|
-
"""
|
1288
|
-
...
|
1289
|
-
|
1290
1256
|
@typing.overload
|
1291
1257
|
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1292
1258
|
"""
|
@@ -1389,95 +1355,53 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
1389
1355
|
...
|
1390
1356
|
|
1391
1357
|
@typing.overload
|
1392
|
-
def
|
1358
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1393
1359
|
"""
|
1394
|
-
Specifies the
|
1395
|
-
|
1396
|
-
```
|
1397
|
-
@trigger(event='foo')
|
1398
|
-
```
|
1399
|
-
or
|
1400
|
-
```
|
1401
|
-
@trigger(events=['foo', 'bar'])
|
1402
|
-
```
|
1403
|
-
|
1404
|
-
Additionally, you can specify the parameter mappings
|
1405
|
-
to map event payload to Metaflow parameters for the flow.
|
1406
|
-
```
|
1407
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1408
|
-
```
|
1409
|
-
or
|
1410
|
-
```
|
1411
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1412
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1413
|
-
```
|
1360
|
+
Specifies the Conda environment for all steps of the flow.
|
1414
1361
|
|
1415
|
-
|
1416
|
-
|
1417
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1418
|
-
```
|
1419
|
-
This is equivalent to:
|
1420
|
-
```
|
1421
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1422
|
-
```
|
1362
|
+
Use `@conda_base` to set common libraries required by all
|
1363
|
+
steps and use `@conda` to specify step-specific additions.
|
1423
1364
|
|
1424
1365
|
|
1425
1366
|
Parameters
|
1426
1367
|
----------
|
1427
|
-
|
1428
|
-
|
1429
|
-
|
1430
|
-
|
1431
|
-
|
1432
|
-
|
1368
|
+
packages : Dict[str, str], default {}
|
1369
|
+
Packages to use for this flow. The key is the name of the package
|
1370
|
+
and the value is the version to use.
|
1371
|
+
libraries : Dict[str, str], default {}
|
1372
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1373
|
+
python : str, optional, default None
|
1374
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1375
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1376
|
+
disabled : bool, default False
|
1377
|
+
If set to True, disables Conda.
|
1433
1378
|
"""
|
1434
1379
|
...
|
1435
1380
|
|
1436
1381
|
@typing.overload
|
1437
|
-
def
|
1382
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1438
1383
|
...
|
1439
1384
|
|
1440
|
-
def
|
1385
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1441
1386
|
"""
|
1442
|
-
Specifies the
|
1443
|
-
|
1444
|
-
```
|
1445
|
-
@trigger(event='foo')
|
1446
|
-
```
|
1447
|
-
or
|
1448
|
-
```
|
1449
|
-
@trigger(events=['foo', 'bar'])
|
1450
|
-
```
|
1451
|
-
|
1452
|
-
Additionally, you can specify the parameter mappings
|
1453
|
-
to map event payload to Metaflow parameters for the flow.
|
1454
|
-
```
|
1455
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1456
|
-
```
|
1457
|
-
or
|
1458
|
-
```
|
1459
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1460
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1461
|
-
```
|
1387
|
+
Specifies the Conda environment for all steps of the flow.
|
1462
1388
|
|
1463
|
-
|
1464
|
-
|
1465
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1466
|
-
```
|
1467
|
-
This is equivalent to:
|
1468
|
-
```
|
1469
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1470
|
-
```
|
1389
|
+
Use `@conda_base` to set common libraries required by all
|
1390
|
+
steps and use `@conda` to specify step-specific additions.
|
1471
1391
|
|
1472
1392
|
|
1473
1393
|
Parameters
|
1474
1394
|
----------
|
1475
|
-
|
1476
|
-
|
1477
|
-
|
1478
|
-
|
1479
|
-
|
1480
|
-
|
1395
|
+
packages : Dict[str, str], default {}
|
1396
|
+
Packages to use for this flow. The key is the name of the package
|
1397
|
+
and the value is the version to use.
|
1398
|
+
libraries : Dict[str, str], default {}
|
1399
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1400
|
+
python : str, optional, default None
|
1401
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1402
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1403
|
+
disabled : bool, default False
|
1404
|
+
If set to True, disables Conda.
|
1481
1405
|
"""
|
1482
1406
|
...
|
1483
1407
|
|
@@ -1524,6 +1448,82 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
|
|
1524
1448
|
"""
|
1525
1449
|
...
|
1526
1450
|
|
1451
|
+
@typing.overload
|
1452
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1453
|
+
"""
|
1454
|
+
Specifies the PyPI packages for all steps of the flow.
|
1455
|
+
|
1456
|
+
Use `@pypi_base` to set common packages required by all
|
1457
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1458
|
+
|
1459
|
+
Parameters
|
1460
|
+
----------
|
1461
|
+
packages : Dict[str, str], default: {}
|
1462
|
+
Packages to use for this flow. The key is the name of the package
|
1463
|
+
and the value is the version to use.
|
1464
|
+
python : str, optional, default: None
|
1465
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1466
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1467
|
+
"""
|
1468
|
+
...
|
1469
|
+
|
1470
|
+
@typing.overload
|
1471
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1472
|
+
...
|
1473
|
+
|
1474
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1475
|
+
"""
|
1476
|
+
Specifies the PyPI packages for all steps of the flow.
|
1477
|
+
|
1478
|
+
Use `@pypi_base` to set common packages required by all
|
1479
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1480
|
+
|
1481
|
+
Parameters
|
1482
|
+
----------
|
1483
|
+
packages : Dict[str, str], default: {}
|
1484
|
+
Packages to use for this flow. The key is the name of the package
|
1485
|
+
and the value is the version to use.
|
1486
|
+
python : str, optional, default: None
|
1487
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1488
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1489
|
+
"""
|
1490
|
+
...
|
1491
|
+
|
1492
|
+
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1493
|
+
"""
|
1494
|
+
Specifies what flows belong to the same project.
|
1495
|
+
|
1496
|
+
A project-specific namespace is created for all flows that
|
1497
|
+
use the same `@project(name)`.
|
1498
|
+
|
1499
|
+
|
1500
|
+
Parameters
|
1501
|
+
----------
|
1502
|
+
name : str
|
1503
|
+
Project name. Make sure that the name is unique amongst all
|
1504
|
+
projects that use the same production scheduler. The name may
|
1505
|
+
contain only lowercase alphanumeric characters and underscores.
|
1506
|
+
|
1507
|
+
branch : Optional[str], default None
|
1508
|
+
The branch to use. If not specified, the branch is set to
|
1509
|
+
`user.<username>` unless `production` is set to `True`. This can
|
1510
|
+
also be set on the command line using `--branch` as a top-level option.
|
1511
|
+
It is an error to specify `branch` in the decorator and on the command line.
|
1512
|
+
|
1513
|
+
production : bool, default False
|
1514
|
+
Whether or not the branch is the production branch. This can also be set on the
|
1515
|
+
command line using `--production` as a top-level option. It is an error to specify
|
1516
|
+
`production` in the decorator and on the command line.
|
1517
|
+
The project branch name will be:
|
1518
|
+
- if `branch` is specified:
|
1519
|
+
- if `production` is True: `prod.<branch>`
|
1520
|
+
- if `production` is False: `test.<branch>`
|
1521
|
+
- if `branch` is not specified:
|
1522
|
+
- if `production` is True: `prod`
|
1523
|
+
- if `production` is False: `user.<username>`
|
1524
|
+
"""
|
1525
|
+
...
|
1526
|
+
|
1527
1527
|
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1528
1528
|
"""
|
1529
1529
|
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|