ob-metaflow-stubs 6.0.3.175rc1__py2.py3-none-any.whl → 6.0.3.176rc2__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +718 -712
- metaflow-stubs/cards.pyi +1 -1
- metaflow-stubs/cli.pyi +1 -1
- metaflow-stubs/cli_components/__init__.pyi +1 -1
- metaflow-stubs/cli_components/utils.pyi +1 -1
- metaflow-stubs/client/__init__.pyi +1 -1
- metaflow-stubs/client/core.pyi +6 -6
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/events.pyi +1 -1
- metaflow-stubs/exception.pyi +1 -1
- metaflow-stubs/flowspec.pyi +5 -5
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +3 -3
- metaflow-stubs/info_file.pyi +1 -1
- metaflow-stubs/metadata_provider/__init__.pyi +1 -1
- metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
- metaflow-stubs/metadata_provider/metadata.pyi +2 -2
- metaflow-stubs/metadata_provider/util.pyi +1 -1
- metaflow-stubs/metaflow_config.pyi +1 -1
- metaflow-stubs/metaflow_current.pyi +90 -90
- metaflow-stubs/metaflow_git.pyi +1 -1
- metaflow-stubs/mf_extensions/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +60 -0
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +60 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +73 -0
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +1 -1
- metaflow-stubs/multicore_utils.pyi +1 -1
- metaflow-stubs/ob_internal.pyi +1 -1
- metaflow-stubs/parameters.pyi +3 -3
- metaflow-stubs/plugins/__init__.pyi +11 -11
- metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
- metaflow-stubs/plugins/airflow/exception.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
- metaflow-stubs/plugins/argo/__init__.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
- metaflow-stubs/plugins/aws/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
- metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
- metaflow-stubs/plugins/azure/__init__.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
- metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
- metaflow-stubs/plugins/cards/__init__.pyi +5 -5
- metaflow-stubs/plugins/cards/card_client.pyi +1 -1
- metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
- metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
- metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
- metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
- metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
- metaflow-stubs/plugins/cards/exception.pyi +1 -1
- metaflow-stubs/plugins/catch_decorator.pyi +1 -1
- metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
- metaflow-stubs/plugins/datatools/local.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
- metaflow-stubs/plugins/debug_logger.pyi +1 -1
- metaflow-stubs/plugins/debug_monitor.pyi +1 -1
- metaflow-stubs/plugins/environment_decorator.pyi +1 -1
- metaflow-stubs/plugins/events_decorator.pyi +1 -1
- metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
- metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
- metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
- metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
- metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
- metaflow-stubs/plugins/ollama/__init__.pyi +12 -3
- metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
- metaflow-stubs/plugins/perimeters.pyi +1 -1
- metaflow-stubs/plugins/project_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/__init__.pyi +1 -1
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
- metaflow-stubs/plugins/pypi/utils.pyi +1 -1
- metaflow-stubs/plugins/resources_decorator.pyi +1 -1
- metaflow-stubs/plugins/retry_decorator.pyi +1 -1
- metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
- metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
- metaflow-stubs/plugins/storage_executor.pyi +1 -1
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +1 -1
- metaflow-stubs/plugins/timeout_decorator.pyi +1 -1
- metaflow-stubs/plugins/torchtune/__init__.pyi +1 -1
- metaflow-stubs/plugins/uv/__init__.pyi +1 -1
- metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +1 -1
- metaflow-stubs/pylint_wrapper.pyi +1 -1
- metaflow-stubs/runner/__init__.pyi +1 -1
- metaflow-stubs/runner/deployer.pyi +6 -6
- metaflow-stubs/runner/deployer_impl.pyi +2 -2
- metaflow-stubs/runner/metaflow_runner.pyi +3 -3
- metaflow-stubs/runner/nbdeploy.pyi +1 -1
- metaflow-stubs/runner/nbrun.pyi +1 -1
- metaflow-stubs/runner/subprocess_manager.pyi +1 -1
- metaflow-stubs/runner/utils.pyi +2 -2
- metaflow-stubs/system/__init__.pyi +1 -1
- metaflow-stubs/system/system_logger.pyi +1 -1
- metaflow-stubs/system/system_monitor.pyi +1 -1
- metaflow-stubs/tagging_util.pyi +1 -1
- metaflow-stubs/tuple_util.pyi +1 -1
- metaflow-stubs/user_configs/__init__.pyi +1 -1
- metaflow-stubs/user_configs/config_decorators.pyi +3 -3
- metaflow-stubs/user_configs/config_options.pyi +3 -3
- metaflow-stubs/user_configs/config_parameters.pyi +6 -6
- {ob_metaflow_stubs-6.0.3.175rc1.dist-info → ob_metaflow_stubs-6.0.3.176rc2.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.176rc2.dist-info/RECORD +218 -0
- ob_metaflow_stubs-6.0.3.175rc1.dist-info/RECORD +0 -216
- {ob_metaflow_stubs-6.0.3.175rc1.dist-info → ob_metaflow_stubs-6.0.3.176rc2.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.175rc1.dist-info → ob_metaflow_stubs-6.0.3.176rc2.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
######################################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
3
|
# MF version: 2.15.14.1+obcheckpoint(0.2.1);ob(v1) #
|
4
|
-
# Generated on 2025-
|
4
|
+
# Generated on 2025-06-03T03:11:47.762585 #
|
5
5
|
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import datetime
|
12
11
|
import typing
|
12
|
+
import datetime
|
13
13
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
14
14
|
StepFlag = typing.NewType("StepFlag", bool)
|
15
15
|
|
@@ -35,17 +35,17 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
|
|
35
35
|
from .user_configs.config_parameters import config_expr as config_expr
|
36
36
|
from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
|
37
37
|
from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
|
38
|
-
from . import tuple_util as tuple_util
|
39
38
|
from . import cards as cards
|
40
|
-
from . import events as events
|
41
39
|
from . import metaflow_git as metaflow_git
|
40
|
+
from . import tuple_util as tuple_util
|
41
|
+
from . import events as events
|
42
42
|
from . import runner as runner
|
43
43
|
from . import plugins as plugins
|
44
44
|
from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
|
45
45
|
from . import includefile as includefile
|
46
46
|
from .includefile import IncludeFile as IncludeFile
|
47
|
-
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
48
47
|
from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
|
48
|
+
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
49
49
|
from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
|
50
50
|
from . import client as client
|
51
51
|
from .client.core import namespace as namespace
|
@@ -154,6 +154,133 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
154
154
|
"""
|
155
155
|
...
|
156
156
|
|
157
|
+
@typing.overload
|
158
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
159
|
+
"""
|
160
|
+
Decorator prototype for all step decorators. This function gets specialized
|
161
|
+
and imported for all decorators types by _import_plugin_decorators().
|
162
|
+
"""
|
163
|
+
...
|
164
|
+
|
165
|
+
@typing.overload
|
166
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
167
|
+
...
|
168
|
+
|
169
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
170
|
+
"""
|
171
|
+
Decorator prototype for all step decorators. This function gets specialized
|
172
|
+
and imported for all decorators types by _import_plugin_decorators().
|
173
|
+
"""
|
174
|
+
...
|
175
|
+
|
176
|
+
@typing.overload
|
177
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
178
|
+
"""
|
179
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
180
|
+
|
181
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
182
|
+
|
183
|
+
|
184
|
+
Parameters
|
185
|
+
----------
|
186
|
+
type : str, default 'default'
|
187
|
+
Card type.
|
188
|
+
id : str, optional, default None
|
189
|
+
If multiple cards are present, use this id to identify this card.
|
190
|
+
options : Dict[str, Any], default {}
|
191
|
+
Options passed to the card. The contents depend on the card type.
|
192
|
+
timeout : int, default 45
|
193
|
+
Interrupt reporting if it takes more than this many seconds.
|
194
|
+
"""
|
195
|
+
...
|
196
|
+
|
197
|
+
@typing.overload
|
198
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
199
|
+
...
|
200
|
+
|
201
|
+
@typing.overload
|
202
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
203
|
+
...
|
204
|
+
|
205
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
206
|
+
"""
|
207
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
208
|
+
|
209
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
210
|
+
|
211
|
+
|
212
|
+
Parameters
|
213
|
+
----------
|
214
|
+
type : str, default 'default'
|
215
|
+
Card type.
|
216
|
+
id : str, optional, default None
|
217
|
+
If multiple cards are present, use this id to identify this card.
|
218
|
+
options : Dict[str, Any], default {}
|
219
|
+
Options passed to the card. The contents depend on the card type.
|
220
|
+
timeout : int, default 45
|
221
|
+
Interrupt reporting if it takes more than this many seconds.
|
222
|
+
"""
|
223
|
+
...
|
224
|
+
|
225
|
+
@typing.overload
|
226
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
227
|
+
"""
|
228
|
+
Specifies a timeout for your step.
|
229
|
+
|
230
|
+
This decorator is useful if this step may hang indefinitely.
|
231
|
+
|
232
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
233
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
234
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
235
|
+
|
236
|
+
Note that all the values specified in parameters are added together so if you specify
|
237
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
238
|
+
|
239
|
+
|
240
|
+
Parameters
|
241
|
+
----------
|
242
|
+
seconds : int, default 0
|
243
|
+
Number of seconds to wait prior to timing out.
|
244
|
+
minutes : int, default 0
|
245
|
+
Number of minutes to wait prior to timing out.
|
246
|
+
hours : int, default 0
|
247
|
+
Number of hours to wait prior to timing out.
|
248
|
+
"""
|
249
|
+
...
|
250
|
+
|
251
|
+
@typing.overload
|
252
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
253
|
+
...
|
254
|
+
|
255
|
+
@typing.overload
|
256
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
257
|
+
...
|
258
|
+
|
259
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
260
|
+
"""
|
261
|
+
Specifies a timeout for your step.
|
262
|
+
|
263
|
+
This decorator is useful if this step may hang indefinitely.
|
264
|
+
|
265
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
266
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
267
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
268
|
+
|
269
|
+
Note that all the values specified in parameters are added together so if you specify
|
270
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
271
|
+
|
272
|
+
|
273
|
+
Parameters
|
274
|
+
----------
|
275
|
+
seconds : int, default 0
|
276
|
+
Number of seconds to wait prior to timing out.
|
277
|
+
minutes : int, default 0
|
278
|
+
Number of minutes to wait prior to timing out.
|
279
|
+
hours : int, default 0
|
280
|
+
Number of hours to wait prior to timing out.
|
281
|
+
"""
|
282
|
+
...
|
283
|
+
|
157
284
|
@typing.overload
|
158
285
|
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
159
286
|
"""
|
@@ -234,297 +361,138 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
234
361
|
...
|
235
362
|
|
236
363
|
@typing.overload
|
237
|
-
def
|
364
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
238
365
|
"""
|
239
|
-
Specifies
|
366
|
+
Specifies the Conda environment for the step.
|
240
367
|
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
368
|
+
Information in this decorator will augment any
|
369
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
370
|
+
you can use `@conda_base` to set packages required by all
|
371
|
+
steps and use `@conda` to specify step-specific overrides.
|
245
372
|
|
246
373
|
|
247
374
|
Parameters
|
248
375
|
----------
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
376
|
+
packages : Dict[str, str], default {}
|
377
|
+
Packages to use for this step. The key is the name of the package
|
378
|
+
and the value is the version to use.
|
379
|
+
libraries : Dict[str, str], default {}
|
380
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
381
|
+
python : str, optional, default None
|
382
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
383
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
384
|
+
disabled : bool, default False
|
385
|
+
If set to True, disables @conda.
|
255
386
|
"""
|
256
387
|
...
|
257
388
|
|
258
389
|
@typing.overload
|
259
|
-
def
|
390
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
260
391
|
...
|
261
392
|
|
262
393
|
@typing.overload
|
263
|
-
def
|
394
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
264
395
|
...
|
265
396
|
|
266
|
-
def
|
397
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
267
398
|
"""
|
268
|
-
Specifies
|
399
|
+
Specifies the Conda environment for the step.
|
269
400
|
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
401
|
+
Information in this decorator will augment any
|
402
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
403
|
+
you can use `@conda_base` to set packages required by all
|
404
|
+
steps and use `@conda` to specify step-specific overrides.
|
274
405
|
|
275
406
|
|
276
407
|
Parameters
|
277
408
|
----------
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
409
|
+
packages : Dict[str, str], default {}
|
410
|
+
Packages to use for this step. The key is the name of the package
|
411
|
+
and the value is the version to use.
|
412
|
+
libraries : Dict[str, str], default {}
|
413
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
414
|
+
python : str, optional, default None
|
415
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
416
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
417
|
+
disabled : bool, default False
|
418
|
+
If set to True, disables @conda.
|
284
419
|
"""
|
285
420
|
...
|
286
421
|
|
287
|
-
|
288
|
-
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
422
|
+
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
289
423
|
"""
|
290
|
-
|
291
|
-
|
424
|
+
Decorator that helps cache, version and store models/datasets from huggingface hub.
|
292
425
|
|
293
426
|
|
294
427
|
Parameters
|
295
428
|
----------
|
296
|
-
|
297
|
-
|
298
|
-
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
299
|
-
- `current.checkpoint`
|
300
|
-
- `current.model`
|
301
|
-
- `current.huggingface_hub`
|
429
|
+
temp_dir_root : str, optional
|
430
|
+
The root directory that will hold the temporary directory where objects will be downloaded.
|
302
431
|
|
303
|
-
|
304
|
-
|
305
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
432
|
+
load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
|
433
|
+
The list of repos (models/datasets) to load.
|
306
434
|
|
307
|
-
|
308
|
-
|
435
|
+
Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
|
436
|
+
|
437
|
+
- If repo (model/dataset) is not found in the datastore:
|
438
|
+
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
439
|
+
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
440
|
+
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
441
|
+
|
442
|
+
- If repo is found in the datastore:
|
443
|
+
- Loads it directly from datastore to local path (can be temporary directory or specified path)
|
309
444
|
"""
|
310
445
|
...
|
311
446
|
|
312
447
|
@typing.overload
|
313
|
-
def
|
448
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
449
|
+
"""
|
450
|
+
Internal decorator to support Fast bakery
|
451
|
+
"""
|
314
452
|
...
|
315
453
|
|
316
454
|
@typing.overload
|
317
|
-
def
|
455
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
318
456
|
...
|
319
457
|
|
320
|
-
def
|
458
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
321
459
|
"""
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
Parameters
|
327
|
-
----------
|
328
|
-
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
329
|
-
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
330
|
-
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
331
|
-
- `current.checkpoint`
|
332
|
-
- `current.model`
|
333
|
-
- `current.huggingface_hub`
|
334
|
-
|
335
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
336
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
337
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
338
|
-
|
339
|
-
temp_dir_root : str, default: None
|
340
|
-
The root directory under which `current.model.loaded` will store loaded models
|
460
|
+
Internal decorator to support Fast bakery
|
341
461
|
"""
|
342
462
|
...
|
343
463
|
|
344
|
-
|
464
|
+
@typing.overload
|
465
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
345
466
|
"""
|
346
|
-
|
347
|
-
|
348
|
-
User code call
|
349
|
-
--------------
|
350
|
-
@ollama(
|
351
|
-
models=[...],
|
352
|
-
...
|
353
|
-
)
|
354
|
-
|
355
|
-
Valid backend options
|
356
|
-
---------------------
|
357
|
-
- 'local': Run as a separate process on the local task machine.
|
358
|
-
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
359
|
-
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
360
|
-
|
361
|
-
Valid model options
|
362
|
-
-------------------
|
363
|
-
Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
|
364
|
-
|
365
|
-
|
366
|
-
Parameters
|
367
|
-
----------
|
368
|
-
models: list[str]
|
369
|
-
List of Ollama containers running models in sidecars.
|
370
|
-
backend: str
|
371
|
-
Determines where and how to run the Ollama process.
|
372
|
-
force_pull: bool
|
373
|
-
Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
|
374
|
-
skip_push_check: bool
|
375
|
-
Whether to skip the check that populates/overwrites remote cache on terminating an ollama model.
|
376
|
-
debug: bool
|
377
|
-
Whether to turn on verbose debugging logs.
|
378
|
-
"""
|
379
|
-
...
|
380
|
-
|
381
|
-
def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
382
|
-
"""
|
383
|
-
Specifies that this step is used to deploy an instance of the app.
|
384
|
-
Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
|
467
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
468
|
+
the execution of a step.
|
385
469
|
|
386
470
|
|
387
471
|
Parameters
|
388
472
|
----------
|
389
|
-
|
390
|
-
|
391
|
-
app_name : str
|
392
|
-
Name of the app to deploy.
|
393
|
-
"""
|
394
|
-
...
|
395
|
-
|
396
|
-
@typing.overload
|
397
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
398
|
-
"""
|
399
|
-
Decorator prototype for all step decorators. This function gets specialized
|
400
|
-
and imported for all decorators types by _import_plugin_decorators().
|
401
|
-
"""
|
402
|
-
...
|
403
|
-
|
404
|
-
@typing.overload
|
405
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
406
|
-
...
|
407
|
-
|
408
|
-
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
409
|
-
"""
|
410
|
-
Decorator prototype for all step decorators. This function gets specialized
|
411
|
-
and imported for all decorators types by _import_plugin_decorators().
|
473
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
474
|
+
List of secret specs, defining how the secrets are to be retrieved
|
412
475
|
"""
|
413
476
|
...
|
414
477
|
|
415
478
|
@typing.overload
|
416
|
-
def
|
417
|
-
"""
|
418
|
-
Internal decorator to support Fast bakery
|
419
|
-
"""
|
479
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
420
480
|
...
|
421
481
|
|
422
482
|
@typing.overload
|
423
|
-
def
|
424
|
-
...
|
425
|
-
|
426
|
-
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
427
|
-
"""
|
428
|
-
Internal decorator to support Fast bakery
|
429
|
-
"""
|
430
|
-
...
|
431
|
-
|
432
|
-
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
433
|
-
"""
|
434
|
-
Specifies that this step should execute on DGX cloud.
|
435
|
-
|
436
|
-
|
437
|
-
Parameters
|
438
|
-
----------
|
439
|
-
gpu : int
|
440
|
-
Number of GPUs to use.
|
441
|
-
gpu_type : str
|
442
|
-
Type of Nvidia GPU to use.
|
443
|
-
"""
|
483
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
444
484
|
...
|
445
485
|
|
446
|
-
def
|
486
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
447
487
|
"""
|
448
|
-
Specifies
|
488
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
489
|
+
the execution of a step.
|
449
490
|
|
450
491
|
|
451
492
|
Parameters
|
452
493
|
----------
|
453
|
-
|
454
|
-
|
455
|
-
also present, the maximum value from all decorators is used.
|
456
|
-
memory : int, default 4096
|
457
|
-
Memory size (in MB) required for this step. If
|
458
|
-
`@resources` is also present, the maximum value from all decorators is
|
459
|
-
used.
|
460
|
-
disk : int, default 10240
|
461
|
-
Disk size (in MB) required for this step. If
|
462
|
-
`@resources` is also present, the maximum value from all decorators is
|
463
|
-
used.
|
464
|
-
image : str, optional, default None
|
465
|
-
Docker image to use when launching on Kubernetes. If not specified, and
|
466
|
-
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
467
|
-
not, a default Docker image mapping to the current version of Python is used.
|
468
|
-
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
469
|
-
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
470
|
-
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
471
|
-
Kubernetes service account to use when launching pod in Kubernetes.
|
472
|
-
secrets : List[str], optional, default None
|
473
|
-
Kubernetes secrets to use when launching pod in Kubernetes. These
|
474
|
-
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
475
|
-
in Metaflow configuration.
|
476
|
-
node_selector: Union[Dict[str,str], str], optional, default None
|
477
|
-
Kubernetes node selector(s) to apply to the pod running the task.
|
478
|
-
Can be passed in as a comma separated string of values e.g.
|
479
|
-
'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
|
480
|
-
{'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
|
481
|
-
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
482
|
-
Kubernetes namespace to use when launching pod in Kubernetes.
|
483
|
-
gpu : int, optional, default None
|
484
|
-
Number of GPUs required for this step. A value of zero implies that
|
485
|
-
the scheduled node should not have GPUs.
|
486
|
-
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
487
|
-
The vendor of the GPUs to be used for this step.
|
488
|
-
tolerations : List[str], default []
|
489
|
-
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
490
|
-
Kubernetes tolerations to use when launching pod in Kubernetes.
|
491
|
-
labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
|
492
|
-
Kubernetes labels to use when launching pod in Kubernetes.
|
493
|
-
annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
|
494
|
-
Kubernetes annotations to use when launching pod in Kubernetes.
|
495
|
-
use_tmpfs : bool, default False
|
496
|
-
This enables an explicit tmpfs mount for this step.
|
497
|
-
tmpfs_tempdir : bool, default True
|
498
|
-
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
499
|
-
tmpfs_size : int, optional, default: None
|
500
|
-
The value for the size (in MiB) of the tmpfs mount for this step.
|
501
|
-
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
502
|
-
memory allocated for this step.
|
503
|
-
tmpfs_path : str, optional, default /metaflow_temp
|
504
|
-
Path to tmpfs mount for this step.
|
505
|
-
persistent_volume_claims : Dict[str, str], optional, default None
|
506
|
-
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
507
|
-
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
508
|
-
shared_memory: int, optional
|
509
|
-
Shared memory size (in MiB) required for this step
|
510
|
-
port: int, optional
|
511
|
-
Port number to specify in the Kubernetes job object
|
512
|
-
compute_pool : str, optional, default None
|
513
|
-
Compute pool to be used for for this step.
|
514
|
-
If not specified, any accessible compute pool within the perimeter is used.
|
515
|
-
hostname_resolution_timeout: int, default 10 * 60
|
516
|
-
Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
|
517
|
-
Only applicable when @parallel is used.
|
518
|
-
qos: str, default: Burstable
|
519
|
-
Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
|
520
|
-
|
521
|
-
security_context: Dict[str, Any], optional, default None
|
522
|
-
Container security context. Applies to the task container. Allows the following keys:
|
523
|
-
- privileged: bool, optional, default None
|
524
|
-
- allow_privilege_escalation: bool, optional, default None
|
525
|
-
- run_as_user: int, optional, default None
|
526
|
-
- run_as_group: int, optional, default None
|
527
|
-
- run_as_non_root: bool, optional, default None
|
494
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
495
|
+
List of secret specs, defining how the secrets are to be retrieved
|
528
496
|
"""
|
529
497
|
...
|
530
498
|
|
@@ -577,198 +545,75 @@ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], Non
|
|
577
545
|
"""
|
578
546
|
...
|
579
547
|
|
580
|
-
def
|
548
|
+
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
581
549
|
"""
|
582
|
-
|
550
|
+
Specifies that this step should execute on DGX cloud.
|
583
551
|
|
584
552
|
|
585
553
|
Parameters
|
586
554
|
----------
|
587
|
-
|
588
|
-
|
589
|
-
|
590
|
-
|
591
|
-
The list of repos (models/datasets) to load.
|
592
|
-
|
593
|
-
Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
|
594
|
-
|
595
|
-
- If repo (model/dataset) is not found in the datastore:
|
596
|
-
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
597
|
-
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
598
|
-
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
599
|
-
|
600
|
-
- If repo is found in the datastore:
|
601
|
-
- Loads it directly from datastore to local path (can be temporary directory or specified path)
|
555
|
+
gpu : int
|
556
|
+
Number of GPUs to use.
|
557
|
+
gpu_type : str
|
558
|
+
Type of Nvidia GPU to use.
|
602
559
|
"""
|
603
560
|
...
|
604
561
|
|
605
562
|
@typing.overload
|
606
|
-
def
|
563
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
607
564
|
"""
|
608
|
-
Specifies the
|
609
|
-
to a step needs to be retried.
|
610
|
-
|
611
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
612
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
613
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
565
|
+
Specifies the PyPI packages for the step.
|
614
566
|
|
615
|
-
|
616
|
-
|
617
|
-
|
567
|
+
Information in this decorator will augment any
|
568
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
569
|
+
you can use `@pypi_base` to set packages required by all
|
570
|
+
steps and use `@pypi` to specify step-specific overrides.
|
618
571
|
|
619
572
|
|
620
573
|
Parameters
|
621
574
|
----------
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
|
575
|
+
packages : Dict[str, str], default: {}
|
576
|
+
Packages to use for this step. The key is the name of the package
|
577
|
+
and the value is the version to use.
|
578
|
+
python : str, optional, default: None
|
579
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
580
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
626
581
|
"""
|
627
582
|
...
|
628
583
|
|
629
584
|
@typing.overload
|
630
|
-
def
|
585
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
631
586
|
...
|
632
587
|
|
633
588
|
@typing.overload
|
634
|
-
def
|
589
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
635
590
|
...
|
636
591
|
|
637
|
-
def
|
592
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
638
593
|
"""
|
639
|
-
Specifies the
|
640
|
-
to a step needs to be retried.
|
641
|
-
|
642
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
643
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
644
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
594
|
+
Specifies the PyPI packages for the step.
|
645
595
|
|
646
|
-
|
647
|
-
|
648
|
-
|
596
|
+
Information in this decorator will augment any
|
597
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
598
|
+
you can use `@pypi_base` to set packages required by all
|
599
|
+
steps and use `@pypi` to specify step-specific overrides.
|
649
600
|
|
650
601
|
|
651
602
|
Parameters
|
652
603
|
----------
|
653
|
-
|
654
|
-
|
655
|
-
|
656
|
-
|
604
|
+
packages : Dict[str, str], default: {}
|
605
|
+
Packages to use for this step. The key is the name of the package
|
606
|
+
and the value is the version to use.
|
607
|
+
python : str, optional, default: None
|
608
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
609
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
657
610
|
"""
|
658
611
|
...
|
659
612
|
|
660
613
|
@typing.overload
|
661
|
-
def
|
614
|
+
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
662
615
|
"""
|
663
|
-
|
664
|
-
|
665
|
-
This decorator is useful if this step may hang indefinitely.
|
666
|
-
|
667
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
668
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
669
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
670
|
-
|
671
|
-
Note that all the values specified in parameters are added together so if you specify
|
672
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
673
|
-
|
674
|
-
|
675
|
-
Parameters
|
676
|
-
----------
|
677
|
-
seconds : int, default 0
|
678
|
-
Number of seconds to wait prior to timing out.
|
679
|
-
minutes : int, default 0
|
680
|
-
Number of minutes to wait prior to timing out.
|
681
|
-
hours : int, default 0
|
682
|
-
Number of hours to wait prior to timing out.
|
683
|
-
"""
|
684
|
-
...
|
685
|
-
|
686
|
-
@typing.overload
|
687
|
-
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
688
|
-
...
|
689
|
-
|
690
|
-
@typing.overload
|
691
|
-
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
692
|
-
...
|
693
|
-
|
694
|
-
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
695
|
-
"""
|
696
|
-
Specifies a timeout for your step.
|
697
|
-
|
698
|
-
This decorator is useful if this step may hang indefinitely.
|
699
|
-
|
700
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
701
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
702
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
703
|
-
|
704
|
-
Note that all the values specified in parameters are added together so if you specify
|
705
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
706
|
-
|
707
|
-
|
708
|
-
Parameters
|
709
|
-
----------
|
710
|
-
seconds : int, default 0
|
711
|
-
Number of seconds to wait prior to timing out.
|
712
|
-
minutes : int, default 0
|
713
|
-
Number of minutes to wait prior to timing out.
|
714
|
-
hours : int, default 0
|
715
|
-
Number of hours to wait prior to timing out.
|
716
|
-
"""
|
717
|
-
...
|
718
|
-
|
719
|
-
@typing.overload
|
720
|
-
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
721
|
-
"""
|
722
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
723
|
-
|
724
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
725
|
-
|
726
|
-
|
727
|
-
Parameters
|
728
|
-
----------
|
729
|
-
type : str, default 'default'
|
730
|
-
Card type.
|
731
|
-
id : str, optional, default None
|
732
|
-
If multiple cards are present, use this id to identify this card.
|
733
|
-
options : Dict[str, Any], default {}
|
734
|
-
Options passed to the card. The contents depend on the card type.
|
735
|
-
timeout : int, default 45
|
736
|
-
Interrupt reporting if it takes more than this many seconds.
|
737
|
-
"""
|
738
|
-
...
|
739
|
-
|
740
|
-
@typing.overload
|
741
|
-
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
742
|
-
...
|
743
|
-
|
744
|
-
@typing.overload
|
745
|
-
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
746
|
-
...
|
747
|
-
|
748
|
-
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
749
|
-
"""
|
750
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
751
|
-
|
752
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
753
|
-
|
754
|
-
|
755
|
-
Parameters
|
756
|
-
----------
|
757
|
-
type : str, default 'default'
|
758
|
-
Card type.
|
759
|
-
id : str, optional, default None
|
760
|
-
If multiple cards are present, use this id to identify this card.
|
761
|
-
options : Dict[str, Any], default {}
|
762
|
-
Options passed to the card. The contents depend on the card type.
|
763
|
-
timeout : int, default 45
|
764
|
-
Interrupt reporting if it takes more than this many seconds.
|
765
|
-
"""
|
766
|
-
...
|
767
|
-
|
768
|
-
@typing.overload
|
769
|
-
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
770
|
-
"""
|
771
|
-
Enables checkpointing for a step.
|
616
|
+
Enables checkpointing for a step.
|
772
617
|
|
773
618
|
|
774
619
|
|
@@ -822,393 +667,529 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
|
|
822
667
|
"""
|
823
668
|
...
|
824
669
|
|
670
|
+
def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
671
|
+
"""
|
672
|
+
Specifies that this step is used to deploy an instance of the app.
|
673
|
+
Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
|
674
|
+
|
675
|
+
|
676
|
+
Parameters
|
677
|
+
----------
|
678
|
+
app_port : int
|
679
|
+
Number of GPUs to use.
|
680
|
+
app_name : str
|
681
|
+
Name of the app to deploy.
|
682
|
+
"""
|
683
|
+
...
|
684
|
+
|
825
685
|
@typing.overload
|
826
|
-
def
|
686
|
+
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
827
687
|
"""
|
828
|
-
|
688
|
+
Enables loading / saving of models within a step.
|
829
689
|
|
830
|
-
Information in this decorator will augment any
|
831
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
832
|
-
you can use `@conda_base` to set packages required by all
|
833
|
-
steps and use `@conda` to specify step-specific overrides.
|
834
690
|
|
835
691
|
|
836
692
|
Parameters
|
837
693
|
----------
|
838
|
-
|
839
|
-
|
840
|
-
|
841
|
-
|
842
|
-
|
843
|
-
|
844
|
-
|
845
|
-
|
846
|
-
|
847
|
-
If
|
694
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
695
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
696
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
697
|
+
- `current.checkpoint`
|
698
|
+
- `current.model`
|
699
|
+
- `current.huggingface_hub`
|
700
|
+
|
701
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
702
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
703
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
704
|
+
|
705
|
+
temp_dir_root : str, default: None
|
706
|
+
The root directory under which `current.model.loaded` will store loaded models
|
848
707
|
"""
|
849
708
|
...
|
850
709
|
|
851
710
|
@typing.overload
|
852
|
-
def
|
711
|
+
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
853
712
|
...
|
854
713
|
|
855
714
|
@typing.overload
|
856
|
-
def
|
715
|
+
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
857
716
|
...
|
858
717
|
|
859
|
-
def
|
718
|
+
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
860
719
|
"""
|
861
|
-
|
720
|
+
Enables loading / saving of models within a step.
|
862
721
|
|
863
|
-
Information in this decorator will augment any
|
864
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
865
|
-
you can use `@conda_base` to set packages required by all
|
866
|
-
steps and use `@conda` to specify step-specific overrides.
|
867
722
|
|
868
723
|
|
869
724
|
Parameters
|
870
725
|
----------
|
871
|
-
|
872
|
-
|
873
|
-
|
874
|
-
|
875
|
-
|
876
|
-
|
877
|
-
|
878
|
-
|
879
|
-
|
880
|
-
If
|
726
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
727
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
728
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
729
|
+
- `current.checkpoint`
|
730
|
+
- `current.model`
|
731
|
+
- `current.huggingface_hub`
|
732
|
+
|
733
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
734
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
735
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
736
|
+
|
737
|
+
temp_dir_root : str, default: None
|
738
|
+
The root directory under which `current.model.loaded` will store loaded models
|
881
739
|
"""
|
882
740
|
...
|
883
741
|
|
884
742
|
@typing.overload
|
885
|
-
def
|
743
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
886
744
|
"""
|
887
|
-
Specifies
|
888
|
-
|
745
|
+
Specifies that the step will success under all circumstances.
|
746
|
+
|
747
|
+
The decorator will create an optional artifact, specified by `var`, which
|
748
|
+
contains the exception raised. You can use it to detect the presence
|
749
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
750
|
+
are missing.
|
889
751
|
|
890
752
|
|
891
753
|
Parameters
|
892
754
|
----------
|
893
|
-
|
894
|
-
|
755
|
+
var : str, optional, default None
|
756
|
+
Name of the artifact in which to store the caught exception.
|
757
|
+
If not specified, the exception is not stored.
|
758
|
+
print_exception : bool, default True
|
759
|
+
Determines whether or not the exception is printed to
|
760
|
+
stdout when caught.
|
895
761
|
"""
|
896
762
|
...
|
897
763
|
|
898
764
|
@typing.overload
|
899
|
-
def
|
765
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
900
766
|
...
|
901
767
|
|
902
768
|
@typing.overload
|
903
|
-
def
|
769
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
904
770
|
...
|
905
771
|
|
906
|
-
def
|
772
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
907
773
|
"""
|
908
|
-
Specifies
|
909
|
-
|
774
|
+
Specifies that the step will success under all circumstances.
|
775
|
+
|
776
|
+
The decorator will create an optional artifact, specified by `var`, which
|
777
|
+
contains the exception raised. You can use it to detect the presence
|
778
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
779
|
+
are missing.
|
910
780
|
|
911
781
|
|
912
782
|
Parameters
|
913
783
|
----------
|
914
|
-
|
915
|
-
|
784
|
+
var : str, optional, default None
|
785
|
+
Name of the artifact in which to store the caught exception.
|
786
|
+
If not specified, the exception is not stored.
|
787
|
+
print_exception : bool, default True
|
788
|
+
Determines whether or not the exception is printed to
|
789
|
+
stdout when caught.
|
916
790
|
"""
|
917
791
|
...
|
918
792
|
|
919
793
|
@typing.overload
|
920
|
-
def
|
794
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
921
795
|
"""
|
922
|
-
Specifies the
|
796
|
+
Specifies the number of times the task corresponding
|
797
|
+
to a step needs to be retried.
|
923
798
|
|
924
|
-
|
925
|
-
|
926
|
-
|
927
|
-
|
799
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
800
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
801
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
802
|
+
|
803
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
804
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
805
|
+
ensuring that the flow execution can continue.
|
928
806
|
|
929
807
|
|
930
808
|
Parameters
|
931
809
|
----------
|
932
|
-
|
933
|
-
|
934
|
-
|
935
|
-
|
936
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
937
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
810
|
+
times : int, default 3
|
811
|
+
Number of times to retry this task.
|
812
|
+
minutes_between_retries : int, default 2
|
813
|
+
Number of minutes between retries.
|
938
814
|
"""
|
939
815
|
...
|
940
816
|
|
941
817
|
@typing.overload
|
942
|
-
def
|
818
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
943
819
|
...
|
944
820
|
|
945
821
|
@typing.overload
|
946
|
-
def
|
822
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
947
823
|
...
|
948
824
|
|
949
|
-
def
|
825
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
950
826
|
"""
|
951
|
-
Specifies the
|
827
|
+
Specifies the number of times the task corresponding
|
828
|
+
to a step needs to be retried.
|
952
829
|
|
953
|
-
|
954
|
-
|
955
|
-
|
956
|
-
|
830
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
831
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
832
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
833
|
+
|
834
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
835
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
836
|
+
ensuring that the flow execution can continue.
|
957
837
|
|
958
838
|
|
959
839
|
Parameters
|
960
840
|
----------
|
961
|
-
|
962
|
-
|
963
|
-
|
964
|
-
|
965
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
966
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
841
|
+
times : int, default 3
|
842
|
+
Number of times to retry this task.
|
843
|
+
minutes_between_retries : int, default 2
|
844
|
+
Number of minutes between retries.
|
967
845
|
"""
|
968
846
|
...
|
969
847
|
|
970
|
-
|
971
|
-
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
848
|
+
def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
972
849
|
"""
|
973
|
-
|
850
|
+
This decorator is used to run Ollama APIs as Metaflow task sidecars.
|
974
851
|
|
975
|
-
|
976
|
-
|
977
|
-
|
978
|
-
|
979
|
-
|
980
|
-
|
981
|
-
```
|
852
|
+
User code call
|
853
|
+
--------------
|
854
|
+
@ollama(
|
855
|
+
models=[...],
|
856
|
+
...
|
857
|
+
)
|
982
858
|
|
983
|
-
|
984
|
-
|
985
|
-
|
986
|
-
|
987
|
-
|
988
|
-
or
|
989
|
-
```
|
990
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
991
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
992
|
-
```
|
859
|
+
Valid backend options
|
860
|
+
---------------------
|
861
|
+
- 'local': Run as a separate process on the local task machine.
|
862
|
+
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
863
|
+
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
993
864
|
|
994
|
-
|
995
|
-
|
996
|
-
|
997
|
-
```
|
998
|
-
This is equivalent to:
|
999
|
-
```
|
1000
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1001
|
-
```
|
865
|
+
Valid model options
|
866
|
+
-------------------
|
867
|
+
Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
|
1002
868
|
|
1003
869
|
|
1004
870
|
Parameters
|
1005
871
|
----------
|
1006
|
-
|
1007
|
-
|
1008
|
-
|
1009
|
-
|
1010
|
-
|
1011
|
-
|
872
|
+
models: list[str]
|
873
|
+
List of Ollama containers running models in sidecars.
|
874
|
+
backend: str
|
875
|
+
Determines where and how to run the Ollama process.
|
876
|
+
force_pull: bool
|
877
|
+
Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
|
878
|
+
cache_update_policy: str
|
879
|
+
Cache update policy: "auto", "force", or "never".
|
880
|
+
force_cache_update: bool
|
881
|
+
Simple override for "force" cache update policy.
|
882
|
+
debug: bool
|
883
|
+
Whether to turn on verbose debugging logs.
|
884
|
+
circuit_breaker_config: dict
|
885
|
+
Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
|
886
|
+
timeout_config: dict
|
887
|
+
Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
|
1012
888
|
"""
|
1013
889
|
...
|
1014
890
|
|
1015
|
-
|
1016
|
-
|
891
|
+
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
892
|
+
"""
|
893
|
+
Specifies that this step should execute on Kubernetes.
|
894
|
+
|
895
|
+
|
896
|
+
Parameters
|
897
|
+
----------
|
898
|
+
cpu : int, default 1
|
899
|
+
Number of CPUs required for this step. If `@resources` is
|
900
|
+
also present, the maximum value from all decorators is used.
|
901
|
+
memory : int, default 4096
|
902
|
+
Memory size (in MB) required for this step. If
|
903
|
+
`@resources` is also present, the maximum value from all decorators is
|
904
|
+
used.
|
905
|
+
disk : int, default 10240
|
906
|
+
Disk size (in MB) required for this step. If
|
907
|
+
`@resources` is also present, the maximum value from all decorators is
|
908
|
+
used.
|
909
|
+
image : str, optional, default None
|
910
|
+
Docker image to use when launching on Kubernetes. If not specified, and
|
911
|
+
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
912
|
+
not, a default Docker image mapping to the current version of Python is used.
|
913
|
+
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
914
|
+
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
915
|
+
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
916
|
+
Kubernetes service account to use when launching pod in Kubernetes.
|
917
|
+
secrets : List[str], optional, default None
|
918
|
+
Kubernetes secrets to use when launching pod in Kubernetes. These
|
919
|
+
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
920
|
+
in Metaflow configuration.
|
921
|
+
node_selector: Union[Dict[str,str], str], optional, default None
|
922
|
+
Kubernetes node selector(s) to apply to the pod running the task.
|
923
|
+
Can be passed in as a comma separated string of values e.g.
|
924
|
+
'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
|
925
|
+
{'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
|
926
|
+
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
927
|
+
Kubernetes namespace to use when launching pod in Kubernetes.
|
928
|
+
gpu : int, optional, default None
|
929
|
+
Number of GPUs required for this step. A value of zero implies that
|
930
|
+
the scheduled node should not have GPUs.
|
931
|
+
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
932
|
+
The vendor of the GPUs to be used for this step.
|
933
|
+
tolerations : List[str], default []
|
934
|
+
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
935
|
+
Kubernetes tolerations to use when launching pod in Kubernetes.
|
936
|
+
labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
|
937
|
+
Kubernetes labels to use when launching pod in Kubernetes.
|
938
|
+
annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
|
939
|
+
Kubernetes annotations to use when launching pod in Kubernetes.
|
940
|
+
use_tmpfs : bool, default False
|
941
|
+
This enables an explicit tmpfs mount for this step.
|
942
|
+
tmpfs_tempdir : bool, default True
|
943
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
944
|
+
tmpfs_size : int, optional, default: None
|
945
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
946
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
947
|
+
memory allocated for this step.
|
948
|
+
tmpfs_path : str, optional, default /metaflow_temp
|
949
|
+
Path to tmpfs mount for this step.
|
950
|
+
persistent_volume_claims : Dict[str, str], optional, default None
|
951
|
+
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
952
|
+
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
953
|
+
shared_memory: int, optional
|
954
|
+
Shared memory size (in MiB) required for this step
|
955
|
+
port: int, optional
|
956
|
+
Port number to specify in the Kubernetes job object
|
957
|
+
compute_pool : str, optional, default None
|
958
|
+
Compute pool to be used for for this step.
|
959
|
+
If not specified, any accessible compute pool within the perimeter is used.
|
960
|
+
hostname_resolution_timeout: int, default 10 * 60
|
961
|
+
Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
|
962
|
+
Only applicable when @parallel is used.
|
963
|
+
qos: str, default: Burstable
|
964
|
+
Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
|
965
|
+
|
966
|
+
security_context: Dict[str, Any], optional, default None
|
967
|
+
Container security context. Applies to the task container. Allows the following keys:
|
968
|
+
- privileged: bool, optional, default None
|
969
|
+
- allow_privilege_escalation: bool, optional, default None
|
970
|
+
- run_as_user: int, optional, default None
|
971
|
+
- run_as_group: int, optional, default None
|
972
|
+
- run_as_non_root: bool, optional, default None
|
973
|
+
"""
|
1017
974
|
...
|
1018
975
|
|
1019
|
-
def
|
976
|
+
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1020
977
|
"""
|
1021
|
-
Specifies
|
978
|
+
Specifies what flows belong to the same project.
|
1022
979
|
|
1023
|
-
|
1024
|
-
|
1025
|
-
```
|
1026
|
-
or
|
1027
|
-
```
|
1028
|
-
@trigger(events=['foo', 'bar'])
|
1029
|
-
```
|
980
|
+
A project-specific namespace is created for all flows that
|
981
|
+
use the same `@project(name)`.
|
1030
982
|
|
1031
|
-
Additionally, you can specify the parameter mappings
|
1032
|
-
to map event payload to Metaflow parameters for the flow.
|
1033
|
-
```
|
1034
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1035
|
-
```
|
1036
|
-
or
|
1037
|
-
```
|
1038
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1039
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1040
|
-
```
|
1041
983
|
|
1042
|
-
|
1043
|
-
|
1044
|
-
|
1045
|
-
|
1046
|
-
|
1047
|
-
|
1048
|
-
|
1049
|
-
|
984
|
+
Parameters
|
985
|
+
----------
|
986
|
+
name : str
|
987
|
+
Project name. Make sure that the name is unique amongst all
|
988
|
+
projects that use the same production scheduler. The name may
|
989
|
+
contain only lowercase alphanumeric characters and underscores.
|
990
|
+
|
991
|
+
branch : Optional[str], default None
|
992
|
+
The branch to use. If not specified, the branch is set to
|
993
|
+
`user.<username>` unless `production` is set to `True`. This can
|
994
|
+
also be set on the command line using `--branch` as a top-level option.
|
995
|
+
It is an error to specify `branch` in the decorator and on the command line.
|
996
|
+
|
997
|
+
production : bool, default False
|
998
|
+
Whether or not the branch is the production branch. This can also be set on the
|
999
|
+
command line using `--production` as a top-level option. It is an error to specify
|
1000
|
+
`production` in the decorator and on the command line.
|
1001
|
+
The project branch name will be:
|
1002
|
+
- if `branch` is specified:
|
1003
|
+
- if `production` is True: `prod.<branch>`
|
1004
|
+
- if `production` is False: `test.<branch>`
|
1005
|
+
- if `branch` is not specified:
|
1006
|
+
- if `production` is True: `prod`
|
1007
|
+
- if `production` is False: `user.<username>`
|
1008
|
+
"""
|
1009
|
+
...
|
1010
|
+
|
1011
|
+
@typing.overload
|
1012
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1013
|
+
"""
|
1014
|
+
Specifies the Conda environment for all steps of the flow.
|
1015
|
+
|
1016
|
+
Use `@conda_base` to set common libraries required by all
|
1017
|
+
steps and use `@conda` to specify step-specific additions.
|
1050
1018
|
|
1051
1019
|
|
1052
1020
|
Parameters
|
1053
1021
|
----------
|
1054
|
-
|
1055
|
-
|
1056
|
-
|
1057
|
-
|
1058
|
-
|
1059
|
-
|
1022
|
+
packages : Dict[str, str], default {}
|
1023
|
+
Packages to use for this flow. The key is the name of the package
|
1024
|
+
and the value is the version to use.
|
1025
|
+
libraries : Dict[str, str], default {}
|
1026
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1027
|
+
python : str, optional, default None
|
1028
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1029
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1030
|
+
disabled : bool, default False
|
1031
|
+
If set to True, disables Conda.
|
1060
1032
|
"""
|
1061
1033
|
...
|
1062
1034
|
|
1063
1035
|
@typing.overload
|
1064
|
-
def
|
1036
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1037
|
+
...
|
1038
|
+
|
1039
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1065
1040
|
"""
|
1066
|
-
Specifies the
|
1067
|
-
|
1068
|
-
```
|
1069
|
-
@trigger_on_finish(flow='FooFlow')
|
1070
|
-
```
|
1071
|
-
or
|
1072
|
-
```
|
1073
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1074
|
-
```
|
1075
|
-
This decorator respects the @project decorator and triggers the flow
|
1076
|
-
when upstream runs within the same namespace complete successfully
|
1041
|
+
Specifies the Conda environment for all steps of the flow.
|
1077
1042
|
|
1078
|
-
|
1079
|
-
|
1080
|
-
```
|
1081
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1082
|
-
```
|
1083
|
-
or
|
1084
|
-
```
|
1085
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1086
|
-
```
|
1043
|
+
Use `@conda_base` to set common libraries required by all
|
1044
|
+
steps and use `@conda` to specify step-specific additions.
|
1087
1045
|
|
1088
|
-
You can also specify just the project or project branch (other values will be
|
1089
|
-
inferred from the current project or project branch):
|
1090
|
-
```
|
1091
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1092
|
-
```
|
1093
1046
|
|
1094
|
-
|
1095
|
-
|
1096
|
-
|
1097
|
-
|
1098
|
-
|
1047
|
+
Parameters
|
1048
|
+
----------
|
1049
|
+
packages : Dict[str, str], default {}
|
1050
|
+
Packages to use for this flow. The key is the name of the package
|
1051
|
+
and the value is the version to use.
|
1052
|
+
libraries : Dict[str, str], default {}
|
1053
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1054
|
+
python : str, optional, default None
|
1055
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1056
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1057
|
+
disabled : bool, default False
|
1058
|
+
If set to True, disables Conda.
|
1059
|
+
"""
|
1060
|
+
...
|
1061
|
+
|
1062
|
+
@typing.overload
|
1063
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1064
|
+
"""
|
1065
|
+
Specifies the PyPI packages for all steps of the flow.
|
1099
1066
|
|
1067
|
+
Use `@pypi_base` to set common packages required by all
|
1068
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1100
1069
|
|
1101
1070
|
Parameters
|
1102
1071
|
----------
|
1103
|
-
|
1104
|
-
|
1105
|
-
|
1106
|
-
|
1107
|
-
|
1108
|
-
|
1072
|
+
packages : Dict[str, str], default: {}
|
1073
|
+
Packages to use for this flow. The key is the name of the package
|
1074
|
+
and the value is the version to use.
|
1075
|
+
python : str, optional, default: None
|
1076
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1077
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1109
1078
|
"""
|
1110
1079
|
...
|
1111
1080
|
|
1112
1081
|
@typing.overload
|
1113
|
-
def
|
1082
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1114
1083
|
...
|
1115
1084
|
|
1116
|
-
def
|
1085
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1117
1086
|
"""
|
1118
|
-
Specifies the
|
1087
|
+
Specifies the PyPI packages for all steps of the flow.
|
1088
|
+
|
1089
|
+
Use `@pypi_base` to set common packages required by all
|
1090
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1091
|
+
|
1092
|
+
Parameters
|
1093
|
+
----------
|
1094
|
+
packages : Dict[str, str], default: {}
|
1095
|
+
Packages to use for this flow. The key is the name of the package
|
1096
|
+
and the value is the version to use.
|
1097
|
+
python : str, optional, default: None
|
1098
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1099
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1100
|
+
"""
|
1101
|
+
...
|
1102
|
+
|
1103
|
+
@typing.overload
|
1104
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1105
|
+
"""
|
1106
|
+
Specifies the event(s) that this flow depends on.
|
1119
1107
|
|
1120
1108
|
```
|
1121
|
-
@
|
1109
|
+
@trigger(event='foo')
|
1122
1110
|
```
|
1123
1111
|
or
|
1124
1112
|
```
|
1125
|
-
@
|
1113
|
+
@trigger(events=['foo', 'bar'])
|
1126
1114
|
```
|
1127
|
-
This decorator respects the @project decorator and triggers the flow
|
1128
|
-
when upstream runs within the same namespace complete successfully
|
1129
1115
|
|
1130
|
-
Additionally, you can specify
|
1131
|
-
|
1116
|
+
Additionally, you can specify the parameter mappings
|
1117
|
+
to map event payload to Metaflow parameters for the flow.
|
1132
1118
|
```
|
1133
|
-
@
|
1119
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1134
1120
|
```
|
1135
1121
|
or
|
1136
1122
|
```
|
1137
|
-
@
|
1138
|
-
|
1139
|
-
|
1140
|
-
You can also specify just the project or project branch (other values will be
|
1141
|
-
inferred from the current project or project branch):
|
1142
|
-
```
|
1143
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1144
|
-
```
|
1145
|
-
|
1146
|
-
Note that `branch` is typically one of:
|
1147
|
-
- `prod`
|
1148
|
-
- `user.bob`
|
1149
|
-
- `test.my_experiment`
|
1150
|
-
- `prod.staging`
|
1151
|
-
|
1152
|
-
|
1153
|
-
Parameters
|
1154
|
-
----------
|
1155
|
-
flow : Union[str, Dict[str, str]], optional, default None
|
1156
|
-
Upstream flow dependency for this flow.
|
1157
|
-
flows : List[Union[str, Dict[str, str]]], default []
|
1158
|
-
Upstream flow dependencies for this flow.
|
1159
|
-
options : Dict[str, Any], default {}
|
1160
|
-
Backend-specific configuration for tuning eventing behavior.
|
1161
|
-
"""
|
1162
|
-
...
|
1163
|
-
|
1164
|
-
@typing.overload
|
1165
|
-
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1166
|
-
"""
|
1167
|
-
Specifies the Conda environment for all steps of the flow.
|
1123
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1124
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1125
|
+
```
|
1168
1126
|
|
1169
|
-
|
1170
|
-
|
1127
|
+
'parameters' can also be a list of strings and tuples like so:
|
1128
|
+
```
|
1129
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1130
|
+
```
|
1131
|
+
This is equivalent to:
|
1132
|
+
```
|
1133
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1134
|
+
```
|
1171
1135
|
|
1172
1136
|
|
1173
1137
|
Parameters
|
1174
1138
|
----------
|
1175
|
-
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1179
|
-
|
1180
|
-
|
1181
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1182
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1183
|
-
disabled : bool, default False
|
1184
|
-
If set to True, disables Conda.
|
1139
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1140
|
+
Event dependency for this flow.
|
1141
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1142
|
+
Events dependency for this flow.
|
1143
|
+
options : Dict[str, Any], default {}
|
1144
|
+
Backend-specific configuration for tuning eventing behavior.
|
1185
1145
|
"""
|
1186
1146
|
...
|
1187
1147
|
|
1188
1148
|
@typing.overload
|
1189
|
-
def
|
1149
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1190
1150
|
...
|
1191
1151
|
|
1192
|
-
def
|
1152
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1193
1153
|
"""
|
1194
|
-
Specifies the
|
1154
|
+
Specifies the event(s) that this flow depends on.
|
1195
1155
|
|
1196
|
-
|
1197
|
-
|
1156
|
+
```
|
1157
|
+
@trigger(event='foo')
|
1158
|
+
```
|
1159
|
+
or
|
1160
|
+
```
|
1161
|
+
@trigger(events=['foo', 'bar'])
|
1162
|
+
```
|
1163
|
+
|
1164
|
+
Additionally, you can specify the parameter mappings
|
1165
|
+
to map event payload to Metaflow parameters for the flow.
|
1166
|
+
```
|
1167
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1168
|
+
```
|
1169
|
+
or
|
1170
|
+
```
|
1171
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1172
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1173
|
+
```
|
1174
|
+
|
1175
|
+
'parameters' can also be a list of strings and tuples like so:
|
1176
|
+
```
|
1177
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1178
|
+
```
|
1179
|
+
This is equivalent to:
|
1180
|
+
```
|
1181
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1182
|
+
```
|
1198
1183
|
|
1199
1184
|
|
1200
1185
|
Parameters
|
1201
1186
|
----------
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
|
1206
|
-
|
1207
|
-
|
1208
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1209
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1210
|
-
disabled : bool, default False
|
1211
|
-
If set to True, disables Conda.
|
1187
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1188
|
+
Event dependency for this flow.
|
1189
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1190
|
+
Events dependency for this flow.
|
1191
|
+
options : Dict[str, Any], default {}
|
1192
|
+
Backend-specific configuration for tuning eventing behavior.
|
1212
1193
|
"""
|
1213
1194
|
...
|
1214
1195
|
|
@@ -1306,82 +1287,6 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
|
|
1306
1287
|
"""
|
1307
1288
|
...
|
1308
1289
|
|
1309
|
-
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1310
|
-
"""
|
1311
|
-
Specifies what flows belong to the same project.
|
1312
|
-
|
1313
|
-
A project-specific namespace is created for all flows that
|
1314
|
-
use the same `@project(name)`.
|
1315
|
-
|
1316
|
-
|
1317
|
-
Parameters
|
1318
|
-
----------
|
1319
|
-
name : str
|
1320
|
-
Project name. Make sure that the name is unique amongst all
|
1321
|
-
projects that use the same production scheduler. The name may
|
1322
|
-
contain only lowercase alphanumeric characters and underscores.
|
1323
|
-
|
1324
|
-
branch : Optional[str], default None
|
1325
|
-
The branch to use. If not specified, the branch is set to
|
1326
|
-
`user.<username>` unless `production` is set to `True`. This can
|
1327
|
-
also be set on the command line using `--branch` as a top-level option.
|
1328
|
-
It is an error to specify `branch` in the decorator and on the command line.
|
1329
|
-
|
1330
|
-
production : bool, default False
|
1331
|
-
Whether or not the branch is the production branch. This can also be set on the
|
1332
|
-
command line using `--production` as a top-level option. It is an error to specify
|
1333
|
-
`production` in the decorator and on the command line.
|
1334
|
-
The project branch name will be:
|
1335
|
-
- if `branch` is specified:
|
1336
|
-
- if `production` is True: `prod.<branch>`
|
1337
|
-
- if `production` is False: `test.<branch>`
|
1338
|
-
- if `branch` is not specified:
|
1339
|
-
- if `production` is True: `prod`
|
1340
|
-
- if `production` is False: `user.<username>`
|
1341
|
-
"""
|
1342
|
-
...
|
1343
|
-
|
1344
|
-
@typing.overload
|
1345
|
-
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1346
|
-
"""
|
1347
|
-
Specifies the PyPI packages for all steps of the flow.
|
1348
|
-
|
1349
|
-
Use `@pypi_base` to set common packages required by all
|
1350
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1351
|
-
|
1352
|
-
Parameters
|
1353
|
-
----------
|
1354
|
-
packages : Dict[str, str], default: {}
|
1355
|
-
Packages to use for this flow. The key is the name of the package
|
1356
|
-
and the value is the version to use.
|
1357
|
-
python : str, optional, default: None
|
1358
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1359
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1360
|
-
"""
|
1361
|
-
...
|
1362
|
-
|
1363
|
-
@typing.overload
|
1364
|
-
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1365
|
-
...
|
1366
|
-
|
1367
|
-
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1368
|
-
"""
|
1369
|
-
Specifies the PyPI packages for all steps of the flow.
|
1370
|
-
|
1371
|
-
Use `@pypi_base` to set common packages required by all
|
1372
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1373
|
-
|
1374
|
-
Parameters
|
1375
|
-
----------
|
1376
|
-
packages : Dict[str, str], default: {}
|
1377
|
-
Packages to use for this flow. The key is the name of the package
|
1378
|
-
and the value is the version to use.
|
1379
|
-
python : str, optional, default: None
|
1380
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1381
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1382
|
-
"""
|
1383
|
-
...
|
1384
|
-
|
1385
1290
|
def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
|
1386
1291
|
"""
|
1387
1292
|
Allows setting external datastores to save data for the
|
@@ -1539,5 +1444,106 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
|
|
1539
1444
|
"""
|
1540
1445
|
...
|
1541
1446
|
|
1447
|
+
@typing.overload
|
1448
|
+
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1449
|
+
"""
|
1450
|
+
Specifies the flow(s) that this flow depends on.
|
1451
|
+
|
1452
|
+
```
|
1453
|
+
@trigger_on_finish(flow='FooFlow')
|
1454
|
+
```
|
1455
|
+
or
|
1456
|
+
```
|
1457
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1458
|
+
```
|
1459
|
+
This decorator respects the @project decorator and triggers the flow
|
1460
|
+
when upstream runs within the same namespace complete successfully
|
1461
|
+
|
1462
|
+
Additionally, you can specify project aware upstream flow dependencies
|
1463
|
+
by specifying the fully qualified project_flow_name.
|
1464
|
+
```
|
1465
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1466
|
+
```
|
1467
|
+
or
|
1468
|
+
```
|
1469
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1470
|
+
```
|
1471
|
+
|
1472
|
+
You can also specify just the project or project branch (other values will be
|
1473
|
+
inferred from the current project or project branch):
|
1474
|
+
```
|
1475
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1476
|
+
```
|
1477
|
+
|
1478
|
+
Note that `branch` is typically one of:
|
1479
|
+
- `prod`
|
1480
|
+
- `user.bob`
|
1481
|
+
- `test.my_experiment`
|
1482
|
+
- `prod.staging`
|
1483
|
+
|
1484
|
+
|
1485
|
+
Parameters
|
1486
|
+
----------
|
1487
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
1488
|
+
Upstream flow dependency for this flow.
|
1489
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
1490
|
+
Upstream flow dependencies for this flow.
|
1491
|
+
options : Dict[str, Any], default {}
|
1492
|
+
Backend-specific configuration for tuning eventing behavior.
|
1493
|
+
"""
|
1494
|
+
...
|
1495
|
+
|
1496
|
+
@typing.overload
|
1497
|
+
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1498
|
+
...
|
1499
|
+
|
1500
|
+
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1501
|
+
"""
|
1502
|
+
Specifies the flow(s) that this flow depends on.
|
1503
|
+
|
1504
|
+
```
|
1505
|
+
@trigger_on_finish(flow='FooFlow')
|
1506
|
+
```
|
1507
|
+
or
|
1508
|
+
```
|
1509
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1510
|
+
```
|
1511
|
+
This decorator respects the @project decorator and triggers the flow
|
1512
|
+
when upstream runs within the same namespace complete successfully
|
1513
|
+
|
1514
|
+
Additionally, you can specify project aware upstream flow dependencies
|
1515
|
+
by specifying the fully qualified project_flow_name.
|
1516
|
+
```
|
1517
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1518
|
+
```
|
1519
|
+
or
|
1520
|
+
```
|
1521
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1522
|
+
```
|
1523
|
+
|
1524
|
+
You can also specify just the project or project branch (other values will be
|
1525
|
+
inferred from the current project or project branch):
|
1526
|
+
```
|
1527
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1528
|
+
```
|
1529
|
+
|
1530
|
+
Note that `branch` is typically one of:
|
1531
|
+
- `prod`
|
1532
|
+
- `user.bob`
|
1533
|
+
- `test.my_experiment`
|
1534
|
+
- `prod.staging`
|
1535
|
+
|
1536
|
+
|
1537
|
+
Parameters
|
1538
|
+
----------
|
1539
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
1540
|
+
Upstream flow dependency for this flow.
|
1541
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
1542
|
+
Upstream flow dependencies for this flow.
|
1543
|
+
options : Dict[str, Any], default {}
|
1544
|
+
Backend-specific configuration for tuning eventing behavior.
|
1545
|
+
"""
|
1546
|
+
...
|
1547
|
+
|
1542
1548
|
pkg_name: str
|
1543
1549
|
|