ob-metaflow-stubs 6.0.3.188rc2__py2.py3-none-any.whl → 6.0.3.188rc4__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +876 -871
- metaflow-stubs/cards.pyi +1 -1
- metaflow-stubs/cli.pyi +1 -1
- metaflow-stubs/cli_components/__init__.pyi +1 -1
- metaflow-stubs/cli_components/utils.pyi +1 -1
- metaflow-stubs/client/__init__.pyi +1 -1
- metaflow-stubs/client/core.pyi +3 -3
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/events.pyi +2 -2
- metaflow-stubs/exception.pyi +1 -1
- metaflow-stubs/flowspec.pyi +4 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +2 -2
- metaflow-stubs/info_file.pyi +1 -1
- metaflow-stubs/metadata_provider/__init__.pyi +1 -1
- metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
- metaflow-stubs/metadata_provider/metadata.pyi +1 -1
- metaflow-stubs/metadata_provider/util.pyi +1 -1
- metaflow-stubs/metaflow_config.pyi +1 -1
- metaflow-stubs/metaflow_current.pyi +41 -28
- metaflow-stubs/metaflow_git.pyi +1 -1
- metaflow-stubs/mf_extensions/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +9 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +5 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +3 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +63 -44
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +52 -0
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +28 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +46 -0
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -1
- metaflow-stubs/multicore_utils.pyi +1 -1
- metaflow-stubs/ob_internal.pyi +2 -1
- metaflow-stubs/parameters.pyi +2 -2
- metaflow-stubs/plugins/__init__.pyi +8 -8
- metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
- metaflow-stubs/plugins/airflow/exception.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
- metaflow-stubs/plugins/argo/__init__.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
- metaflow-stubs/plugins/aws/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
- metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
- metaflow-stubs/plugins/azure/__init__.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
- metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
- metaflow-stubs/plugins/cards/__init__.pyi +5 -5
- metaflow-stubs/plugins/cards/card_client.pyi +2 -2
- metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
- metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
- metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
- metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
- metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
- metaflow-stubs/plugins/cards/exception.pyi +1 -1
- metaflow-stubs/plugins/catch_decorator.pyi +2 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
- metaflow-stubs/plugins/datatools/local.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
- metaflow-stubs/plugins/debug_logger.pyi +1 -1
- metaflow-stubs/plugins/debug_monitor.pyi +1 -1
- metaflow-stubs/plugins/environment_decorator.pyi +1 -1
- metaflow-stubs/plugins/events_decorator.pyi +1 -1
- metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
- metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
- metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
- metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
- metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
- metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
- metaflow-stubs/plugins/perimeters.pyi +1 -1
- metaflow-stubs/plugins/project_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
- metaflow-stubs/plugins/pypi/utils.pyi +1 -1
- metaflow-stubs/plugins/resources_decorator.pyi +1 -1
- metaflow-stubs/plugins/retry_decorator.pyi +1 -1
- metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
- metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
- metaflow-stubs/plugins/storage_executor.pyi +1 -1
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/plugins/torchtune/__init__.pyi +1 -1
- metaflow-stubs/plugins/uv/__init__.pyi +1 -1
- metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +1 -1
- metaflow-stubs/pylint_wrapper.pyi +1 -1
- metaflow-stubs/runner/__init__.pyi +1 -1
- metaflow-stubs/runner/deployer.pyi +4 -4
- metaflow-stubs/runner/deployer_impl.pyi +1 -1
- metaflow-stubs/runner/metaflow_runner.pyi +1 -1
- metaflow-stubs/runner/nbdeploy.pyi +1 -1
- metaflow-stubs/runner/nbrun.pyi +1 -1
- metaflow-stubs/runner/subprocess_manager.pyi +1 -1
- metaflow-stubs/runner/utils.pyi +3 -3
- metaflow-stubs/system/__init__.pyi +1 -1
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +1 -1
- metaflow-stubs/tagging_util.pyi +1 -1
- metaflow-stubs/tuple_util.pyi +1 -1
- metaflow-stubs/user_configs/__init__.pyi +1 -1
- metaflow-stubs/user_configs/config_decorators.pyi +4 -4
- metaflow-stubs/user_configs/config_options.pyi +1 -1
- metaflow-stubs/user_configs/config_parameters.pyi +5 -5
- {ob_metaflow_stubs-6.0.3.188rc2.dist-info → ob_metaflow_stubs-6.0.3.188rc4.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.188rc4.dist-info/RECORD +243 -0
- ob_metaflow_stubs-6.0.3.188rc2.dist-info/RECORD +0 -241
- {ob_metaflow_stubs-6.0.3.188rc2.dist-info → ob_metaflow_stubs-6.0.3.188rc4.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.188rc2.dist-info → ob_metaflow_stubs-6.0.3.188rc4.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
######################################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
3
|
# MF version: 2.15.18.1+obcheckpoint(0.2.4);ob(v1) #
|
4
|
-
# Generated on 2025-07-
|
4
|
+
# Generated on 2025-07-10T23:14:33.334494 #
|
5
5
|
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -37,16 +37,16 @@ from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDec
|
|
37
37
|
from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
|
38
38
|
from . import metaflow_git as metaflow_git
|
39
39
|
from . import cards as cards
|
40
|
-
from . import events as events
|
41
40
|
from . import tuple_util as tuple_util
|
41
|
+
from . import events as events
|
42
42
|
from . import runner as runner
|
43
43
|
from . import plugins as plugins
|
44
44
|
from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
|
45
45
|
from . import includefile as includefile
|
46
46
|
from .includefile import IncludeFile as IncludeFile
|
47
|
-
from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
|
48
47
|
from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
|
49
48
|
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
49
|
+
from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
|
50
50
|
from . import client as client
|
51
51
|
from .client.core import namespace as namespace
|
52
52
|
from .client.core import get_namespace as get_namespace
|
@@ -73,6 +73,7 @@ from .mf_extensions.outerbounds.plugins.snowflake.snowflake import Snowflake as
|
|
73
73
|
from .mf_extensions.outerbounds.plugins.checkpoint_datastores.nebius import nebius_checkpoints as nebius_checkpoints
|
74
74
|
from .mf_extensions.outerbounds.plugins.checkpoint_datastores.coreweave import coreweave_checkpoints as coreweave_checkpoints
|
75
75
|
from .mf_extensions.outerbounds.plugins.aws.assume_role_decorator import assume_role as assume_role
|
76
|
+
from .mf_extensions.outerbounds.plugins.apps.core.deployer import AppDeployer as AppDeployer
|
76
77
|
from . import cli_components as cli_components
|
77
78
|
from . import system as system
|
78
79
|
from . import pylint_wrapper as pylint_wrapper
|
@@ -155,6 +156,154 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
155
156
|
"""
|
156
157
|
...
|
157
158
|
|
159
|
+
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
160
|
+
"""
|
161
|
+
Specifies that this step should execute on Kubernetes.
|
162
|
+
|
163
|
+
|
164
|
+
Parameters
|
165
|
+
----------
|
166
|
+
cpu : int, default 1
|
167
|
+
Number of CPUs required for this step. If `@resources` is
|
168
|
+
also present, the maximum value from all decorators is used.
|
169
|
+
memory : int, default 4096
|
170
|
+
Memory size (in MB) required for this step. If
|
171
|
+
`@resources` is also present, the maximum value from all decorators is
|
172
|
+
used.
|
173
|
+
disk : int, default 10240
|
174
|
+
Disk size (in MB) required for this step. If
|
175
|
+
`@resources` is also present, the maximum value from all decorators is
|
176
|
+
used.
|
177
|
+
image : str, optional, default None
|
178
|
+
Docker image to use when launching on Kubernetes. If not specified, and
|
179
|
+
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
180
|
+
not, a default Docker image mapping to the current version of Python is used.
|
181
|
+
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
182
|
+
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
183
|
+
image_pull_secrets: List[str], default []
|
184
|
+
The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
|
185
|
+
Kubernetes image pull secrets to use when pulling container images
|
186
|
+
in Kubernetes.
|
187
|
+
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
188
|
+
Kubernetes service account to use when launching pod in Kubernetes.
|
189
|
+
secrets : List[str], optional, default None
|
190
|
+
Kubernetes secrets to use when launching pod in Kubernetes. These
|
191
|
+
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
192
|
+
in Metaflow configuration.
|
193
|
+
node_selector: Union[Dict[str,str], str], optional, default None
|
194
|
+
Kubernetes node selector(s) to apply to the pod running the task.
|
195
|
+
Can be passed in as a comma separated string of values e.g.
|
196
|
+
'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
|
197
|
+
{'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
|
198
|
+
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
199
|
+
Kubernetes namespace to use when launching pod in Kubernetes.
|
200
|
+
gpu : int, optional, default None
|
201
|
+
Number of GPUs required for this step. A value of zero implies that
|
202
|
+
the scheduled node should not have GPUs.
|
203
|
+
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
204
|
+
The vendor of the GPUs to be used for this step.
|
205
|
+
tolerations : List[str], default []
|
206
|
+
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
207
|
+
Kubernetes tolerations to use when launching pod in Kubernetes.
|
208
|
+
labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
|
209
|
+
Kubernetes labels to use when launching pod in Kubernetes.
|
210
|
+
annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
|
211
|
+
Kubernetes annotations to use when launching pod in Kubernetes.
|
212
|
+
use_tmpfs : bool, default False
|
213
|
+
This enables an explicit tmpfs mount for this step.
|
214
|
+
tmpfs_tempdir : bool, default True
|
215
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
216
|
+
tmpfs_size : int, optional, default: None
|
217
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
218
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
219
|
+
memory allocated for this step.
|
220
|
+
tmpfs_path : str, optional, default /metaflow_temp
|
221
|
+
Path to tmpfs mount for this step.
|
222
|
+
persistent_volume_claims : Dict[str, str], optional, default None
|
223
|
+
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
224
|
+
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
225
|
+
shared_memory: int, optional
|
226
|
+
Shared memory size (in MiB) required for this step
|
227
|
+
port: int, optional
|
228
|
+
Port number to specify in the Kubernetes job object
|
229
|
+
compute_pool : str, optional, default None
|
230
|
+
Compute pool to be used for for this step.
|
231
|
+
If not specified, any accessible compute pool within the perimeter is used.
|
232
|
+
hostname_resolution_timeout: int, default 10 * 60
|
233
|
+
Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
|
234
|
+
Only applicable when @parallel is used.
|
235
|
+
qos: str, default: Burstable
|
236
|
+
Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
|
237
|
+
|
238
|
+
security_context: Dict[str, Any], optional, default None
|
239
|
+
Container security context. Applies to the task container. Allows the following keys:
|
240
|
+
- privileged: bool, optional, default None
|
241
|
+
- allow_privilege_escalation: bool, optional, default None
|
242
|
+
- run_as_user: int, optional, default None
|
243
|
+
- run_as_group: int, optional, default None
|
244
|
+
- run_as_non_root: bool, optional, default None
|
245
|
+
"""
|
246
|
+
...
|
247
|
+
|
248
|
+
@typing.overload
|
249
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
250
|
+
"""
|
251
|
+
Specifies the Conda environment for the step.
|
252
|
+
|
253
|
+
Information in this decorator will augment any
|
254
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
255
|
+
you can use `@conda_base` to set packages required by all
|
256
|
+
steps and use `@conda` to specify step-specific overrides.
|
257
|
+
|
258
|
+
|
259
|
+
Parameters
|
260
|
+
----------
|
261
|
+
packages : Dict[str, str], default {}
|
262
|
+
Packages to use for this step. The key is the name of the package
|
263
|
+
and the value is the version to use.
|
264
|
+
libraries : Dict[str, str], default {}
|
265
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
266
|
+
python : str, optional, default None
|
267
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
268
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
269
|
+
disabled : bool, default False
|
270
|
+
If set to True, disables @conda.
|
271
|
+
"""
|
272
|
+
...
|
273
|
+
|
274
|
+
@typing.overload
|
275
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
276
|
+
...
|
277
|
+
|
278
|
+
@typing.overload
|
279
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
280
|
+
...
|
281
|
+
|
282
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
283
|
+
"""
|
284
|
+
Specifies the Conda environment for the step.
|
285
|
+
|
286
|
+
Information in this decorator will augment any
|
287
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
288
|
+
you can use `@conda_base` to set packages required by all
|
289
|
+
steps and use `@conda` to specify step-specific overrides.
|
290
|
+
|
291
|
+
|
292
|
+
Parameters
|
293
|
+
----------
|
294
|
+
packages : Dict[str, str], default {}
|
295
|
+
Packages to use for this step. The key is the name of the package
|
296
|
+
and the value is the version to use.
|
297
|
+
libraries : Dict[str, str], default {}
|
298
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
299
|
+
python : str, optional, default None
|
300
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
301
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
302
|
+
disabled : bool, default False
|
303
|
+
If set to True, disables @conda.
|
304
|
+
"""
|
305
|
+
...
|
306
|
+
|
158
307
|
@typing.overload
|
159
308
|
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
160
309
|
"""
|
@@ -234,186 +383,175 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
234
383
|
"""
|
235
384
|
...
|
236
385
|
|
237
|
-
def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
238
|
-
"""
|
239
|
-
Specifies that this step should execute on DGX cloud.
|
240
|
-
|
241
|
-
|
242
|
-
Parameters
|
243
|
-
----------
|
244
|
-
gpu : int
|
245
|
-
Number of GPUs to use.
|
246
|
-
gpu_type : str
|
247
|
-
Type of Nvidia GPU to use.
|
248
|
-
queue_timeout : int
|
249
|
-
Time to keep the job in NVCF's queue.
|
250
|
-
"""
|
251
|
-
...
|
252
|
-
|
253
386
|
@typing.overload
|
254
|
-
def
|
387
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
255
388
|
"""
|
256
|
-
|
257
|
-
|
258
|
-
This decorator is useful if this step may hang indefinitely.
|
259
|
-
|
260
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
261
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
262
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
389
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
263
390
|
|
264
|
-
Note that
|
265
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
391
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
266
392
|
|
267
393
|
|
268
394
|
Parameters
|
269
395
|
----------
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
396
|
+
type : str, default 'default'
|
397
|
+
Card type.
|
398
|
+
id : str, optional, default None
|
399
|
+
If multiple cards are present, use this id to identify this card.
|
400
|
+
options : Dict[str, Any], default {}
|
401
|
+
Options passed to the card. The contents depend on the card type.
|
402
|
+
timeout : int, default 45
|
403
|
+
Interrupt reporting if it takes more than this many seconds.
|
276
404
|
"""
|
277
405
|
...
|
278
406
|
|
279
407
|
@typing.overload
|
280
|
-
def
|
408
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
281
409
|
...
|
282
410
|
|
283
411
|
@typing.overload
|
284
|
-
def
|
412
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
285
413
|
...
|
286
414
|
|
287
|
-
def
|
415
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
288
416
|
"""
|
289
|
-
|
290
|
-
|
291
|
-
This decorator is useful if this step may hang indefinitely.
|
292
|
-
|
293
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
294
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
295
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
417
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
296
418
|
|
297
|
-
Note that
|
298
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
419
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
299
420
|
|
300
421
|
|
301
422
|
Parameters
|
302
423
|
----------
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
424
|
+
type : str, default 'default'
|
425
|
+
Card type.
|
426
|
+
id : str, optional, default None
|
427
|
+
If multiple cards are present, use this id to identify this card.
|
428
|
+
options : Dict[str, Any], default {}
|
429
|
+
Options passed to the card. The contents depend on the card type.
|
430
|
+
timeout : int, default 45
|
431
|
+
Interrupt reporting if it takes more than this many seconds.
|
309
432
|
"""
|
310
433
|
...
|
311
434
|
|
312
435
|
@typing.overload
|
313
|
-
def
|
436
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
314
437
|
"""
|
315
|
-
Specifies the
|
438
|
+
Specifies that the step will success under all circumstances.
|
316
439
|
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
440
|
+
The decorator will create an optional artifact, specified by `var`, which
|
441
|
+
contains the exception raised. You can use it to detect the presence
|
442
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
443
|
+
are missing.
|
321
444
|
|
322
445
|
|
323
446
|
Parameters
|
324
447
|
----------
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
448
|
+
var : str, optional, default None
|
449
|
+
Name of the artifact in which to store the caught exception.
|
450
|
+
If not specified, the exception is not stored.
|
451
|
+
print_exception : bool, default True
|
452
|
+
Determines whether or not the exception is printed to
|
453
|
+
stdout when caught.
|
331
454
|
"""
|
332
455
|
...
|
333
456
|
|
334
457
|
@typing.overload
|
335
|
-
def
|
458
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
336
459
|
...
|
337
460
|
|
338
461
|
@typing.overload
|
339
|
-
def
|
462
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
340
463
|
...
|
341
464
|
|
342
|
-
def
|
465
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
343
466
|
"""
|
344
|
-
Specifies the
|
467
|
+
Specifies that the step will success under all circumstances.
|
345
468
|
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
469
|
+
The decorator will create an optional artifact, specified by `var`, which
|
470
|
+
contains the exception raised. You can use it to detect the presence
|
471
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
472
|
+
are missing.
|
350
473
|
|
351
474
|
|
352
475
|
Parameters
|
353
476
|
----------
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
477
|
+
var : str, optional, default None
|
478
|
+
Name of the artifact in which to store the caught exception.
|
479
|
+
If not specified, the exception is not stored.
|
480
|
+
print_exception : bool, default True
|
481
|
+
Determines whether or not the exception is printed to
|
482
|
+
stdout when caught.
|
360
483
|
"""
|
361
484
|
...
|
362
485
|
|
363
|
-
|
486
|
+
@typing.overload
|
487
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
364
488
|
"""
|
365
|
-
|
489
|
+
Internal decorator to support Fast bakery
|
490
|
+
"""
|
491
|
+
...
|
492
|
+
|
493
|
+
@typing.overload
|
494
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
495
|
+
...
|
496
|
+
|
497
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
498
|
+
"""
|
499
|
+
Internal decorator to support Fast bakery
|
500
|
+
"""
|
501
|
+
...
|
502
|
+
|
503
|
+
@typing.overload
|
504
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
505
|
+
"""
|
506
|
+
Specifies the number of times the task corresponding
|
507
|
+
to a step needs to be retried.
|
508
|
+
|
509
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
510
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
511
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
512
|
+
|
513
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
514
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
515
|
+
ensuring that the flow execution can continue.
|
366
516
|
|
367
517
|
|
368
518
|
Parameters
|
369
519
|
----------
|
370
|
-
|
371
|
-
Number of
|
372
|
-
|
373
|
-
|
520
|
+
times : int, default 3
|
521
|
+
Number of times to retry this task.
|
522
|
+
minutes_between_retries : int, default 2
|
523
|
+
Number of minutes between retries.
|
374
524
|
"""
|
375
525
|
...
|
376
526
|
|
377
|
-
|
527
|
+
@typing.overload
|
528
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
529
|
+
...
|
530
|
+
|
531
|
+
@typing.overload
|
532
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
533
|
+
...
|
534
|
+
|
535
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
378
536
|
"""
|
379
|
-
|
380
|
-
|
381
|
-
User code call
|
382
|
-
--------------
|
383
|
-
@ollama(
|
384
|
-
models=[...],
|
385
|
-
...
|
386
|
-
)
|
537
|
+
Specifies the number of times the task corresponding
|
538
|
+
to a step needs to be retried.
|
387
539
|
|
388
|
-
|
389
|
-
|
390
|
-
|
391
|
-
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
392
|
-
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
540
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
541
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
542
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
393
543
|
|
394
|
-
|
395
|
-
|
396
|
-
|
544
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
545
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
546
|
+
ensuring that the flow execution can continue.
|
397
547
|
|
398
548
|
|
399
549
|
Parameters
|
400
550
|
----------
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
force_pull: bool
|
406
|
-
Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
|
407
|
-
cache_update_policy: str
|
408
|
-
Cache update policy: "auto", "force", or "never".
|
409
|
-
force_cache_update: bool
|
410
|
-
Simple override for "force" cache update policy.
|
411
|
-
debug: bool
|
412
|
-
Whether to turn on verbose debugging logs.
|
413
|
-
circuit_breaker_config: dict
|
414
|
-
Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
|
415
|
-
timeout_config: dict
|
416
|
-
Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
|
551
|
+
times : int, default 3
|
552
|
+
Number of times to retry this task.
|
553
|
+
minutes_between_retries : int, default 2
|
554
|
+
Number of minutes between retries.
|
417
555
|
"""
|
418
556
|
...
|
419
557
|
|
@@ -467,116 +605,6 @@ def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card
|
|
467
605
|
"""
|
468
606
|
...
|
469
607
|
|
470
|
-
@typing.overload
|
471
|
-
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
472
|
-
"""
|
473
|
-
Specifies the Conda environment for the step.
|
474
|
-
|
475
|
-
Information in this decorator will augment any
|
476
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
477
|
-
you can use `@conda_base` to set packages required by all
|
478
|
-
steps and use `@conda` to specify step-specific overrides.
|
479
|
-
|
480
|
-
|
481
|
-
Parameters
|
482
|
-
----------
|
483
|
-
packages : Dict[str, str], default {}
|
484
|
-
Packages to use for this step. The key is the name of the package
|
485
|
-
and the value is the version to use.
|
486
|
-
libraries : Dict[str, str], default {}
|
487
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
488
|
-
python : str, optional, default None
|
489
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
490
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
491
|
-
disabled : bool, default False
|
492
|
-
If set to True, disables @conda.
|
493
|
-
"""
|
494
|
-
...
|
495
|
-
|
496
|
-
@typing.overload
|
497
|
-
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
498
|
-
...
|
499
|
-
|
500
|
-
@typing.overload
|
501
|
-
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
502
|
-
...
|
503
|
-
|
504
|
-
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
505
|
-
"""
|
506
|
-
Specifies the Conda environment for the step.
|
507
|
-
|
508
|
-
Information in this decorator will augment any
|
509
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
510
|
-
you can use `@conda_base` to set packages required by all
|
511
|
-
steps and use `@conda` to specify step-specific overrides.
|
512
|
-
|
513
|
-
|
514
|
-
Parameters
|
515
|
-
----------
|
516
|
-
packages : Dict[str, str], default {}
|
517
|
-
Packages to use for this step. The key is the name of the package
|
518
|
-
and the value is the version to use.
|
519
|
-
libraries : Dict[str, str], default {}
|
520
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
521
|
-
python : str, optional, default None
|
522
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
523
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
524
|
-
disabled : bool, default False
|
525
|
-
If set to True, disables @conda.
|
526
|
-
"""
|
527
|
-
...
|
528
|
-
|
529
|
-
@typing.overload
|
530
|
-
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
531
|
-
"""
|
532
|
-
Specifies that the step will success under all circumstances.
|
533
|
-
|
534
|
-
The decorator will create an optional artifact, specified by `var`, which
|
535
|
-
contains the exception raised. You can use it to detect the presence
|
536
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
537
|
-
are missing.
|
538
|
-
|
539
|
-
|
540
|
-
Parameters
|
541
|
-
----------
|
542
|
-
var : str, optional, default None
|
543
|
-
Name of the artifact in which to store the caught exception.
|
544
|
-
If not specified, the exception is not stored.
|
545
|
-
print_exception : bool, default True
|
546
|
-
Determines whether or not the exception is printed to
|
547
|
-
stdout when caught.
|
548
|
-
"""
|
549
|
-
...
|
550
|
-
|
551
|
-
@typing.overload
|
552
|
-
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
553
|
-
...
|
554
|
-
|
555
|
-
@typing.overload
|
556
|
-
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
557
|
-
...
|
558
|
-
|
559
|
-
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
560
|
-
"""
|
561
|
-
Specifies that the step will success under all circumstances.
|
562
|
-
|
563
|
-
The decorator will create an optional artifact, specified by `var`, which
|
564
|
-
contains the exception raised. You can use it to detect the presence
|
565
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
566
|
-
are missing.
|
567
|
-
|
568
|
-
|
569
|
-
Parameters
|
570
|
-
----------
|
571
|
-
var : str, optional, default None
|
572
|
-
Name of the artifact in which to store the caught exception.
|
573
|
-
If not specified, the exception is not stored.
|
574
|
-
print_exception : bool, default True
|
575
|
-
Determines whether or not the exception is printed to
|
576
|
-
stdout when caught.
|
577
|
-
"""
|
578
|
-
...
|
579
|
-
|
580
608
|
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
581
609
|
"""
|
582
610
|
Decorator that helps cache, version and store models/datasets from huggingface hub.
|
@@ -658,267 +686,16 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.
|
|
658
686
|
...
|
659
687
|
|
660
688
|
@typing.overload
|
661
|
-
def
|
689
|
+
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
662
690
|
"""
|
663
|
-
Enables
|
691
|
+
Enables checkpointing for a step.
|
664
692
|
|
665
693
|
> Examples
|
666
|
-
- Saving Models
|
667
|
-
```python
|
668
|
-
@model
|
669
|
-
@step
|
670
|
-
def train(self):
|
671
|
-
# current.model.save returns a dictionary reference to the model saved
|
672
|
-
self.my_model = current.model.save(
|
673
|
-
path_to_my_model,
|
674
|
-
label="my_model",
|
675
|
-
metadata={
|
676
|
-
"epochs": 10,
|
677
|
-
"batch-size": 32,
|
678
|
-
"learning-rate": 0.001,
|
679
|
-
}
|
680
|
-
)
|
681
|
-
self.next(self.test)
|
682
694
|
|
683
|
-
|
684
|
-
@step
|
685
|
-
def test(self):
|
686
|
-
# `current.model.loaded` returns a dictionary of the loaded models
|
687
|
-
# where the key is the name of the artifact and the value is the path to the model
|
688
|
-
print(os.listdir(current.model.loaded["my_model"]))
|
689
|
-
self.next(self.end)
|
690
|
-
```
|
695
|
+
- Saving Checkpoints
|
691
696
|
|
692
|
-
- Loading models
|
693
697
|
```python
|
694
|
-
@
|
695
|
-
def train(self):
|
696
|
-
# current.model.load returns the path to the model loaded
|
697
|
-
checkpoint_path = current.model.load(
|
698
|
-
self.checkpoint_key,
|
699
|
-
)
|
700
|
-
model_path = current.model.load(
|
701
|
-
self.model,
|
702
|
-
)
|
703
|
-
self.next(self.test)
|
704
|
-
```
|
705
|
-
|
706
|
-
|
707
|
-
Parameters
|
708
|
-
----------
|
709
|
-
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
710
|
-
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
711
|
-
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
712
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
713
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
714
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
715
|
-
|
716
|
-
temp_dir_root : str, default: None
|
717
|
-
The root directory under which `current.model.loaded` will store loaded models
|
718
|
-
"""
|
719
|
-
...
|
720
|
-
|
721
|
-
@typing.overload
|
722
|
-
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
723
|
-
...
|
724
|
-
|
725
|
-
@typing.overload
|
726
|
-
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
727
|
-
...
|
728
|
-
|
729
|
-
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
730
|
-
"""
|
731
|
-
Enables loading / saving of models within a step.
|
732
|
-
|
733
|
-
> Examples
|
734
|
-
- Saving Models
|
735
|
-
```python
|
736
|
-
@model
|
737
|
-
@step
|
738
|
-
def train(self):
|
739
|
-
# current.model.save returns a dictionary reference to the model saved
|
740
|
-
self.my_model = current.model.save(
|
741
|
-
path_to_my_model,
|
742
|
-
label="my_model",
|
743
|
-
metadata={
|
744
|
-
"epochs": 10,
|
745
|
-
"batch-size": 32,
|
746
|
-
"learning-rate": 0.001,
|
747
|
-
}
|
748
|
-
)
|
749
|
-
self.next(self.test)
|
750
|
-
|
751
|
-
@model(load="my_model")
|
752
|
-
@step
|
753
|
-
def test(self):
|
754
|
-
# `current.model.loaded` returns a dictionary of the loaded models
|
755
|
-
# where the key is the name of the artifact and the value is the path to the model
|
756
|
-
print(os.listdir(current.model.loaded["my_model"]))
|
757
|
-
self.next(self.end)
|
758
|
-
```
|
759
|
-
|
760
|
-
- Loading models
|
761
|
-
```python
|
762
|
-
@step
|
763
|
-
def train(self):
|
764
|
-
# current.model.load returns the path to the model loaded
|
765
|
-
checkpoint_path = current.model.load(
|
766
|
-
self.checkpoint_key,
|
767
|
-
)
|
768
|
-
model_path = current.model.load(
|
769
|
-
self.model,
|
770
|
-
)
|
771
|
-
self.next(self.test)
|
772
|
-
```
|
773
|
-
|
774
|
-
|
775
|
-
Parameters
|
776
|
-
----------
|
777
|
-
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
778
|
-
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
779
|
-
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
780
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
781
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
782
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
783
|
-
|
784
|
-
temp_dir_root : str, default: None
|
785
|
-
The root directory under which `current.model.loaded` will store loaded models
|
786
|
-
"""
|
787
|
-
...
|
788
|
-
|
789
|
-
@typing.overload
|
790
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
791
|
-
"""
|
792
|
-
Specifies environment variables to be set prior to the execution of a step.
|
793
|
-
|
794
|
-
|
795
|
-
Parameters
|
796
|
-
----------
|
797
|
-
vars : Dict[str, str], default {}
|
798
|
-
Dictionary of environment variables to set.
|
799
|
-
"""
|
800
|
-
...
|
801
|
-
|
802
|
-
@typing.overload
|
803
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
804
|
-
...
|
805
|
-
|
806
|
-
@typing.overload
|
807
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
808
|
-
...
|
809
|
-
|
810
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
811
|
-
"""
|
812
|
-
Specifies environment variables to be set prior to the execution of a step.
|
813
|
-
|
814
|
-
|
815
|
-
Parameters
|
816
|
-
----------
|
817
|
-
vars : Dict[str, str], default {}
|
818
|
-
Dictionary of environment variables to set.
|
819
|
-
"""
|
820
|
-
...
|
821
|
-
|
822
|
-
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
823
|
-
"""
|
824
|
-
Specifies that this step should execute on Kubernetes.
|
825
|
-
|
826
|
-
|
827
|
-
Parameters
|
828
|
-
----------
|
829
|
-
cpu : int, default 1
|
830
|
-
Number of CPUs required for this step. If `@resources` is
|
831
|
-
also present, the maximum value from all decorators is used.
|
832
|
-
memory : int, default 4096
|
833
|
-
Memory size (in MB) required for this step. If
|
834
|
-
`@resources` is also present, the maximum value from all decorators is
|
835
|
-
used.
|
836
|
-
disk : int, default 10240
|
837
|
-
Disk size (in MB) required for this step. If
|
838
|
-
`@resources` is also present, the maximum value from all decorators is
|
839
|
-
used.
|
840
|
-
image : str, optional, default None
|
841
|
-
Docker image to use when launching on Kubernetes. If not specified, and
|
842
|
-
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
843
|
-
not, a default Docker image mapping to the current version of Python is used.
|
844
|
-
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
845
|
-
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
846
|
-
image_pull_secrets: List[str], default []
|
847
|
-
The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
|
848
|
-
Kubernetes image pull secrets to use when pulling container images
|
849
|
-
in Kubernetes.
|
850
|
-
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
851
|
-
Kubernetes service account to use when launching pod in Kubernetes.
|
852
|
-
secrets : List[str], optional, default None
|
853
|
-
Kubernetes secrets to use when launching pod in Kubernetes. These
|
854
|
-
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
855
|
-
in Metaflow configuration.
|
856
|
-
node_selector: Union[Dict[str,str], str], optional, default None
|
857
|
-
Kubernetes node selector(s) to apply to the pod running the task.
|
858
|
-
Can be passed in as a comma separated string of values e.g.
|
859
|
-
'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
|
860
|
-
{'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
|
861
|
-
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
862
|
-
Kubernetes namespace to use when launching pod in Kubernetes.
|
863
|
-
gpu : int, optional, default None
|
864
|
-
Number of GPUs required for this step. A value of zero implies that
|
865
|
-
the scheduled node should not have GPUs.
|
866
|
-
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
867
|
-
The vendor of the GPUs to be used for this step.
|
868
|
-
tolerations : List[str], default []
|
869
|
-
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
870
|
-
Kubernetes tolerations to use when launching pod in Kubernetes.
|
871
|
-
labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
|
872
|
-
Kubernetes labels to use when launching pod in Kubernetes.
|
873
|
-
annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
|
874
|
-
Kubernetes annotations to use when launching pod in Kubernetes.
|
875
|
-
use_tmpfs : bool, default False
|
876
|
-
This enables an explicit tmpfs mount for this step.
|
877
|
-
tmpfs_tempdir : bool, default True
|
878
|
-
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
879
|
-
tmpfs_size : int, optional, default: None
|
880
|
-
The value for the size (in MiB) of the tmpfs mount for this step.
|
881
|
-
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
882
|
-
memory allocated for this step.
|
883
|
-
tmpfs_path : str, optional, default /metaflow_temp
|
884
|
-
Path to tmpfs mount for this step.
|
885
|
-
persistent_volume_claims : Dict[str, str], optional, default None
|
886
|
-
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
887
|
-
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
888
|
-
shared_memory: int, optional
|
889
|
-
Shared memory size (in MiB) required for this step
|
890
|
-
port: int, optional
|
891
|
-
Port number to specify in the Kubernetes job object
|
892
|
-
compute_pool : str, optional, default None
|
893
|
-
Compute pool to be used for for this step.
|
894
|
-
If not specified, any accessible compute pool within the perimeter is used.
|
895
|
-
hostname_resolution_timeout: int, default 10 * 60
|
896
|
-
Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
|
897
|
-
Only applicable when @parallel is used.
|
898
|
-
qos: str, default: Burstable
|
899
|
-
Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
|
900
|
-
|
901
|
-
security_context: Dict[str, Any], optional, default None
|
902
|
-
Container security context. Applies to the task container. Allows the following keys:
|
903
|
-
- privileged: bool, optional, default None
|
904
|
-
- allow_privilege_escalation: bool, optional, default None
|
905
|
-
- run_as_user: int, optional, default None
|
906
|
-
- run_as_group: int, optional, default None
|
907
|
-
- run_as_non_root: bool, optional, default None
|
908
|
-
"""
|
909
|
-
...
|
910
|
-
|
911
|
-
@typing.overload
|
912
|
-
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
913
|
-
"""
|
914
|
-
Enables checkpointing for a step.
|
915
|
-
|
916
|
-
> Examples
|
917
|
-
|
918
|
-
- Saving Checkpoints
|
919
|
-
|
920
|
-
```python
|
921
|
-
@checkpoint
|
698
|
+
@checkpoint
|
922
699
|
@step
|
923
700
|
def train(self):
|
924
701
|
model = create_model(self.parameters, checkpoint_path = None)
|
@@ -1055,125 +832,97 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
|
|
1055
832
|
"""
|
1056
833
|
...
|
1057
834
|
|
1058
|
-
|
1059
|
-
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
835
|
+
def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1060
836
|
"""
|
1061
|
-
|
1062
|
-
to a step needs to be retried.
|
837
|
+
This decorator is used to run Ollama APIs as Metaflow task sidecars.
|
1063
838
|
|
1064
|
-
|
1065
|
-
|
1066
|
-
|
839
|
+
User code call
|
840
|
+
--------------
|
841
|
+
@ollama(
|
842
|
+
models=[...],
|
843
|
+
...
|
844
|
+
)
|
1067
845
|
|
1068
|
-
|
1069
|
-
|
1070
|
-
|
846
|
+
Valid backend options
|
847
|
+
---------------------
|
848
|
+
- 'local': Run as a separate process on the local task machine.
|
849
|
+
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
850
|
+
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
851
|
+
|
852
|
+
Valid model options
|
853
|
+
-------------------
|
854
|
+
Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
|
1071
855
|
|
1072
856
|
|
1073
857
|
Parameters
|
1074
858
|
----------
|
1075
|
-
|
1076
|
-
|
1077
|
-
|
1078
|
-
|
859
|
+
models: list[str]
|
860
|
+
List of Ollama containers running models in sidecars.
|
861
|
+
backend: str
|
862
|
+
Determines where and how to run the Ollama process.
|
863
|
+
force_pull: bool
|
864
|
+
Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
|
865
|
+
cache_update_policy: str
|
866
|
+
Cache update policy: "auto", "force", or "never".
|
867
|
+
force_cache_update: bool
|
868
|
+
Simple override for "force" cache update policy.
|
869
|
+
debug: bool
|
870
|
+
Whether to turn on verbose debugging logs.
|
871
|
+
circuit_breaker_config: dict
|
872
|
+
Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
|
873
|
+
timeout_config: dict
|
874
|
+
Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
|
1079
875
|
"""
|
1080
876
|
...
|
1081
877
|
|
1082
878
|
@typing.overload
|
1083
|
-
def
|
1084
|
-
...
|
1085
|
-
|
1086
|
-
@typing.overload
|
1087
|
-
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1088
|
-
...
|
1089
|
-
|
1090
|
-
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
879
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1091
880
|
"""
|
1092
|
-
Specifies the
|
1093
|
-
to a step needs to be retried.
|
1094
|
-
|
1095
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
1096
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
1097
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
1098
|
-
|
1099
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
1100
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
1101
|
-
ensuring that the flow execution can continue.
|
1102
|
-
|
881
|
+
Specifies the PyPI packages for the step.
|
1103
882
|
|
1104
|
-
|
1105
|
-
|
1106
|
-
|
1107
|
-
|
1108
|
-
minutes_between_retries : int, default 2
|
1109
|
-
Number of minutes between retries.
|
1110
|
-
"""
|
1111
|
-
...
|
1112
|
-
|
1113
|
-
@typing.overload
|
1114
|
-
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1115
|
-
"""
|
1116
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
1117
|
-
the execution of a step.
|
883
|
+
Information in this decorator will augment any
|
884
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
885
|
+
you can use `@pypi_base` to set packages required by all
|
886
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1118
887
|
|
1119
888
|
|
1120
889
|
Parameters
|
1121
890
|
----------
|
1122
|
-
|
1123
|
-
|
891
|
+
packages : Dict[str, str], default: {}
|
892
|
+
Packages to use for this step. The key is the name of the package
|
893
|
+
and the value is the version to use.
|
894
|
+
python : str, optional, default: None
|
895
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
896
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1124
897
|
"""
|
1125
898
|
...
|
1126
899
|
|
1127
900
|
@typing.overload
|
1128
|
-
def
|
901
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1129
902
|
...
|
1130
903
|
|
1131
904
|
@typing.overload
|
1132
|
-
def
|
905
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1133
906
|
...
|
1134
907
|
|
1135
|
-
def
|
908
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1136
909
|
"""
|
1137
|
-
Specifies
|
1138
|
-
the execution of a step.
|
1139
|
-
|
910
|
+
Specifies the PyPI packages for the step.
|
1140
911
|
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1145
|
-
"""
|
1146
|
-
...
|
1147
|
-
|
1148
|
-
@typing.overload
|
1149
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1150
|
-
"""
|
1151
|
-
Internal decorator to support Fast bakery
|
1152
|
-
"""
|
1153
|
-
...
|
1154
|
-
|
1155
|
-
@typing.overload
|
1156
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1157
|
-
...
|
1158
|
-
|
1159
|
-
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1160
|
-
"""
|
1161
|
-
Internal decorator to support Fast bakery
|
1162
|
-
"""
|
1163
|
-
...
|
1164
|
-
|
1165
|
-
def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1166
|
-
"""
|
1167
|
-
Specifies that this step is used to deploy an instance of the app.
|
1168
|
-
Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
|
912
|
+
Information in this decorator will augment any
|
913
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
914
|
+
you can use `@pypi_base` to set packages required by all
|
915
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1169
916
|
|
1170
917
|
|
1171
918
|
Parameters
|
1172
919
|
----------
|
1173
|
-
|
1174
|
-
|
1175
|
-
|
1176
|
-
|
920
|
+
packages : Dict[str, str], default: {}
|
921
|
+
Packages to use for this step. The key is the name of the package
|
922
|
+
and the value is the version to use.
|
923
|
+
python : str, optional, default: None
|
924
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
925
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1177
926
|
"""
|
1178
927
|
...
|
1179
928
|
|
@@ -1197,92 +946,307 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
1197
946
|
...
|
1198
947
|
|
1199
948
|
@typing.overload
|
1200
|
-
def
|
949
|
+
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1201
950
|
"""
|
1202
|
-
|
951
|
+
Enables loading / saving of models within a step.
|
1203
952
|
|
1204
|
-
|
953
|
+
> Examples
|
954
|
+
- Saving Models
|
955
|
+
```python
|
956
|
+
@model
|
957
|
+
@step
|
958
|
+
def train(self):
|
959
|
+
# current.model.save returns a dictionary reference to the model saved
|
960
|
+
self.my_model = current.model.save(
|
961
|
+
path_to_my_model,
|
962
|
+
label="my_model",
|
963
|
+
metadata={
|
964
|
+
"epochs": 10,
|
965
|
+
"batch-size": 32,
|
966
|
+
"learning-rate": 0.001,
|
967
|
+
}
|
968
|
+
)
|
969
|
+
self.next(self.test)
|
970
|
+
|
971
|
+
@model(load="my_model")
|
972
|
+
@step
|
973
|
+
def test(self):
|
974
|
+
# `current.model.loaded` returns a dictionary of the loaded models
|
975
|
+
# where the key is the name of the artifact and the value is the path to the model
|
976
|
+
print(os.listdir(current.model.loaded["my_model"]))
|
977
|
+
self.next(self.end)
|
978
|
+
```
|
979
|
+
|
980
|
+
- Loading models
|
981
|
+
```python
|
982
|
+
@step
|
983
|
+
def train(self):
|
984
|
+
# current.model.load returns the path to the model loaded
|
985
|
+
checkpoint_path = current.model.load(
|
986
|
+
self.checkpoint_key,
|
987
|
+
)
|
988
|
+
model_path = current.model.load(
|
989
|
+
self.model,
|
990
|
+
)
|
991
|
+
self.next(self.test)
|
992
|
+
```
|
1205
993
|
|
1206
994
|
|
1207
995
|
Parameters
|
1208
996
|
----------
|
1209
|
-
|
1210
|
-
|
1211
|
-
|
1212
|
-
If
|
1213
|
-
|
1214
|
-
|
1215
|
-
|
1216
|
-
|
997
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
998
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
999
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
1000
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
1001
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
1002
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
1003
|
+
|
1004
|
+
temp_dir_root : str, default: None
|
1005
|
+
The root directory under which `current.model.loaded` will store loaded models
|
1217
1006
|
"""
|
1218
1007
|
...
|
1219
1008
|
|
1220
1009
|
@typing.overload
|
1221
|
-
def
|
1010
|
+
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1222
1011
|
...
|
1223
1012
|
|
1224
1013
|
@typing.overload
|
1225
|
-
def
|
1014
|
+
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1226
1015
|
...
|
1227
1016
|
|
1228
|
-
def
|
1017
|
+
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
1229
1018
|
"""
|
1230
|
-
|
1019
|
+
Enables loading / saving of models within a step.
|
1231
1020
|
|
1232
|
-
|
1021
|
+
> Examples
|
1022
|
+
- Saving Models
|
1023
|
+
```python
|
1024
|
+
@model
|
1025
|
+
@step
|
1026
|
+
def train(self):
|
1027
|
+
# current.model.save returns a dictionary reference to the model saved
|
1028
|
+
self.my_model = current.model.save(
|
1029
|
+
path_to_my_model,
|
1030
|
+
label="my_model",
|
1031
|
+
metadata={
|
1032
|
+
"epochs": 10,
|
1033
|
+
"batch-size": 32,
|
1034
|
+
"learning-rate": 0.001,
|
1035
|
+
}
|
1036
|
+
)
|
1037
|
+
self.next(self.test)
|
1038
|
+
|
1039
|
+
@model(load="my_model")
|
1040
|
+
@step
|
1041
|
+
def test(self):
|
1042
|
+
# `current.model.loaded` returns a dictionary of the loaded models
|
1043
|
+
# where the key is the name of the artifact and the value is the path to the model
|
1044
|
+
print(os.listdir(current.model.loaded["my_model"]))
|
1045
|
+
self.next(self.end)
|
1046
|
+
```
|
1047
|
+
|
1048
|
+
- Loading models
|
1049
|
+
```python
|
1050
|
+
@step
|
1051
|
+
def train(self):
|
1052
|
+
# current.model.load returns the path to the model loaded
|
1053
|
+
checkpoint_path = current.model.load(
|
1054
|
+
self.checkpoint_key,
|
1055
|
+
)
|
1056
|
+
model_path = current.model.load(
|
1057
|
+
self.model,
|
1058
|
+
)
|
1059
|
+
self.next(self.test)
|
1060
|
+
```
|
1233
1061
|
|
1234
1062
|
|
1235
1063
|
Parameters
|
1236
1064
|
----------
|
1237
|
-
|
1238
|
-
|
1239
|
-
|
1240
|
-
If
|
1241
|
-
|
1242
|
-
|
1243
|
-
|
1244
|
-
|
1065
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
1066
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
1067
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
1068
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
1069
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
1070
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
1071
|
+
|
1072
|
+
temp_dir_root : str, default: None
|
1073
|
+
The root directory under which `current.model.loaded` will store loaded models
|
1245
1074
|
"""
|
1246
1075
|
...
|
1247
1076
|
|
1248
|
-
|
1249
|
-
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1077
|
+
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1250
1078
|
"""
|
1251
|
-
Specifies
|
1079
|
+
Specifies that this step should execute on DGX cloud.
|
1252
1080
|
|
1253
|
-
Use `@pypi_base` to set common packages required by all
|
1254
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1255
1081
|
|
1256
1082
|
Parameters
|
1257
1083
|
----------
|
1258
|
-
|
1259
|
-
|
1260
|
-
|
1261
|
-
|
1262
|
-
|
1263
|
-
|
1264
|
-
|
1265
|
-
...
|
1266
|
-
|
1084
|
+
gpu : int
|
1085
|
+
Number of GPUs to use.
|
1086
|
+
gpu_type : str
|
1087
|
+
Type of Nvidia GPU to use.
|
1088
|
+
"""
|
1089
|
+
...
|
1090
|
+
|
1267
1091
|
@typing.overload
|
1268
|
-
def
|
1092
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1093
|
+
"""
|
1094
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1095
|
+
the execution of a step.
|
1096
|
+
|
1097
|
+
|
1098
|
+
Parameters
|
1099
|
+
----------
|
1100
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1101
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1102
|
+
"""
|
1269
1103
|
...
|
1270
1104
|
|
1271
|
-
|
1105
|
+
@typing.overload
|
1106
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1107
|
+
...
|
1108
|
+
|
1109
|
+
@typing.overload
|
1110
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1111
|
+
...
|
1112
|
+
|
1113
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
1272
1114
|
"""
|
1273
|
-
Specifies
|
1115
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1116
|
+
the execution of a step.
|
1274
1117
|
|
1275
|
-
Use `@pypi_base` to set common packages required by all
|
1276
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1277
1118
|
|
1278
1119
|
Parameters
|
1279
1120
|
----------
|
1280
|
-
|
1281
|
-
|
1282
|
-
|
1283
|
-
|
1284
|
-
|
1285
|
-
|
1121
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1122
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1123
|
+
"""
|
1124
|
+
...
|
1125
|
+
|
1126
|
+
def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1127
|
+
"""
|
1128
|
+
Specifies that this step should execute on DGX cloud.
|
1129
|
+
|
1130
|
+
|
1131
|
+
Parameters
|
1132
|
+
----------
|
1133
|
+
gpu : int
|
1134
|
+
Number of GPUs to use.
|
1135
|
+
gpu_type : str
|
1136
|
+
Type of Nvidia GPU to use.
|
1137
|
+
queue_timeout : int
|
1138
|
+
Time to keep the job in NVCF's queue.
|
1139
|
+
"""
|
1140
|
+
...
|
1141
|
+
|
1142
|
+
@typing.overload
|
1143
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1144
|
+
"""
|
1145
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1146
|
+
|
1147
|
+
|
1148
|
+
Parameters
|
1149
|
+
----------
|
1150
|
+
vars : Dict[str, str], default {}
|
1151
|
+
Dictionary of environment variables to set.
|
1152
|
+
"""
|
1153
|
+
...
|
1154
|
+
|
1155
|
+
@typing.overload
|
1156
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1157
|
+
...
|
1158
|
+
|
1159
|
+
@typing.overload
|
1160
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1161
|
+
...
|
1162
|
+
|
1163
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1164
|
+
"""
|
1165
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1166
|
+
|
1167
|
+
|
1168
|
+
Parameters
|
1169
|
+
----------
|
1170
|
+
vars : Dict[str, str], default {}
|
1171
|
+
Dictionary of environment variables to set.
|
1172
|
+
"""
|
1173
|
+
...
|
1174
|
+
|
1175
|
+
@typing.overload
|
1176
|
+
def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1177
|
+
"""
|
1178
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1179
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1180
|
+
"""
|
1181
|
+
...
|
1182
|
+
|
1183
|
+
@typing.overload
|
1184
|
+
def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1185
|
+
...
|
1186
|
+
|
1187
|
+
def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1188
|
+
"""
|
1189
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1190
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1191
|
+
"""
|
1192
|
+
...
|
1193
|
+
|
1194
|
+
@typing.overload
|
1195
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1196
|
+
"""
|
1197
|
+
Specifies a timeout for your step.
|
1198
|
+
|
1199
|
+
This decorator is useful if this step may hang indefinitely.
|
1200
|
+
|
1201
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1202
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1203
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1204
|
+
|
1205
|
+
Note that all the values specified in parameters are added together so if you specify
|
1206
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1207
|
+
|
1208
|
+
|
1209
|
+
Parameters
|
1210
|
+
----------
|
1211
|
+
seconds : int, default 0
|
1212
|
+
Number of seconds to wait prior to timing out.
|
1213
|
+
minutes : int, default 0
|
1214
|
+
Number of minutes to wait prior to timing out.
|
1215
|
+
hours : int, default 0
|
1216
|
+
Number of hours to wait prior to timing out.
|
1217
|
+
"""
|
1218
|
+
...
|
1219
|
+
|
1220
|
+
@typing.overload
|
1221
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1222
|
+
...
|
1223
|
+
|
1224
|
+
@typing.overload
|
1225
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1226
|
+
...
|
1227
|
+
|
1228
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
1229
|
+
"""
|
1230
|
+
Specifies a timeout for your step.
|
1231
|
+
|
1232
|
+
This decorator is useful if this step may hang indefinitely.
|
1233
|
+
|
1234
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1235
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1236
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1237
|
+
|
1238
|
+
Note that all the values specified in parameters are added together so if you specify
|
1239
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1240
|
+
|
1241
|
+
|
1242
|
+
Parameters
|
1243
|
+
----------
|
1244
|
+
seconds : int, default 0
|
1245
|
+
Number of seconds to wait prior to timing out.
|
1246
|
+
minutes : int, default 0
|
1247
|
+
Number of minutes to wait prior to timing out.
|
1248
|
+
hours : int, default 0
|
1249
|
+
Number of hours to wait prior to timing out.
|
1286
1250
|
"""
|
1287
1251
|
...
|
1288
1252
|
|
@@ -1400,283 +1364,87 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
|
|
1400
1364
|
"""
|
1401
1365
|
...
|
1402
1366
|
|
1403
|
-
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1404
|
-
"""
|
1405
|
-
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1406
|
-
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1407
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1408
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1409
|
-
starts only after all sensors finish.
|
1410
|
-
|
1411
|
-
|
1412
|
-
Parameters
|
1413
|
-
----------
|
1414
|
-
timeout : int
|
1415
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1416
|
-
poke_interval : int
|
1417
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1418
|
-
mode : str
|
1419
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1420
|
-
exponential_backoff : bool
|
1421
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1422
|
-
pool : str
|
1423
|
-
the slot pool this task should run in,
|
1424
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1425
|
-
soft_fail : bool
|
1426
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1427
|
-
name : str
|
1428
|
-
Name of the sensor on Airflow
|
1429
|
-
description : str
|
1430
|
-
Description of sensor in the Airflow UI
|
1431
|
-
bucket_key : Union[str, List[str]]
|
1432
|
-
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1433
|
-
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1434
|
-
bucket_name : str
|
1435
|
-
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1436
|
-
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1437
|
-
wildcard_match : bool
|
1438
|
-
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1439
|
-
aws_conn_id : str
|
1440
|
-
a reference to the s3 connection on Airflow. (Default: None)
|
1441
|
-
verify : bool
|
1442
|
-
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1443
|
-
"""
|
1444
|
-
...
|
1445
|
-
|
1446
|
-
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1447
|
-
"""
|
1448
|
-
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1449
|
-
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1450
|
-
|
1451
|
-
|
1452
|
-
Parameters
|
1453
|
-
----------
|
1454
|
-
timeout : int
|
1455
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1456
|
-
poke_interval : int
|
1457
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1458
|
-
mode : str
|
1459
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1460
|
-
exponential_backoff : bool
|
1461
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1462
|
-
pool : str
|
1463
|
-
the slot pool this task should run in,
|
1464
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1465
|
-
soft_fail : bool
|
1466
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1467
|
-
name : str
|
1468
|
-
Name of the sensor on Airflow
|
1469
|
-
description : str
|
1470
|
-
Description of sensor in the Airflow UI
|
1471
|
-
external_dag_id : str
|
1472
|
-
The dag_id that contains the task you want to wait for.
|
1473
|
-
external_task_ids : List[str]
|
1474
|
-
The list of task_ids that you want to wait for.
|
1475
|
-
If None (default value) the sensor waits for the DAG. (Default: None)
|
1476
|
-
allowed_states : List[str]
|
1477
|
-
Iterable of allowed states, (Default: ['success'])
|
1478
|
-
failed_states : List[str]
|
1479
|
-
Iterable of failed or dis-allowed states. (Default: None)
|
1480
|
-
execution_delta : datetime.timedelta
|
1481
|
-
time difference with the previous execution to look at,
|
1482
|
-
the default is the same logical date as the current task or DAG. (Default: None)
|
1483
|
-
check_existence: bool
|
1484
|
-
Set to True to check if the external task exists or check if
|
1485
|
-
the DAG to wait for exists. (Default: True)
|
1486
|
-
"""
|
1487
|
-
...
|
1488
|
-
|
1489
1367
|
@typing.overload
|
1490
|
-
def
|
1368
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1491
1369
|
"""
|
1492
|
-
Specifies the
|
1493
|
-
|
1494
|
-
```
|
1495
|
-
@trigger(event='foo')
|
1496
|
-
```
|
1497
|
-
or
|
1498
|
-
```
|
1499
|
-
@trigger(events=['foo', 'bar'])
|
1500
|
-
```
|
1501
|
-
|
1502
|
-
Additionally, you can specify the parameter mappings
|
1503
|
-
to map event payload to Metaflow parameters for the flow.
|
1504
|
-
```
|
1505
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1506
|
-
```
|
1507
|
-
or
|
1508
|
-
```
|
1509
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1510
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1511
|
-
```
|
1512
|
-
|
1513
|
-
'parameters' can also be a list of strings and tuples like so:
|
1514
|
-
```
|
1515
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1516
|
-
```
|
1517
|
-
This is equivalent to:
|
1518
|
-
```
|
1519
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1520
|
-
```
|
1370
|
+
Specifies the PyPI packages for all steps of the flow.
|
1521
1371
|
|
1372
|
+
Use `@pypi_base` to set common packages required by all
|
1373
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1522
1374
|
|
1523
1375
|
Parameters
|
1524
1376
|
----------
|
1525
|
-
|
1526
|
-
|
1527
|
-
|
1528
|
-
|
1529
|
-
|
1530
|
-
|
1377
|
+
packages : Dict[str, str], default: {}
|
1378
|
+
Packages to use for this flow. The key is the name of the package
|
1379
|
+
and the value is the version to use.
|
1380
|
+
python : str, optional, default: None
|
1381
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1382
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1531
1383
|
"""
|
1532
1384
|
...
|
1533
1385
|
|
1534
1386
|
@typing.overload
|
1535
|
-
def
|
1536
|
-
...
|
1537
|
-
|
1538
|
-
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1539
|
-
"""
|
1540
|
-
Specifies the event(s) that this flow depends on.
|
1541
|
-
|
1542
|
-
```
|
1543
|
-
@trigger(event='foo')
|
1544
|
-
```
|
1545
|
-
or
|
1546
|
-
```
|
1547
|
-
@trigger(events=['foo', 'bar'])
|
1548
|
-
```
|
1549
|
-
|
1550
|
-
Additionally, you can specify the parameter mappings
|
1551
|
-
to map event payload to Metaflow parameters for the flow.
|
1552
|
-
```
|
1553
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1554
|
-
```
|
1555
|
-
or
|
1556
|
-
```
|
1557
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1558
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1559
|
-
```
|
1560
|
-
|
1561
|
-
'parameters' can also be a list of strings and tuples like so:
|
1562
|
-
```
|
1563
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1564
|
-
```
|
1565
|
-
This is equivalent to:
|
1566
|
-
```
|
1567
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1568
|
-
```
|
1569
|
-
|
1570
|
-
|
1571
|
-
Parameters
|
1572
|
-
----------
|
1573
|
-
event : Union[str, Dict[str, Any]], optional, default None
|
1574
|
-
Event dependency for this flow.
|
1575
|
-
events : List[Union[str, Dict[str, Any]]], default []
|
1576
|
-
Events dependency for this flow.
|
1577
|
-
options : Dict[str, Any], default {}
|
1578
|
-
Backend-specific configuration for tuning eventing behavior.
|
1579
|
-
"""
|
1387
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1580
1388
|
...
|
1581
1389
|
|
1582
|
-
|
1583
|
-
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1390
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1584
1391
|
"""
|
1585
|
-
Specifies the
|
1586
|
-
|
1587
|
-
```
|
1588
|
-
@trigger_on_finish(flow='FooFlow')
|
1589
|
-
```
|
1590
|
-
or
|
1591
|
-
```
|
1592
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1593
|
-
```
|
1594
|
-
This decorator respects the @project decorator and triggers the flow
|
1595
|
-
when upstream runs within the same namespace complete successfully
|
1596
|
-
|
1597
|
-
Additionally, you can specify project aware upstream flow dependencies
|
1598
|
-
by specifying the fully qualified project_flow_name.
|
1599
|
-
```
|
1600
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1601
|
-
```
|
1602
|
-
or
|
1603
|
-
```
|
1604
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1605
|
-
```
|
1606
|
-
|
1607
|
-
You can also specify just the project or project branch (other values will be
|
1608
|
-
inferred from the current project or project branch):
|
1609
|
-
```
|
1610
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1611
|
-
```
|
1612
|
-
|
1613
|
-
Note that `branch` is typically one of:
|
1614
|
-
- `prod`
|
1615
|
-
- `user.bob`
|
1616
|
-
- `test.my_experiment`
|
1617
|
-
- `prod.staging`
|
1392
|
+
Specifies the PyPI packages for all steps of the flow.
|
1618
1393
|
|
1394
|
+
Use `@pypi_base` to set common packages required by all
|
1395
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1619
1396
|
|
1620
1397
|
Parameters
|
1621
1398
|
----------
|
1622
|
-
|
1623
|
-
|
1624
|
-
|
1625
|
-
|
1626
|
-
|
1627
|
-
|
1399
|
+
packages : Dict[str, str], default: {}
|
1400
|
+
Packages to use for this flow. The key is the name of the package
|
1401
|
+
and the value is the version to use.
|
1402
|
+
python : str, optional, default: None
|
1403
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1404
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1628
1405
|
"""
|
1629
1406
|
...
|
1630
1407
|
|
1631
|
-
|
1632
|
-
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1633
|
-
...
|
1634
|
-
|
1635
|
-
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1408
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1636
1409
|
"""
|
1637
|
-
|
1638
|
-
|
1639
|
-
|
1640
|
-
|
1641
|
-
|
1642
|
-
or
|
1643
|
-
```
|
1644
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1645
|
-
```
|
1646
|
-
This decorator respects the @project decorator and triggers the flow
|
1647
|
-
when upstream runs within the same namespace complete successfully
|
1648
|
-
|
1649
|
-
Additionally, you can specify project aware upstream flow dependencies
|
1650
|
-
by specifying the fully qualified project_flow_name.
|
1651
|
-
```
|
1652
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1653
|
-
```
|
1654
|
-
or
|
1655
|
-
```
|
1656
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1657
|
-
```
|
1658
|
-
|
1659
|
-
You can also specify just the project or project branch (other values will be
|
1660
|
-
inferred from the current project or project branch):
|
1661
|
-
```
|
1662
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1663
|
-
```
|
1664
|
-
|
1665
|
-
Note that `branch` is typically one of:
|
1666
|
-
- `prod`
|
1667
|
-
- `user.bob`
|
1668
|
-
- `test.my_experiment`
|
1669
|
-
- `prod.staging`
|
1410
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1411
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1412
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1413
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1414
|
+
starts only after all sensors finish.
|
1670
1415
|
|
1671
1416
|
|
1672
1417
|
Parameters
|
1673
1418
|
----------
|
1674
|
-
|
1675
|
-
|
1676
|
-
|
1677
|
-
|
1678
|
-
|
1679
|
-
|
1419
|
+
timeout : int
|
1420
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1421
|
+
poke_interval : int
|
1422
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1423
|
+
mode : str
|
1424
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1425
|
+
exponential_backoff : bool
|
1426
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1427
|
+
pool : str
|
1428
|
+
the slot pool this task should run in,
|
1429
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1430
|
+
soft_fail : bool
|
1431
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1432
|
+
name : str
|
1433
|
+
Name of the sensor on Airflow
|
1434
|
+
description : str
|
1435
|
+
Description of sensor in the Airflow UI
|
1436
|
+
bucket_key : Union[str, List[str]]
|
1437
|
+
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1438
|
+
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1439
|
+
bucket_name : str
|
1440
|
+
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1441
|
+
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1442
|
+
wildcard_match : bool
|
1443
|
+
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1444
|
+
aws_conn_id : str
|
1445
|
+
a reference to the s3 connection on Airflow. (Default: None)
|
1446
|
+
verify : bool
|
1447
|
+
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1680
1448
|
"""
|
1681
1449
|
...
|
1682
1450
|
|
@@ -1731,6 +1499,142 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
|
|
1731
1499
|
"""
|
1732
1500
|
...
|
1733
1501
|
|
1502
|
+
@typing.overload
|
1503
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1504
|
+
"""
|
1505
|
+
Specifies the event(s) that this flow depends on.
|
1506
|
+
|
1507
|
+
```
|
1508
|
+
@trigger(event='foo')
|
1509
|
+
```
|
1510
|
+
or
|
1511
|
+
```
|
1512
|
+
@trigger(events=['foo', 'bar'])
|
1513
|
+
```
|
1514
|
+
|
1515
|
+
Additionally, you can specify the parameter mappings
|
1516
|
+
to map event payload to Metaflow parameters for the flow.
|
1517
|
+
```
|
1518
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1519
|
+
```
|
1520
|
+
or
|
1521
|
+
```
|
1522
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1523
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1524
|
+
```
|
1525
|
+
|
1526
|
+
'parameters' can also be a list of strings and tuples like so:
|
1527
|
+
```
|
1528
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1529
|
+
```
|
1530
|
+
This is equivalent to:
|
1531
|
+
```
|
1532
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1533
|
+
```
|
1534
|
+
|
1535
|
+
|
1536
|
+
Parameters
|
1537
|
+
----------
|
1538
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1539
|
+
Event dependency for this flow.
|
1540
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1541
|
+
Events dependency for this flow.
|
1542
|
+
options : Dict[str, Any], default {}
|
1543
|
+
Backend-specific configuration for tuning eventing behavior.
|
1544
|
+
"""
|
1545
|
+
...
|
1546
|
+
|
1547
|
+
@typing.overload
|
1548
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1549
|
+
...
|
1550
|
+
|
1551
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1552
|
+
"""
|
1553
|
+
Specifies the event(s) that this flow depends on.
|
1554
|
+
|
1555
|
+
```
|
1556
|
+
@trigger(event='foo')
|
1557
|
+
```
|
1558
|
+
or
|
1559
|
+
```
|
1560
|
+
@trigger(events=['foo', 'bar'])
|
1561
|
+
```
|
1562
|
+
|
1563
|
+
Additionally, you can specify the parameter mappings
|
1564
|
+
to map event payload to Metaflow parameters for the flow.
|
1565
|
+
```
|
1566
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1567
|
+
```
|
1568
|
+
or
|
1569
|
+
```
|
1570
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1571
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1572
|
+
```
|
1573
|
+
|
1574
|
+
'parameters' can also be a list of strings and tuples like so:
|
1575
|
+
```
|
1576
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1577
|
+
```
|
1578
|
+
This is equivalent to:
|
1579
|
+
```
|
1580
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1581
|
+
```
|
1582
|
+
|
1583
|
+
|
1584
|
+
Parameters
|
1585
|
+
----------
|
1586
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1587
|
+
Event dependency for this flow.
|
1588
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1589
|
+
Events dependency for this flow.
|
1590
|
+
options : Dict[str, Any], default {}
|
1591
|
+
Backend-specific configuration for tuning eventing behavior.
|
1592
|
+
"""
|
1593
|
+
...
|
1594
|
+
|
1595
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1596
|
+
"""
|
1597
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1598
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1599
|
+
|
1600
|
+
|
1601
|
+
Parameters
|
1602
|
+
----------
|
1603
|
+
timeout : int
|
1604
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1605
|
+
poke_interval : int
|
1606
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1607
|
+
mode : str
|
1608
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1609
|
+
exponential_backoff : bool
|
1610
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1611
|
+
pool : str
|
1612
|
+
the slot pool this task should run in,
|
1613
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1614
|
+
soft_fail : bool
|
1615
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1616
|
+
name : str
|
1617
|
+
Name of the sensor on Airflow
|
1618
|
+
description : str
|
1619
|
+
Description of sensor in the Airflow UI
|
1620
|
+
external_dag_id : str
|
1621
|
+
The dag_id that contains the task you want to wait for.
|
1622
|
+
external_task_ids : List[str]
|
1623
|
+
The list of task_ids that you want to wait for.
|
1624
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
1625
|
+
allowed_states : List[str]
|
1626
|
+
Iterable of allowed states, (Default: ['success'])
|
1627
|
+
failed_states : List[str]
|
1628
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
1629
|
+
execution_delta : datetime.timedelta
|
1630
|
+
time difference with the previous execution to look at,
|
1631
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
1632
|
+
check_existence: bool
|
1633
|
+
Set to True to check if the external task exists or check if
|
1634
|
+
the DAG to wait for exists. (Default: True)
|
1635
|
+
"""
|
1636
|
+
...
|
1637
|
+
|
1734
1638
|
@typing.overload
|
1735
1639
|
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1736
1640
|
"""
|
@@ -1817,5 +1721,106 @@ def project(*, name: str, branch: typing.Optional[str] = None, production: bool
|
|
1817
1721
|
"""
|
1818
1722
|
...
|
1819
1723
|
|
1724
|
+
@typing.overload
|
1725
|
+
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1726
|
+
"""
|
1727
|
+
Specifies the flow(s) that this flow depends on.
|
1728
|
+
|
1729
|
+
```
|
1730
|
+
@trigger_on_finish(flow='FooFlow')
|
1731
|
+
```
|
1732
|
+
or
|
1733
|
+
```
|
1734
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1735
|
+
```
|
1736
|
+
This decorator respects the @project decorator and triggers the flow
|
1737
|
+
when upstream runs within the same namespace complete successfully
|
1738
|
+
|
1739
|
+
Additionally, you can specify project aware upstream flow dependencies
|
1740
|
+
by specifying the fully qualified project_flow_name.
|
1741
|
+
```
|
1742
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1743
|
+
```
|
1744
|
+
or
|
1745
|
+
```
|
1746
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1747
|
+
```
|
1748
|
+
|
1749
|
+
You can also specify just the project or project branch (other values will be
|
1750
|
+
inferred from the current project or project branch):
|
1751
|
+
```
|
1752
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1753
|
+
```
|
1754
|
+
|
1755
|
+
Note that `branch` is typically one of:
|
1756
|
+
- `prod`
|
1757
|
+
- `user.bob`
|
1758
|
+
- `test.my_experiment`
|
1759
|
+
- `prod.staging`
|
1760
|
+
|
1761
|
+
|
1762
|
+
Parameters
|
1763
|
+
----------
|
1764
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
1765
|
+
Upstream flow dependency for this flow.
|
1766
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
1767
|
+
Upstream flow dependencies for this flow.
|
1768
|
+
options : Dict[str, Any], default {}
|
1769
|
+
Backend-specific configuration for tuning eventing behavior.
|
1770
|
+
"""
|
1771
|
+
...
|
1772
|
+
|
1773
|
+
@typing.overload
|
1774
|
+
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1775
|
+
...
|
1776
|
+
|
1777
|
+
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1778
|
+
"""
|
1779
|
+
Specifies the flow(s) that this flow depends on.
|
1780
|
+
|
1781
|
+
```
|
1782
|
+
@trigger_on_finish(flow='FooFlow')
|
1783
|
+
```
|
1784
|
+
or
|
1785
|
+
```
|
1786
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1787
|
+
```
|
1788
|
+
This decorator respects the @project decorator and triggers the flow
|
1789
|
+
when upstream runs within the same namespace complete successfully
|
1790
|
+
|
1791
|
+
Additionally, you can specify project aware upstream flow dependencies
|
1792
|
+
by specifying the fully qualified project_flow_name.
|
1793
|
+
```
|
1794
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1795
|
+
```
|
1796
|
+
or
|
1797
|
+
```
|
1798
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1799
|
+
```
|
1800
|
+
|
1801
|
+
You can also specify just the project or project branch (other values will be
|
1802
|
+
inferred from the current project or project branch):
|
1803
|
+
```
|
1804
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1805
|
+
```
|
1806
|
+
|
1807
|
+
Note that `branch` is typically one of:
|
1808
|
+
- `prod`
|
1809
|
+
- `user.bob`
|
1810
|
+
- `test.my_experiment`
|
1811
|
+
- `prod.staging`
|
1812
|
+
|
1813
|
+
|
1814
|
+
Parameters
|
1815
|
+
----------
|
1816
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
1817
|
+
Upstream flow dependency for this flow.
|
1818
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
1819
|
+
Upstream flow dependencies for this flow.
|
1820
|
+
options : Dict[str, Any], default {}
|
1821
|
+
Backend-specific configuration for tuning eventing behavior.
|
1822
|
+
"""
|
1823
|
+
...
|
1824
|
+
|
1820
1825
|
pkg_name: str
|
1821
1826
|
|