ob-metaflow-stubs 6.0.10.14__py2.py3-none-any.whl → 6.0.10.16__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ob-metaflow-stubs might be problematic. Click here for more details.
- metaflow-stubs/__init__.pyi +1035 -1035
- metaflow-stubs/cards.pyi +2 -2
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/cli_components/__init__.pyi +2 -2
- metaflow-stubs/cli_components/utils.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +2 -2
- metaflow-stubs/client/core.pyi +5 -5
- metaflow-stubs/client/filecache.pyi +3 -3
- metaflow-stubs/events.pyi +2 -2
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +3 -3
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +5 -5
- metaflow-stubs/meta_files.pyi +2 -2
- metaflow-stubs/metadata_provider/__init__.pyi +2 -2
- metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
- metaflow-stubs/metadata_provider/metadata.pyi +2 -2
- metaflow-stubs/metadata_provider/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +93 -93
- metaflow-stubs/metaflow_git.pyi +2 -2
- metaflow-stubs/mf_extensions/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/cards/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/cards/hf_hub_card.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +5 -5
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +5 -5
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/ob_internal.pyi +2 -2
- metaflow-stubs/packaging_sys/__init__.pyi +7 -7
- metaflow-stubs/packaging_sys/backend.pyi +3 -3
- metaflow-stubs/packaging_sys/distribution_support.pyi +3 -3
- metaflow-stubs/packaging_sys/tar_backend.pyi +4 -4
- metaflow-stubs/packaging_sys/utils.pyi +2 -2
- metaflow-stubs/packaging_sys/v1.pyi +3 -3
- metaflow-stubs/parameters.pyi +5 -5
- metaflow-stubs/plugins/__init__.pyi +10 -10
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
- metaflow-stubs/plugins/argo/exit_hooks.pyi +3 -3
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_client.pyi +2 -2
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +5 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/json_viewer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +3 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +2 -2
- metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
- metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
- metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
- metaflow-stubs/plugins/optuna/__init__.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/parsers.pyi +2 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
- metaflow-stubs/plugins/secrets/utils.pyi +2 -2
- metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
- metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
- metaflow-stubs/plugins/uv/__init__.pyi +2 -2
- metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +34 -34
- metaflow-stubs/runner/deployer_impl.pyi +3 -3
- metaflow-stubs/runner/metaflow_runner.pyi +4 -4
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +3 -3
- metaflow-stubs/system/__init__.pyi +2 -2
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/user_configs/__init__.pyi +2 -2
- metaflow-stubs/user_configs/config_options.pyi +3 -3
- metaflow-stubs/user_configs/config_parameters.pyi +6 -6
- metaflow-stubs/user_decorators/__init__.pyi +2 -2
- metaflow-stubs/user_decorators/common.pyi +2 -2
- metaflow-stubs/user_decorators/mutable_flow.pyi +6 -6
- metaflow-stubs/user_decorators/mutable_step.pyi +5 -5
- metaflow-stubs/user_decorators/user_flow_decorator.pyi +3 -3
- metaflow-stubs/user_decorators/user_step_decorator.pyi +7 -7
- {ob_metaflow_stubs-6.0.10.14.dist-info → ob_metaflow_stubs-6.0.10.16.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.10.16.dist-info/RECORD +266 -0
- ob_metaflow_stubs-6.0.10.14.dist-info/RECORD +0 -266
- {ob_metaflow_stubs-6.0.10.14.dist-info → ob_metaflow_stubs-6.0.10.16.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.10.14.dist-info → ob_metaflow_stubs-6.0.10.16.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
######################################################################################################
|
|
2
2
|
# Auto-generated Metaflow stub file #
|
|
3
|
-
# MF version: 2.18.
|
|
4
|
-
# Generated on 2025-10-
|
|
3
|
+
# MF version: 2.18.11.1+obcheckpoint(0.2.8);ob(v1) #
|
|
4
|
+
# Generated on 2025-10-13T07:07:26.927215 #
|
|
5
5
|
######################################################################################################
|
|
6
6
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import typing
|
|
10
10
|
if typing.TYPE_CHECKING:
|
|
11
|
-
import typing
|
|
12
11
|
import datetime
|
|
12
|
+
import typing
|
|
13
13
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
|
14
14
|
StepFlag = typing.NewType("StepFlag", bool)
|
|
15
15
|
|
|
@@ -40,18 +40,18 @@ from .user_decorators.user_step_decorator import StepMutator as StepMutator
|
|
|
40
40
|
from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
|
|
41
41
|
from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
|
|
42
42
|
from . import cards as cards
|
|
43
|
-
from . import events as events
|
|
44
43
|
from . import tuple_util as tuple_util
|
|
45
44
|
from . import metaflow_git as metaflow_git
|
|
45
|
+
from . import events as events
|
|
46
46
|
from . import runner as runner
|
|
47
47
|
from . import plugins as plugins
|
|
48
48
|
from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
|
|
49
49
|
from . import includefile as includefile
|
|
50
50
|
from .includefile import IncludeFile as IncludeFile
|
|
51
|
-
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
|
52
|
-
from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
|
|
53
51
|
from .plugins.parsers import yaml_parser as yaml_parser
|
|
52
|
+
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
|
54
53
|
from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
|
|
54
|
+
from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
|
|
55
55
|
from . import client as client
|
|
56
56
|
from .client.core import namespace as namespace
|
|
57
57
|
from .client.core import get_namespace as get_namespace
|
|
@@ -169,108 +169,101 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
|
169
169
|
"""
|
|
170
170
|
...
|
|
171
171
|
|
|
172
|
-
|
|
173
|
-
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
172
|
+
def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
174
173
|
"""
|
|
175
|
-
|
|
174
|
+
This decorator is used to run vllm APIs as Metaflow task sidecars.
|
|
176
175
|
|
|
177
|
-
|
|
176
|
+
User code call
|
|
177
|
+
--------------
|
|
178
|
+
@vllm(
|
|
179
|
+
model="...",
|
|
180
|
+
...
|
|
181
|
+
)
|
|
178
182
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
183
|
+
Valid backend options
|
|
184
|
+
---------------------
|
|
185
|
+
- 'local': Run as a separate process on the local task machine.
|
|
182
186
|
|
|
183
|
-
|
|
184
|
-
|
|
187
|
+
Valid model options
|
|
188
|
+
-------------------
|
|
189
|
+
Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
|
|
190
|
+
|
|
191
|
+
NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
|
|
192
|
+
If you need multiple models, you must create multiple @vllm decorators.
|
|
185
193
|
|
|
186
194
|
|
|
187
195
|
Parameters
|
|
188
196
|
----------
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
197
|
+
model: str
|
|
198
|
+
HuggingFace model identifier to be served by vLLM.
|
|
199
|
+
backend: str
|
|
200
|
+
Determines where and how to run the vLLM process.
|
|
201
|
+
openai_api_server: bool
|
|
202
|
+
Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
|
|
203
|
+
Default is False (uses native engine).
|
|
204
|
+
Set to True for backward compatibility with existing code.
|
|
205
|
+
debug: bool
|
|
206
|
+
Whether to turn on verbose debugging logs.
|
|
207
|
+
card_refresh_interval: int
|
|
208
|
+
Interval in seconds for refreshing the vLLM status card.
|
|
209
|
+
Only used when openai_api_server=True.
|
|
210
|
+
max_retries: int
|
|
211
|
+
Maximum number of retries checking for vLLM server startup.
|
|
212
|
+
Only used when openai_api_server=True.
|
|
213
|
+
retry_alert_frequency: int
|
|
214
|
+
Frequency of alert logs for vLLM server startup retries.
|
|
215
|
+
Only used when openai_api_server=True.
|
|
216
|
+
engine_args : dict
|
|
217
|
+
Additional keyword arguments to pass to the vLLM engine.
|
|
218
|
+
For example, `tensor_parallel_size=2`.
|
|
195
219
|
"""
|
|
196
220
|
...
|
|
197
221
|
|
|
198
|
-
|
|
199
|
-
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
200
|
-
...
|
|
201
|
-
|
|
202
|
-
@typing.overload
|
|
203
|
-
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
204
|
-
...
|
|
205
|
-
|
|
206
|
-
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
|
222
|
+
def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
207
223
|
"""
|
|
208
|
-
|
|
224
|
+
This decorator is used to run Ollama APIs as Metaflow task sidecars.
|
|
209
225
|
|
|
210
|
-
|
|
226
|
+
User code call
|
|
227
|
+
--------------
|
|
228
|
+
@ollama(
|
|
229
|
+
models=[...],
|
|
230
|
+
...
|
|
231
|
+
)
|
|
211
232
|
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
233
|
+
Valid backend options
|
|
234
|
+
---------------------
|
|
235
|
+
- 'local': Run as a separate process on the local task machine.
|
|
236
|
+
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
|
237
|
+
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
|
215
238
|
|
|
216
|
-
|
|
217
|
-
|
|
239
|
+
Valid model options
|
|
240
|
+
-------------------
|
|
241
|
+
Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
|
|
218
242
|
|
|
219
243
|
|
|
220
244
|
Parameters
|
|
221
245
|
----------
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
@typing.overload
|
|
239
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
240
|
-
...
|
|
241
|
-
|
|
242
|
-
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
243
|
-
"""
|
|
244
|
-
Internal decorator to support Fast bakery
|
|
245
|
-
"""
|
|
246
|
-
...
|
|
247
|
-
|
|
248
|
-
@typing.overload
|
|
249
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
250
|
-
"""
|
|
251
|
-
Decorator prototype for all step decorators. This function gets specialized
|
|
252
|
-
and imported for all decorators types by _import_plugin_decorators().
|
|
253
|
-
"""
|
|
254
|
-
...
|
|
255
|
-
|
|
256
|
-
@typing.overload
|
|
257
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
258
|
-
...
|
|
259
|
-
|
|
260
|
-
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
261
|
-
"""
|
|
262
|
-
Decorator prototype for all step decorators. This function gets specialized
|
|
263
|
-
and imported for all decorators types by _import_plugin_decorators().
|
|
246
|
+
models: list[str]
|
|
247
|
+
List of Ollama containers running models in sidecars.
|
|
248
|
+
backend: str
|
|
249
|
+
Determines where and how to run the Ollama process.
|
|
250
|
+
force_pull: bool
|
|
251
|
+
Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
|
|
252
|
+
cache_update_policy: str
|
|
253
|
+
Cache update policy: "auto", "force", or "never".
|
|
254
|
+
force_cache_update: bool
|
|
255
|
+
Simple override for "force" cache update policy.
|
|
256
|
+
debug: bool
|
|
257
|
+
Whether to turn on verbose debugging logs.
|
|
258
|
+
circuit_breaker_config: dict
|
|
259
|
+
Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
|
|
260
|
+
timeout_config: dict
|
|
261
|
+
Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
|
|
264
262
|
"""
|
|
265
263
|
...
|
|
266
264
|
|
|
267
|
-
def
|
|
265
|
+
def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
268
266
|
"""
|
|
269
|
-
`@coreweave_s3_proxy` is a CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
270
|
-
It exists to make it easier for users to know that this decorator should only be used with
|
|
271
|
-
a Neo Cloud like CoreWeave. The underlying mechanics of the decorator is the same as the `@s3_proxy`:
|
|
272
|
-
|
|
273
|
-
|
|
274
267
|
Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
|
|
275
268
|
for S3 read and write requests.
|
|
276
269
|
|
|
@@ -328,200 +321,228 @@ def coreweave_s3_proxy(*, integration_name: typing.Optional[str] = None, write_m
|
|
|
328
321
|
"""
|
|
329
322
|
...
|
|
330
323
|
|
|
331
|
-
|
|
332
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
324
|
+
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
333
325
|
"""
|
|
334
|
-
Specifies
|
|
326
|
+
Specifies that this step should execute on Kubernetes.
|
|
335
327
|
|
|
336
328
|
|
|
337
329
|
Parameters
|
|
338
330
|
----------
|
|
339
|
-
|
|
340
|
-
|
|
331
|
+
cpu : int, default 1
|
|
332
|
+
Number of CPUs required for this step. If `@resources` is
|
|
333
|
+
also present, the maximum value from all decorators is used.
|
|
334
|
+
memory : int, default 4096
|
|
335
|
+
Memory size (in MB) required for this step. If
|
|
336
|
+
`@resources` is also present, the maximum value from all decorators is
|
|
337
|
+
used.
|
|
338
|
+
disk : int, default 10240
|
|
339
|
+
Disk size (in MB) required for this step. If
|
|
340
|
+
`@resources` is also present, the maximum value from all decorators is
|
|
341
|
+
used.
|
|
342
|
+
image : str, optional, default None
|
|
343
|
+
Docker image to use when launching on Kubernetes. If not specified, and
|
|
344
|
+
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
|
345
|
+
not, a default Docker image mapping to the current version of Python is used.
|
|
346
|
+
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
|
347
|
+
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
|
348
|
+
image_pull_secrets: List[str], default []
|
|
349
|
+
The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
|
|
350
|
+
Kubernetes image pull secrets to use when pulling container images
|
|
351
|
+
in Kubernetes.
|
|
352
|
+
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
|
353
|
+
Kubernetes service account to use when launching pod in Kubernetes.
|
|
354
|
+
secrets : List[str], optional, default None
|
|
355
|
+
Kubernetes secrets to use when launching pod in Kubernetes. These
|
|
356
|
+
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
|
357
|
+
in Metaflow configuration.
|
|
358
|
+
node_selector: Union[Dict[str,str], str], optional, default None
|
|
359
|
+
Kubernetes node selector(s) to apply to the pod running the task.
|
|
360
|
+
Can be passed in as a comma separated string of values e.g.
|
|
361
|
+
'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
|
|
362
|
+
{'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
|
|
363
|
+
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
|
364
|
+
Kubernetes namespace to use when launching pod in Kubernetes.
|
|
365
|
+
gpu : int, optional, default None
|
|
366
|
+
Number of GPUs required for this step. A value of zero implies that
|
|
367
|
+
the scheduled node should not have GPUs.
|
|
368
|
+
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
|
369
|
+
The vendor of the GPUs to be used for this step.
|
|
370
|
+
tolerations : List[Dict[str,str]], default []
|
|
371
|
+
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
|
372
|
+
Kubernetes tolerations to use when launching pod in Kubernetes.
|
|
373
|
+
labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
|
|
374
|
+
Kubernetes labels to use when launching pod in Kubernetes.
|
|
375
|
+
annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
|
|
376
|
+
Kubernetes annotations to use when launching pod in Kubernetes.
|
|
377
|
+
use_tmpfs : bool, default False
|
|
378
|
+
This enables an explicit tmpfs mount for this step.
|
|
379
|
+
tmpfs_tempdir : bool, default True
|
|
380
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
|
381
|
+
tmpfs_size : int, optional, default: None
|
|
382
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
|
383
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
|
384
|
+
memory allocated for this step.
|
|
385
|
+
tmpfs_path : str, optional, default /metaflow_temp
|
|
386
|
+
Path to tmpfs mount for this step.
|
|
387
|
+
persistent_volume_claims : Dict[str, str], optional, default None
|
|
388
|
+
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
|
389
|
+
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
|
390
|
+
shared_memory: int, optional
|
|
391
|
+
Shared memory size (in MiB) required for this step
|
|
392
|
+
port: int, optional
|
|
393
|
+
Port number to specify in the Kubernetes job object
|
|
394
|
+
compute_pool : str, optional, default None
|
|
395
|
+
Compute pool to be used for for this step.
|
|
396
|
+
If not specified, any accessible compute pool within the perimeter is used.
|
|
397
|
+
hostname_resolution_timeout: int, default 10 * 60
|
|
398
|
+
Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
|
|
399
|
+
Only applicable when @parallel is used.
|
|
400
|
+
qos: str, default: Burstable
|
|
401
|
+
Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
|
|
402
|
+
|
|
403
|
+
security_context: Dict[str, Any], optional, default None
|
|
404
|
+
Container security context. Applies to the task container. Allows the following keys:
|
|
405
|
+
- privileged: bool, optional, default None
|
|
406
|
+
- allow_privilege_escalation: bool, optional, default None
|
|
407
|
+
- run_as_user: int, optional, default None
|
|
408
|
+
- run_as_group: int, optional, default None
|
|
409
|
+
- run_as_non_root: bool, optional, default None
|
|
341
410
|
"""
|
|
342
411
|
...
|
|
343
412
|
|
|
344
413
|
@typing.overload
|
|
345
|
-
def
|
|
414
|
+
def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
415
|
+
"""
|
|
416
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
417
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
418
|
+
"""
|
|
346
419
|
...
|
|
347
420
|
|
|
348
421
|
@typing.overload
|
|
349
|
-
def
|
|
422
|
+
def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
350
423
|
...
|
|
351
424
|
|
|
352
|
-
def
|
|
425
|
+
def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
353
426
|
"""
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
Parameters
|
|
358
|
-
----------
|
|
359
|
-
vars : Dict[str, str], default {}
|
|
360
|
-
Dictionary of environment variables to set.
|
|
427
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
428
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
361
429
|
"""
|
|
362
430
|
...
|
|
363
431
|
|
|
364
|
-
def
|
|
432
|
+
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, cache_scope: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
365
433
|
"""
|
|
366
|
-
|
|
434
|
+
Decorator that helps cache, version, and store models/datasets from the Hugging Face Hub.
|
|
367
435
|
|
|
436
|
+
Examples
|
|
437
|
+
--------
|
|
368
438
|
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
@typing.overload
|
|
381
|
-
def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
382
|
-
"""
|
|
383
|
-
A simple decorator that demonstrates using CardDecoratorInjector
|
|
384
|
-
to inject a card and render simple markdown content.
|
|
385
|
-
"""
|
|
386
|
-
...
|
|
387
|
-
|
|
388
|
-
@typing.overload
|
|
389
|
-
def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
390
|
-
...
|
|
391
|
-
|
|
392
|
-
def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
393
|
-
"""
|
|
394
|
-
A simple decorator that demonstrates using CardDecoratorInjector
|
|
395
|
-
to inject a card and render simple markdown content.
|
|
396
|
-
"""
|
|
397
|
-
...
|
|
398
|
-
|
|
399
|
-
@typing.overload
|
|
400
|
-
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
401
|
-
"""
|
|
402
|
-
Specifies the resources needed when executing this step.
|
|
439
|
+
```python
|
|
440
|
+
# **Usage: creating references to models from the Hugging Face Hub that may be loaded in downstream steps**
|
|
441
|
+
@huggingface_hub
|
|
442
|
+
@step
|
|
443
|
+
def pull_model_from_huggingface(self):
|
|
444
|
+
# `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
|
|
445
|
+
# and saves it in the backend storage based on the model's `repo_id`. If there exists a model
|
|
446
|
+
# with the same `repo_id` in the backend storage, it will not download the model again. The return
|
|
447
|
+
# value of the function is a reference to the model in the backend storage.
|
|
448
|
+
# This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
|
|
403
449
|
|
|
404
|
-
|
|
405
|
-
|
|
450
|
+
self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
|
|
451
|
+
self.llama_model = current.huggingface_hub.snapshot_download(
|
|
452
|
+
repo_id=self.model_id,
|
|
453
|
+
allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
|
|
454
|
+
)
|
|
455
|
+
self.next(self.train)
|
|
406
456
|
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
457
|
+
# **Usage: explicitly loading models at runtime from the Hugging Face Hub or from cache (from Metaflow's datastore)**
|
|
458
|
+
@huggingface_hub
|
|
459
|
+
@step
|
|
460
|
+
def run_training(self):
|
|
461
|
+
# Temporary directory (auto-cleaned on exit)
|
|
462
|
+
with current.huggingface_hub.load(
|
|
463
|
+
repo_id="google-bert/bert-base-uncased",
|
|
464
|
+
allow_patterns=["*.bin"],
|
|
465
|
+
) as local_path:
|
|
466
|
+
# Use files under local_path
|
|
467
|
+
train_model(local_path)
|
|
468
|
+
...
|
|
417
469
|
|
|
470
|
+
# **Usage: loading models directly from the Hugging Face Hub or from cache (from Metaflow's datastore)**
|
|
418
471
|
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
gpu : int, optional, default None
|
|
424
|
-
Number of GPUs required for this step.
|
|
425
|
-
disk : int, optional, default None
|
|
426
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
|
427
|
-
memory : int, default 4096
|
|
428
|
-
Memory size (in MB) required for this step.
|
|
429
|
-
shared_memory : int, optional, default None
|
|
430
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
|
431
|
-
This parameter maps to the `--shm-size` option in Docker.
|
|
432
|
-
"""
|
|
433
|
-
...
|
|
434
|
-
|
|
435
|
-
@typing.overload
|
|
436
|
-
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
437
|
-
...
|
|
438
|
-
|
|
439
|
-
@typing.overload
|
|
440
|
-
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
441
|
-
...
|
|
442
|
-
|
|
443
|
-
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
|
444
|
-
"""
|
|
445
|
-
Specifies the resources needed when executing this step.
|
|
472
|
+
@huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
|
|
473
|
+
@step
|
|
474
|
+
def pull_model_from_huggingface(self):
|
|
475
|
+
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
446
476
|
|
|
447
|
-
|
|
448
|
-
|
|
477
|
+
@huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora", "/my-lora-directory")])
|
|
478
|
+
@step
|
|
479
|
+
def finetune_model(self):
|
|
480
|
+
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
481
|
+
# path_to_model will be /my-directory
|
|
449
482
|
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
483
|
+
|
|
484
|
+
# Takes all the arguments passed to `snapshot_download`
|
|
485
|
+
# except for `local_dir`
|
|
486
|
+
@huggingface_hub(load=[
|
|
487
|
+
{
|
|
488
|
+
"repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
|
|
489
|
+
},
|
|
490
|
+
{
|
|
491
|
+
"repo_id": "myorg/mistral-lora",
|
|
492
|
+
"repo_type": "model",
|
|
493
|
+
},
|
|
494
|
+
])
|
|
495
|
+
@step
|
|
496
|
+
def finetune_model(self):
|
|
497
|
+
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
498
|
+
# path_to_model will be /my-directory
|
|
457
499
|
```
|
|
458
|
-
which executes the flow on the desired system using the
|
|
459
|
-
requirements specified in `@resources`.
|
|
460
500
|
|
|
461
501
|
|
|
462
502
|
Parameters
|
|
463
503
|
----------
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
gpu : int, optional, default None
|
|
467
|
-
Number of GPUs required for this step.
|
|
468
|
-
disk : int, optional, default None
|
|
469
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
|
470
|
-
memory : int, default 4096
|
|
471
|
-
Memory size (in MB) required for this step.
|
|
472
|
-
shared_memory : int, optional, default None
|
|
473
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
|
474
|
-
This parameter maps to the `--shm-size` option in Docker.
|
|
475
|
-
"""
|
|
476
|
-
...
|
|
477
|
-
|
|
478
|
-
def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
479
|
-
"""
|
|
480
|
-
This decorator is used to run vllm APIs as Metaflow task sidecars.
|
|
504
|
+
temp_dir_root : str, optional
|
|
505
|
+
The root directory that will hold the temporary directory where objects will be downloaded.
|
|
481
506
|
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
)
|
|
507
|
+
cache_scope : str, optional
|
|
508
|
+
The scope of the cache. Can be `checkpoint` / `flow` / `global`.
|
|
509
|
+
- `checkpoint` (default): All repos are stored like objects saved by `@checkpoint`.
|
|
510
|
+
i.e., the cached path is derived from the namespace, flow, step, and Metaflow foreach iteration.
|
|
511
|
+
Any repo downloaded under this scope will only be retrieved from the cache when the step runs under the same namespace in the same flow (at the same foreach index).
|
|
488
512
|
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
513
|
+
- `flow`: All repos are cached under the flow, regardless of namespace.
|
|
514
|
+
i.e., the cached path is derived solely from the flow name.
|
|
515
|
+
When to use this mode: (1) Multiple users are executing the same flow and want shared access to the repos cached by the decorator. (2) Multiple versions of a flow are deployed, all needing access to the same repos cached by the decorator.
|
|
492
516
|
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
517
|
+
- `global`: All repos are cached under a globally static path.
|
|
518
|
+
i.e., the base path of the cache is static and all repos are stored under it.
|
|
519
|
+
When to use this mode:
|
|
520
|
+
- All repos from the Hugging Face Hub need to be shared by users across all flow executions.
|
|
521
|
+
- Each caching scope comes with its own trade-offs:
|
|
522
|
+
- `checkpoint`:
|
|
523
|
+
- Has explicit control over when caches are populated (controlled by the same flow that has the `@huggingface_hub` decorator) but ends up hitting the Hugging Face Hub more often if there are many users/namespaces/steps.
|
|
524
|
+
- Since objects are written on a `namespace/flow/step` basis, the blast radius of a bad checkpoint is limited to a particular flow in a namespace.
|
|
525
|
+
- `flow`:
|
|
526
|
+
- Has less control over when caches are populated (can be written by any execution instance of a flow from any namespace) but results in more cache hits.
|
|
527
|
+
- The blast radius of a bad checkpoint is limited to all runs of a particular flow.
|
|
528
|
+
- It doesn't promote cache reuse across flows.
|
|
529
|
+
- `global`:
|
|
530
|
+
- Has no control over when caches are populated (can be written by any flow execution) but has the highest cache hit rate.
|
|
531
|
+
- It promotes cache reuse across flows.
|
|
532
|
+
- The blast radius of a bad checkpoint spans every flow that could be using a particular repo.
|
|
496
533
|
|
|
497
|
-
|
|
498
|
-
|
|
534
|
+
load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
|
|
535
|
+
The list of repos (models/datasets) to load.
|
|
499
536
|
|
|
537
|
+
Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
|
|
500
538
|
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
|
|
509
|
-
Default is False (uses native engine).
|
|
510
|
-
Set to True for backward compatibility with existing code.
|
|
511
|
-
debug: bool
|
|
512
|
-
Whether to turn on verbose debugging logs.
|
|
513
|
-
card_refresh_interval: int
|
|
514
|
-
Interval in seconds for refreshing the vLLM status card.
|
|
515
|
-
Only used when openai_api_server=True.
|
|
516
|
-
max_retries: int
|
|
517
|
-
Maximum number of retries checking for vLLM server startup.
|
|
518
|
-
Only used when openai_api_server=True.
|
|
519
|
-
retry_alert_frequency: int
|
|
520
|
-
Frequency of alert logs for vLLM server startup retries.
|
|
521
|
-
Only used when openai_api_server=True.
|
|
522
|
-
engine_args : dict
|
|
523
|
-
Additional keyword arguments to pass to the vLLM engine.
|
|
524
|
-
For example, `tensor_parallel_size=2`.
|
|
539
|
+
- If repo (model/dataset) is not found in the datastore:
|
|
540
|
+
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
|
541
|
+
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
|
542
|
+
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
|
543
|
+
|
|
544
|
+
- If repo is found in the datastore:
|
|
545
|
+
- Loads it directly from datastore to local path (can be temporary directory or specified path)
|
|
525
546
|
"""
|
|
526
547
|
...
|
|
527
548
|
|
|
@@ -590,96 +611,33 @@ def nebius_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode
|
|
|
590
611
|
...
|
|
591
612
|
|
|
592
613
|
@typing.overload
|
|
593
|
-
def
|
|
614
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
594
615
|
"""
|
|
595
|
-
|
|
596
|
-
to a step needs to be retried.
|
|
597
|
-
|
|
598
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
|
599
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
600
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
|
601
|
-
|
|
602
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
|
603
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
|
604
|
-
ensuring that the flow execution can continue.
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
Parameters
|
|
608
|
-
----------
|
|
609
|
-
times : int, default 3
|
|
610
|
-
Number of times to retry this task.
|
|
611
|
-
minutes_between_retries : int, default 2
|
|
612
|
-
Number of minutes between retries.
|
|
616
|
+
Internal decorator to support Fast bakery
|
|
613
617
|
"""
|
|
614
618
|
...
|
|
615
619
|
|
|
616
620
|
@typing.overload
|
|
617
|
-
def
|
|
618
|
-
...
|
|
619
|
-
|
|
620
|
-
@typing.overload
|
|
621
|
-
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
622
|
-
...
|
|
623
|
-
|
|
624
|
-
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
|
625
|
-
"""
|
|
626
|
-
Specifies the number of times the task corresponding
|
|
627
|
-
to a step needs to be retried.
|
|
628
|
-
|
|
629
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
|
630
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
631
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
|
632
|
-
|
|
633
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
|
634
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
|
635
|
-
ensuring that the flow execution can continue.
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
Parameters
|
|
639
|
-
----------
|
|
640
|
-
times : int, default 3
|
|
641
|
-
Number of times to retry this task.
|
|
642
|
-
minutes_between_retries : int, default 2
|
|
643
|
-
Number of minutes between retries.
|
|
644
|
-
"""
|
|
621
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
645
622
|
...
|
|
646
623
|
|
|
647
|
-
|
|
648
|
-
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
624
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
649
625
|
"""
|
|
650
|
-
|
|
651
|
-
the execution of a step.
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
Parameters
|
|
655
|
-
----------
|
|
656
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
|
657
|
-
List of secret specs, defining how the secrets are to be retrieved
|
|
658
|
-
role : str, optional, default: None
|
|
659
|
-
Role to use for fetching secrets
|
|
626
|
+
Internal decorator to support Fast bakery
|
|
660
627
|
"""
|
|
661
628
|
...
|
|
662
629
|
|
|
663
|
-
|
|
664
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
665
|
-
...
|
|
666
|
-
|
|
667
|
-
@typing.overload
|
|
668
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
669
|
-
...
|
|
670
|
-
|
|
671
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
|
|
630
|
+
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
672
631
|
"""
|
|
673
|
-
Specifies
|
|
674
|
-
the execution of a step.
|
|
632
|
+
Specifies that this step should execute on DGX cloud.
|
|
675
633
|
|
|
676
634
|
|
|
677
635
|
Parameters
|
|
678
636
|
----------
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
637
|
+
gpu : int
|
|
638
|
+
Number of GPUs to use.
|
|
639
|
+
gpu_type : str
|
|
640
|
+
Type of Nvidia GPU to use.
|
|
683
641
|
"""
|
|
684
642
|
...
|
|
685
643
|
|
|
@@ -812,76 +770,126 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
|
812
770
|
"""
|
|
813
771
|
...
|
|
814
772
|
|
|
815
|
-
|
|
773
|
+
@typing.overload
|
|
774
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
816
775
|
"""
|
|
817
|
-
Specifies
|
|
776
|
+
Specifies the number of times the task corresponding
|
|
777
|
+
to a step needs to be retried.
|
|
778
|
+
|
|
779
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
|
780
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
781
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
|
782
|
+
|
|
783
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
|
784
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
|
785
|
+
ensuring that the flow execution can continue.
|
|
818
786
|
|
|
819
787
|
|
|
820
788
|
Parameters
|
|
821
789
|
----------
|
|
822
|
-
|
|
823
|
-
Number of
|
|
824
|
-
|
|
825
|
-
|
|
790
|
+
times : int, default 3
|
|
791
|
+
Number of times to retry this task.
|
|
792
|
+
minutes_between_retries : int, default 2
|
|
793
|
+
Number of minutes between retries.
|
|
826
794
|
"""
|
|
827
795
|
...
|
|
828
796
|
|
|
829
797
|
@typing.overload
|
|
830
|
-
def
|
|
798
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
799
|
+
...
|
|
800
|
+
|
|
801
|
+
@typing.overload
|
|
802
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
803
|
+
...
|
|
804
|
+
|
|
805
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
|
831
806
|
"""
|
|
832
|
-
Specifies the
|
|
807
|
+
Specifies the number of times the task corresponding
|
|
808
|
+
to a step needs to be retried.
|
|
833
809
|
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
810
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
|
811
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
812
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
|
813
|
+
|
|
814
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
|
815
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
|
816
|
+
ensuring that the flow execution can continue.
|
|
838
817
|
|
|
839
818
|
|
|
840
819
|
Parameters
|
|
841
820
|
----------
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
847
|
-
python : str, optional, default None
|
|
848
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
849
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
850
|
-
disabled : bool, default False
|
|
851
|
-
If set to True, disables @conda.
|
|
821
|
+
times : int, default 3
|
|
822
|
+
Number of times to retry this task.
|
|
823
|
+
minutes_between_retries : int, default 2
|
|
824
|
+
Number of minutes between retries.
|
|
852
825
|
"""
|
|
853
826
|
...
|
|
854
827
|
|
|
855
828
|
@typing.overload
|
|
856
|
-
def
|
|
829
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
830
|
+
"""
|
|
831
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
|
832
|
+
|
|
833
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
|
834
|
+
|
|
835
|
+
|
|
836
|
+
Parameters
|
|
837
|
+
----------
|
|
838
|
+
type : str, default 'default'
|
|
839
|
+
Card type.
|
|
840
|
+
id : str, optional, default None
|
|
841
|
+
If multiple cards are present, use this id to identify this card.
|
|
842
|
+
options : Dict[str, Any], default {}
|
|
843
|
+
Options passed to the card. The contents depend on the card type.
|
|
844
|
+
timeout : int, default 45
|
|
845
|
+
Interrupt reporting if it takes more than this many seconds.
|
|
846
|
+
"""
|
|
857
847
|
...
|
|
858
848
|
|
|
859
849
|
@typing.overload
|
|
860
|
-
def
|
|
850
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
861
851
|
...
|
|
862
852
|
|
|
863
|
-
|
|
853
|
+
@typing.overload
|
|
854
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
855
|
+
...
|
|
856
|
+
|
|
857
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
|
864
858
|
"""
|
|
865
|
-
|
|
859
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
|
866
860
|
|
|
867
|
-
|
|
868
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
|
869
|
-
you can use `@conda_base` to set packages required by all
|
|
870
|
-
steps and use `@conda` to specify step-specific overrides.
|
|
861
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
|
871
862
|
|
|
872
863
|
|
|
873
864
|
Parameters
|
|
874
865
|
----------
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
866
|
+
type : str, default 'default'
|
|
867
|
+
Card type.
|
|
868
|
+
id : str, optional, default None
|
|
869
|
+
If multiple cards are present, use this id to identify this card.
|
|
870
|
+
options : Dict[str, Any], default {}
|
|
871
|
+
Options passed to the card. The contents depend on the card type.
|
|
872
|
+
timeout : int, default 45
|
|
873
|
+
Interrupt reporting if it takes more than this many seconds.
|
|
874
|
+
"""
|
|
875
|
+
...
|
|
876
|
+
|
|
877
|
+
@typing.overload
|
|
878
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
879
|
+
"""
|
|
880
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
881
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
882
|
+
"""
|
|
883
|
+
...
|
|
884
|
+
|
|
885
|
+
@typing.overload
|
|
886
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
887
|
+
...
|
|
888
|
+
|
|
889
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
890
|
+
"""
|
|
891
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
892
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
885
893
|
"""
|
|
886
894
|
...
|
|
887
895
|
|
|
@@ -1033,309 +1041,283 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
|
|
|
1033
1041
|
...
|
|
1034
1042
|
|
|
1035
1043
|
@typing.overload
|
|
1036
|
-
def
|
|
1044
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1037
1045
|
"""
|
|
1038
|
-
Specifies
|
|
1046
|
+
Specifies a timeout for your step.
|
|
1039
1047
|
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1048
|
+
This decorator is useful if this step may hang indefinitely.
|
|
1049
|
+
|
|
1050
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
|
1051
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
|
1052
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
|
1053
|
+
|
|
1054
|
+
Note that all the values specified in parameters are added together so if you specify
|
|
1055
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
|
1044
1056
|
|
|
1045
1057
|
|
|
1046
1058
|
Parameters
|
|
1047
1059
|
----------
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1060
|
+
seconds : int, default 0
|
|
1061
|
+
Number of seconds to wait prior to timing out.
|
|
1062
|
+
minutes : int, default 0
|
|
1063
|
+
Number of minutes to wait prior to timing out.
|
|
1064
|
+
hours : int, default 0
|
|
1065
|
+
Number of hours to wait prior to timing out.
|
|
1054
1066
|
"""
|
|
1055
1067
|
...
|
|
1056
1068
|
|
|
1057
1069
|
@typing.overload
|
|
1058
|
-
def
|
|
1070
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1059
1071
|
...
|
|
1060
1072
|
|
|
1061
1073
|
@typing.overload
|
|
1062
|
-
def
|
|
1074
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1063
1075
|
...
|
|
1064
1076
|
|
|
1065
|
-
def
|
|
1077
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
|
1066
1078
|
"""
|
|
1067
|
-
Specifies
|
|
1079
|
+
Specifies a timeout for your step.
|
|
1068
1080
|
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1081
|
+
This decorator is useful if this step may hang indefinitely.
|
|
1082
|
+
|
|
1083
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
|
1084
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
|
1085
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
|
1086
|
+
|
|
1087
|
+
Note that all the values specified in parameters are added together so if you specify
|
|
1088
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
|
1073
1089
|
|
|
1074
1090
|
|
|
1075
1091
|
Parameters
|
|
1076
1092
|
----------
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
"""
|
|
1084
|
-
...
|
|
1085
|
-
|
|
1086
|
-
def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1087
|
-
"""
|
|
1088
|
-
This decorator is used to run Ollama APIs as Metaflow task sidecars.
|
|
1089
|
-
|
|
1090
|
-
User code call
|
|
1091
|
-
--------------
|
|
1092
|
-
@ollama(
|
|
1093
|
-
models=[...],
|
|
1094
|
-
...
|
|
1095
|
-
)
|
|
1096
|
-
|
|
1097
|
-
Valid backend options
|
|
1098
|
-
---------------------
|
|
1099
|
-
- 'local': Run as a separate process on the local task machine.
|
|
1100
|
-
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
|
1101
|
-
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
|
1102
|
-
|
|
1103
|
-
Valid model options
|
|
1104
|
-
-------------------
|
|
1105
|
-
Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
Parameters
|
|
1109
|
-
----------
|
|
1110
|
-
models: list[str]
|
|
1111
|
-
List of Ollama containers running models in sidecars.
|
|
1112
|
-
backend: str
|
|
1113
|
-
Determines where and how to run the Ollama process.
|
|
1114
|
-
force_pull: bool
|
|
1115
|
-
Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
|
|
1116
|
-
cache_update_policy: str
|
|
1117
|
-
Cache update policy: "auto", "force", or "never".
|
|
1118
|
-
force_cache_update: bool
|
|
1119
|
-
Simple override for "force" cache update policy.
|
|
1120
|
-
debug: bool
|
|
1121
|
-
Whether to turn on verbose debugging logs.
|
|
1122
|
-
circuit_breaker_config: dict
|
|
1123
|
-
Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
|
|
1124
|
-
timeout_config: dict
|
|
1125
|
-
Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
|
|
1126
|
-
"""
|
|
1127
|
-
...
|
|
1128
|
-
|
|
1129
|
-
@typing.overload
|
|
1130
|
-
def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1131
|
-
"""
|
|
1132
|
-
Decorator prototype for all step decorators. This function gets specialized
|
|
1133
|
-
and imported for all decorators types by _import_plugin_decorators().
|
|
1134
|
-
"""
|
|
1135
|
-
...
|
|
1136
|
-
|
|
1137
|
-
@typing.overload
|
|
1138
|
-
def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1139
|
-
...
|
|
1140
|
-
|
|
1141
|
-
def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
1142
|
-
"""
|
|
1143
|
-
Decorator prototype for all step decorators. This function gets specialized
|
|
1144
|
-
and imported for all decorators types by _import_plugin_decorators().
|
|
1093
|
+
seconds : int, default 0
|
|
1094
|
+
Number of seconds to wait prior to timing out.
|
|
1095
|
+
minutes : int, default 0
|
|
1096
|
+
Number of minutes to wait prior to timing out.
|
|
1097
|
+
hours : int, default 0
|
|
1098
|
+
Number of hours to wait prior to timing out.
|
|
1145
1099
|
"""
|
|
1146
1100
|
...
|
|
1147
1101
|
|
|
1148
1102
|
@typing.overload
|
|
1149
|
-
def
|
|
1103
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1150
1104
|
"""
|
|
1151
|
-
Specifies the
|
|
1105
|
+
Specifies the Conda environment for the step.
|
|
1152
1106
|
|
|
1153
1107
|
Information in this decorator will augment any
|
|
1154
|
-
attributes set in the `@
|
|
1155
|
-
you can use `@
|
|
1156
|
-
steps and use `@
|
|
1108
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
|
1109
|
+
you can use `@conda_base` to set packages required by all
|
|
1110
|
+
steps and use `@conda` to specify step-specific overrides.
|
|
1157
1111
|
|
|
1158
1112
|
|
|
1159
1113
|
Parameters
|
|
1160
1114
|
----------
|
|
1161
|
-
packages : Dict[str, str], default
|
|
1115
|
+
packages : Dict[str, str], default {}
|
|
1162
1116
|
Packages to use for this step. The key is the name of the package
|
|
1163
1117
|
and the value is the version to use.
|
|
1164
|
-
|
|
1118
|
+
libraries : Dict[str, str], default {}
|
|
1119
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1120
|
+
python : str, optional, default None
|
|
1165
1121
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1166
1122
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1123
|
+
disabled : bool, default False
|
|
1124
|
+
If set to True, disables @conda.
|
|
1167
1125
|
"""
|
|
1168
1126
|
...
|
|
1169
1127
|
|
|
1170
1128
|
@typing.overload
|
|
1171
|
-
def
|
|
1129
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1172
1130
|
...
|
|
1173
1131
|
|
|
1174
1132
|
@typing.overload
|
|
1175
|
-
def
|
|
1133
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1176
1134
|
...
|
|
1177
1135
|
|
|
1178
|
-
def
|
|
1136
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
|
1179
1137
|
"""
|
|
1180
|
-
Specifies the
|
|
1138
|
+
Specifies the Conda environment for the step.
|
|
1181
1139
|
|
|
1182
1140
|
Information in this decorator will augment any
|
|
1183
|
-
attributes set in the `@
|
|
1184
|
-
you can use `@
|
|
1185
|
-
steps and use `@
|
|
1141
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
|
1142
|
+
you can use `@conda_base` to set packages required by all
|
|
1143
|
+
steps and use `@conda` to specify step-specific overrides.
|
|
1186
1144
|
|
|
1187
1145
|
|
|
1188
1146
|
Parameters
|
|
1189
1147
|
----------
|
|
1190
|
-
packages : Dict[str, str], default
|
|
1148
|
+
packages : Dict[str, str], default {}
|
|
1191
1149
|
Packages to use for this step. The key is the name of the package
|
|
1192
1150
|
and the value is the version to use.
|
|
1193
|
-
|
|
1151
|
+
libraries : Dict[str, str], default {}
|
|
1152
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1153
|
+
python : str, optional, default None
|
|
1194
1154
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1195
1155
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1200
|
-
"""
|
|
1201
|
-
Specifies that this step should execute on Kubernetes.
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
Parameters
|
|
1205
|
-
----------
|
|
1206
|
-
cpu : int, default 1
|
|
1207
|
-
Number of CPUs required for this step. If `@resources` is
|
|
1208
|
-
also present, the maximum value from all decorators is used.
|
|
1209
|
-
memory : int, default 4096
|
|
1210
|
-
Memory size (in MB) required for this step. If
|
|
1211
|
-
`@resources` is also present, the maximum value from all decorators is
|
|
1212
|
-
used.
|
|
1213
|
-
disk : int, default 10240
|
|
1214
|
-
Disk size (in MB) required for this step. If
|
|
1215
|
-
`@resources` is also present, the maximum value from all decorators is
|
|
1216
|
-
used.
|
|
1217
|
-
image : str, optional, default None
|
|
1218
|
-
Docker image to use when launching on Kubernetes. If not specified, and
|
|
1219
|
-
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
|
1220
|
-
not, a default Docker image mapping to the current version of Python is used.
|
|
1221
|
-
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
|
1222
|
-
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
|
1223
|
-
image_pull_secrets: List[str], default []
|
|
1224
|
-
The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
|
|
1225
|
-
Kubernetes image pull secrets to use when pulling container images
|
|
1226
|
-
in Kubernetes.
|
|
1227
|
-
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
|
1228
|
-
Kubernetes service account to use when launching pod in Kubernetes.
|
|
1229
|
-
secrets : List[str], optional, default None
|
|
1230
|
-
Kubernetes secrets to use when launching pod in Kubernetes. These
|
|
1231
|
-
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
|
1232
|
-
in Metaflow configuration.
|
|
1233
|
-
node_selector: Union[Dict[str,str], str], optional, default None
|
|
1234
|
-
Kubernetes node selector(s) to apply to the pod running the task.
|
|
1235
|
-
Can be passed in as a comma separated string of values e.g.
|
|
1236
|
-
'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
|
|
1237
|
-
{'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
|
|
1238
|
-
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
|
1239
|
-
Kubernetes namespace to use when launching pod in Kubernetes.
|
|
1240
|
-
gpu : int, optional, default None
|
|
1241
|
-
Number of GPUs required for this step. A value of zero implies that
|
|
1242
|
-
the scheduled node should not have GPUs.
|
|
1243
|
-
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
|
1244
|
-
The vendor of the GPUs to be used for this step.
|
|
1245
|
-
tolerations : List[Dict[str,str]], default []
|
|
1246
|
-
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
|
1247
|
-
Kubernetes tolerations to use when launching pod in Kubernetes.
|
|
1248
|
-
labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
|
|
1249
|
-
Kubernetes labels to use when launching pod in Kubernetes.
|
|
1250
|
-
annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
|
|
1251
|
-
Kubernetes annotations to use when launching pod in Kubernetes.
|
|
1252
|
-
use_tmpfs : bool, default False
|
|
1253
|
-
This enables an explicit tmpfs mount for this step.
|
|
1254
|
-
tmpfs_tempdir : bool, default True
|
|
1255
|
-
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
|
1256
|
-
tmpfs_size : int, optional, default: None
|
|
1257
|
-
The value for the size (in MiB) of the tmpfs mount for this step.
|
|
1258
|
-
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
|
1259
|
-
memory allocated for this step.
|
|
1260
|
-
tmpfs_path : str, optional, default /metaflow_temp
|
|
1261
|
-
Path to tmpfs mount for this step.
|
|
1262
|
-
persistent_volume_claims : Dict[str, str], optional, default None
|
|
1263
|
-
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
|
1264
|
-
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
|
1265
|
-
shared_memory: int, optional
|
|
1266
|
-
Shared memory size (in MiB) required for this step
|
|
1267
|
-
port: int, optional
|
|
1268
|
-
Port number to specify in the Kubernetes job object
|
|
1269
|
-
compute_pool : str, optional, default None
|
|
1270
|
-
Compute pool to be used for for this step.
|
|
1271
|
-
If not specified, any accessible compute pool within the perimeter is used.
|
|
1272
|
-
hostname_resolution_timeout: int, default 10 * 60
|
|
1273
|
-
Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
|
|
1274
|
-
Only applicable when @parallel is used.
|
|
1275
|
-
qos: str, default: Burstable
|
|
1276
|
-
Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
|
|
1277
|
-
|
|
1278
|
-
security_context: Dict[str, Any], optional, default None
|
|
1279
|
-
Container security context. Applies to the task container. Allows the following keys:
|
|
1280
|
-
- privileged: bool, optional, default None
|
|
1281
|
-
- allow_privilege_escalation: bool, optional, default None
|
|
1282
|
-
- run_as_user: int, optional, default None
|
|
1283
|
-
- run_as_group: int, optional, default None
|
|
1284
|
-
- run_as_non_root: bool, optional, default None
|
|
1156
|
+
disabled : bool, default False
|
|
1157
|
+
If set to True, disables @conda.
|
|
1285
1158
|
"""
|
|
1286
1159
|
...
|
|
1287
1160
|
|
|
1288
1161
|
@typing.overload
|
|
1289
|
-
def
|
|
1162
|
+
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1290
1163
|
"""
|
|
1291
|
-
|
|
1164
|
+
Specifies the resources needed when executing this step.
|
|
1292
1165
|
|
|
1293
|
-
|
|
1166
|
+
Use `@resources` to specify the resource requirements
|
|
1167
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
|
1168
|
+
|
|
1169
|
+
You can choose the compute layer on the command line by executing e.g.
|
|
1170
|
+
```
|
|
1171
|
+
python myflow.py run --with batch
|
|
1172
|
+
```
|
|
1173
|
+
or
|
|
1174
|
+
```
|
|
1175
|
+
python myflow.py run --with kubernetes
|
|
1176
|
+
```
|
|
1177
|
+
which executes the flow on the desired system using the
|
|
1178
|
+
requirements specified in `@resources`.
|
|
1294
1179
|
|
|
1295
1180
|
|
|
1296
1181
|
Parameters
|
|
1297
1182
|
----------
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1183
|
+
cpu : int, default 1
|
|
1184
|
+
Number of CPUs required for this step.
|
|
1185
|
+
gpu : int, optional, default None
|
|
1186
|
+
Number of GPUs required for this step.
|
|
1187
|
+
disk : int, optional, default None
|
|
1188
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
|
1189
|
+
memory : int, default 4096
|
|
1190
|
+
Memory size (in MB) required for this step.
|
|
1191
|
+
shared_memory : int, optional, default None
|
|
1192
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
|
1193
|
+
This parameter maps to the `--shm-size` option in Docker.
|
|
1306
1194
|
"""
|
|
1307
1195
|
...
|
|
1308
1196
|
|
|
1309
1197
|
@typing.overload
|
|
1310
|
-
def
|
|
1198
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1311
1199
|
...
|
|
1312
1200
|
|
|
1313
1201
|
@typing.overload
|
|
1314
|
-
def
|
|
1202
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1315
1203
|
...
|
|
1316
1204
|
|
|
1317
|
-
def
|
|
1205
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
|
1318
1206
|
"""
|
|
1319
|
-
|
|
1207
|
+
Specifies the resources needed when executing this step.
|
|
1320
1208
|
|
|
1321
|
-
|
|
1209
|
+
Use `@resources` to specify the resource requirements
|
|
1210
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
|
1211
|
+
|
|
1212
|
+
You can choose the compute layer on the command line by executing e.g.
|
|
1213
|
+
```
|
|
1214
|
+
python myflow.py run --with batch
|
|
1215
|
+
```
|
|
1216
|
+
or
|
|
1217
|
+
```
|
|
1218
|
+
python myflow.py run --with kubernetes
|
|
1219
|
+
```
|
|
1220
|
+
which executes the flow on the desired system using the
|
|
1221
|
+
requirements specified in `@resources`.
|
|
1322
1222
|
|
|
1323
1223
|
|
|
1324
1224
|
Parameters
|
|
1325
1225
|
----------
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1226
|
+
cpu : int, default 1
|
|
1227
|
+
Number of CPUs required for this step.
|
|
1228
|
+
gpu : int, optional, default None
|
|
1229
|
+
Number of GPUs required for this step.
|
|
1230
|
+
disk : int, optional, default None
|
|
1231
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
|
1232
|
+
memory : int, default 4096
|
|
1233
|
+
Memory size (in MB) required for this step.
|
|
1234
|
+
shared_memory : int, optional, default None
|
|
1235
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
|
1236
|
+
This parameter maps to the `--shm-size` option in Docker.
|
|
1334
1237
|
"""
|
|
1335
1238
|
...
|
|
1336
1239
|
|
|
1337
|
-
|
|
1240
|
+
@typing.overload
|
|
1241
|
+
def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1242
|
+
"""
|
|
1243
|
+
A simple decorator that demonstrates using CardDecoratorInjector
|
|
1244
|
+
to inject a card and render simple markdown content.
|
|
1245
|
+
"""
|
|
1246
|
+
...
|
|
1247
|
+
|
|
1248
|
+
@typing.overload
|
|
1249
|
+
def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1250
|
+
...
|
|
1251
|
+
|
|
1252
|
+
def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
1253
|
+
"""
|
|
1254
|
+
A simple decorator that demonstrates using CardDecoratorInjector
|
|
1255
|
+
to inject a card and render simple markdown content.
|
|
1256
|
+
"""
|
|
1257
|
+
...
|
|
1258
|
+
|
|
1259
|
+
def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1260
|
+
"""
|
|
1261
|
+
Specifies that this step should execute on DGX cloud.
|
|
1262
|
+
|
|
1263
|
+
|
|
1264
|
+
Parameters
|
|
1265
|
+
----------
|
|
1266
|
+
gpu : int
|
|
1267
|
+
Number of GPUs to use.
|
|
1268
|
+
gpu_type : str
|
|
1269
|
+
Type of Nvidia GPU to use.
|
|
1270
|
+
queue_timeout : int
|
|
1271
|
+
Time to keep the job in NVCF's queue.
|
|
1272
|
+
"""
|
|
1273
|
+
...
|
|
1274
|
+
|
|
1275
|
+
@typing.overload
|
|
1276
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1277
|
+
"""
|
|
1278
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
|
1279
|
+
the execution of a step.
|
|
1280
|
+
|
|
1281
|
+
|
|
1282
|
+
Parameters
|
|
1283
|
+
----------
|
|
1284
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
|
1285
|
+
List of secret specs, defining how the secrets are to be retrieved
|
|
1286
|
+
role : str, optional, default: None
|
|
1287
|
+
Role to use for fetching secrets
|
|
1288
|
+
"""
|
|
1289
|
+
...
|
|
1290
|
+
|
|
1291
|
+
@typing.overload
|
|
1292
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1293
|
+
...
|
|
1294
|
+
|
|
1295
|
+
@typing.overload
|
|
1296
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1297
|
+
...
|
|
1298
|
+
|
|
1299
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
|
|
1338
1300
|
"""
|
|
1301
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
|
1302
|
+
the execution of a step.
|
|
1303
|
+
|
|
1304
|
+
|
|
1305
|
+
Parameters
|
|
1306
|
+
----------
|
|
1307
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
|
1308
|
+
List of secret specs, defining how the secrets are to be retrieved
|
|
1309
|
+
role : str, optional, default: None
|
|
1310
|
+
Role to use for fetching secrets
|
|
1311
|
+
"""
|
|
1312
|
+
...
|
|
1313
|
+
|
|
1314
|
+
def coreweave_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1315
|
+
"""
|
|
1316
|
+
`@coreweave_s3_proxy` is a CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
1317
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
|
1318
|
+
a Neo Cloud like CoreWeave. The underlying mechanics of the decorator is the same as the `@s3_proxy`:
|
|
1319
|
+
|
|
1320
|
+
|
|
1339
1321
|
Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
|
|
1340
1322
|
for S3 read and write requests.
|
|
1341
1323
|
|
|
@@ -1393,221 +1375,336 @@ def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typin
|
|
|
1393
1375
|
"""
|
|
1394
1376
|
...
|
|
1395
1377
|
|
|
1396
|
-
|
|
1378
|
+
@typing.overload
|
|
1379
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1397
1380
|
"""
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
Examples
|
|
1401
|
-
--------
|
|
1381
|
+
Specifies environment variables to be set prior to the execution of a step.
|
|
1402
1382
|
|
|
1403
|
-
```python
|
|
1404
|
-
# **Usage: creating references to models from the Hugging Face Hub that may be loaded in downstream steps**
|
|
1405
|
-
@huggingface_hub
|
|
1406
|
-
@step
|
|
1407
|
-
def pull_model_from_huggingface(self):
|
|
1408
|
-
# `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
|
|
1409
|
-
# and saves it in the backend storage based on the model's `repo_id`. If there exists a model
|
|
1410
|
-
# with the same `repo_id` in the backend storage, it will not download the model again. The return
|
|
1411
|
-
# value of the function is a reference to the model in the backend storage.
|
|
1412
|
-
# This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
|
|
1413
1383
|
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1384
|
+
Parameters
|
|
1385
|
+
----------
|
|
1386
|
+
vars : Dict[str, str], default {}
|
|
1387
|
+
Dictionary of environment variables to set.
|
|
1388
|
+
"""
|
|
1389
|
+
...
|
|
1390
|
+
|
|
1391
|
+
@typing.overload
|
|
1392
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1393
|
+
...
|
|
1394
|
+
|
|
1395
|
+
@typing.overload
|
|
1396
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1397
|
+
...
|
|
1398
|
+
|
|
1399
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
|
1400
|
+
"""
|
|
1401
|
+
Specifies environment variables to be set prior to the execution of a step.
|
|
1420
1402
|
|
|
1421
|
-
# **Usage: explicitly loading models at runtime from the Hugging Face Hub or from cache (from Metaflow's datastore)**
|
|
1422
|
-
@huggingface_hub
|
|
1423
|
-
@step
|
|
1424
|
-
def run_training(self):
|
|
1425
|
-
# Temporary directory (auto-cleaned on exit)
|
|
1426
|
-
with current.huggingface_hub.load(
|
|
1427
|
-
repo_id="google-bert/bert-base-uncased",
|
|
1428
|
-
allow_patterns=["*.bin"],
|
|
1429
|
-
) as local_path:
|
|
1430
|
-
# Use files under local_path
|
|
1431
|
-
train_model(local_path)
|
|
1432
|
-
...
|
|
1433
1403
|
|
|
1434
|
-
|
|
1404
|
+
Parameters
|
|
1405
|
+
----------
|
|
1406
|
+
vars : Dict[str, str], default {}
|
|
1407
|
+
Dictionary of environment variables to set.
|
|
1408
|
+
"""
|
|
1409
|
+
...
|
|
1410
|
+
|
|
1411
|
+
@typing.overload
|
|
1412
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1413
|
+
"""
|
|
1414
|
+
Specifies the PyPI packages for the step.
|
|
1435
1415
|
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1416
|
+
Information in this decorator will augment any
|
|
1417
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
|
1418
|
+
you can use `@pypi_base` to set packages required by all
|
|
1419
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
1440
1420
|
|
|
1441
|
-
@huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora", "/my-lora-directory")])
|
|
1442
|
-
@step
|
|
1443
|
-
def finetune_model(self):
|
|
1444
|
-
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
1445
|
-
# path_to_model will be /my-directory
|
|
1446
1421
|
|
|
1422
|
+
Parameters
|
|
1423
|
+
----------
|
|
1424
|
+
packages : Dict[str, str], default: {}
|
|
1425
|
+
Packages to use for this step. The key is the name of the package
|
|
1426
|
+
and the value is the version to use.
|
|
1427
|
+
python : str, optional, default: None
|
|
1428
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1429
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1430
|
+
"""
|
|
1431
|
+
...
|
|
1432
|
+
|
|
1433
|
+
@typing.overload
|
|
1434
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1435
|
+
...
|
|
1436
|
+
|
|
1437
|
+
@typing.overload
|
|
1438
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1439
|
+
...
|
|
1440
|
+
|
|
1441
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
|
1442
|
+
"""
|
|
1443
|
+
Specifies the PyPI packages for the step.
|
|
1447
1444
|
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
"repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
|
|
1453
|
-
},
|
|
1454
|
-
{
|
|
1455
|
-
"repo_id": "myorg/mistral-lora",
|
|
1456
|
-
"repo_type": "model",
|
|
1457
|
-
},
|
|
1458
|
-
])
|
|
1459
|
-
@step
|
|
1460
|
-
def finetune_model(self):
|
|
1461
|
-
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
1462
|
-
# path_to_model will be /my-directory
|
|
1463
|
-
```
|
|
1445
|
+
Information in this decorator will augment any
|
|
1446
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
|
1447
|
+
you can use `@pypi_base` to set packages required by all
|
|
1448
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
1464
1449
|
|
|
1465
1450
|
|
|
1466
1451
|
Parameters
|
|
1467
1452
|
----------
|
|
1468
|
-
|
|
1469
|
-
|
|
1453
|
+
packages : Dict[str, str], default: {}
|
|
1454
|
+
Packages to use for this step. The key is the name of the package
|
|
1455
|
+
and the value is the version to use.
|
|
1456
|
+
python : str, optional, default: None
|
|
1457
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1458
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1459
|
+
"""
|
|
1460
|
+
...
|
|
1461
|
+
|
|
1462
|
+
@typing.overload
|
|
1463
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1464
|
+
"""
|
|
1465
|
+
Specifies that the step will success under all circumstances.
|
|
1470
1466
|
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
Any repo downloaded under this scope will only be retrieved from the cache when the step runs under the same namespace in the same flow (at the same foreach index).
|
|
1467
|
+
The decorator will create an optional artifact, specified by `var`, which
|
|
1468
|
+
contains the exception raised. You can use it to detect the presence
|
|
1469
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
|
1470
|
+
are missing.
|
|
1476
1471
|
|
|
1477
|
-
- `flow`: All repos are cached under the flow, regardless of namespace.
|
|
1478
|
-
i.e., the cached path is derived solely from the flow name.
|
|
1479
|
-
When to use this mode: (1) Multiple users are executing the same flow and want shared access to the repos cached by the decorator. (2) Multiple versions of a flow are deployed, all needing access to the same repos cached by the decorator.
|
|
1480
1472
|
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1473
|
+
Parameters
|
|
1474
|
+
----------
|
|
1475
|
+
var : str, optional, default None
|
|
1476
|
+
Name of the artifact in which to store the caught exception.
|
|
1477
|
+
If not specified, the exception is not stored.
|
|
1478
|
+
print_exception : bool, default True
|
|
1479
|
+
Determines whether or not the exception is printed to
|
|
1480
|
+
stdout when caught.
|
|
1481
|
+
"""
|
|
1482
|
+
...
|
|
1483
|
+
|
|
1484
|
+
@typing.overload
|
|
1485
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1486
|
+
...
|
|
1487
|
+
|
|
1488
|
+
@typing.overload
|
|
1489
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1490
|
+
...
|
|
1491
|
+
|
|
1492
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
|
1493
|
+
"""
|
|
1494
|
+
Specifies that the step will success under all circumstances.
|
|
1500
1495
|
|
|
1501
|
-
|
|
1496
|
+
The decorator will create an optional artifact, specified by `var`, which
|
|
1497
|
+
contains the exception raised. You can use it to detect the presence
|
|
1498
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
|
1499
|
+
are missing.
|
|
1502
1500
|
|
|
1503
|
-
- If repo (model/dataset) is not found in the datastore:
|
|
1504
|
-
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
|
1505
|
-
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
|
1506
|
-
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
|
1507
1501
|
|
|
1508
|
-
|
|
1509
|
-
|
|
1502
|
+
Parameters
|
|
1503
|
+
----------
|
|
1504
|
+
var : str, optional, default None
|
|
1505
|
+
Name of the artifact in which to store the caught exception.
|
|
1506
|
+
If not specified, the exception is not stored.
|
|
1507
|
+
print_exception : bool, default True
|
|
1508
|
+
Determines whether or not the exception is printed to
|
|
1509
|
+
stdout when caught.
|
|
1510
1510
|
"""
|
|
1511
1511
|
...
|
|
1512
1512
|
|
|
1513
|
-
|
|
1514
|
-
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1513
|
+
def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
|
|
1515
1514
|
"""
|
|
1516
|
-
|
|
1515
|
+
Allows setting external datastores to save data for the
|
|
1516
|
+
`@checkpoint`/`@model`/`@huggingface_hub` decorators.
|
|
1517
1517
|
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
```
|
|
1521
|
-
or
|
|
1522
|
-
```
|
|
1523
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
|
1524
|
-
```
|
|
1525
|
-
This decorator respects the @project decorator and triggers the flow
|
|
1526
|
-
when upstream runs within the same namespace complete successfully
|
|
1518
|
+
This decorator is useful when users wish to save data to a different datastore
|
|
1519
|
+
than what is configured in Metaflow. This can be for variety of reasons:
|
|
1527
1520
|
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
or
|
|
1534
|
-
```
|
|
1535
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
|
1536
|
-
```
|
|
1521
|
+
1. Data security: The objects needs to be stored in a bucket (object storage) that is not accessible by other flows.
|
|
1522
|
+
2. Data Locality: The location where the task is executing is not located in the same region as the datastore.
|
|
1523
|
+
- Example: Metaflow datastore lives in US East, but the task is executing in Finland datacenters.
|
|
1524
|
+
3. Data Lifecycle Policies: The objects need to be archived / managed separately from the Metaflow managed objects.
|
|
1525
|
+
- Example: Flow is training very large models that need to be stored separately and will be deleted more aggressively than the Metaflow managed objects.
|
|
1537
1526
|
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
```
|
|
1541
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
|
1542
|
-
```
|
|
1527
|
+
Usage:
|
|
1528
|
+
----------
|
|
1543
1529
|
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1530
|
+
- Using a custom IAM role to access the datastore.
|
|
1531
|
+
|
|
1532
|
+
```python
|
|
1533
|
+
@with_artifact_store(
|
|
1534
|
+
type="s3",
|
|
1535
|
+
config=lambda: {
|
|
1536
|
+
"root": "s3://my-bucket-foo/path/to/root",
|
|
1537
|
+
"role_arn": ROLE,
|
|
1538
|
+
},
|
|
1539
|
+
)
|
|
1540
|
+
class MyFlow(FlowSpec):
|
|
1549
1541
|
|
|
1542
|
+
@checkpoint
|
|
1543
|
+
@step
|
|
1544
|
+
def start(self):
|
|
1545
|
+
with open("my_file.txt", "w") as f:
|
|
1546
|
+
f.write("Hello, World!")
|
|
1547
|
+
self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
|
|
1548
|
+
self.next(self.end)
|
|
1550
1549
|
|
|
1551
|
-
|
|
1550
|
+
```
|
|
1551
|
+
|
|
1552
|
+
- Using credentials to access the s3-compatible datastore.
|
|
1553
|
+
|
|
1554
|
+
```python
|
|
1555
|
+
@with_artifact_store(
|
|
1556
|
+
type="s3",
|
|
1557
|
+
config=lambda: {
|
|
1558
|
+
"root": "s3://my-bucket-foo/path/to/root",
|
|
1559
|
+
"client_params": {
|
|
1560
|
+
"aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
|
|
1561
|
+
"aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
|
|
1562
|
+
},
|
|
1563
|
+
},
|
|
1564
|
+
)
|
|
1565
|
+
class MyFlow(FlowSpec):
|
|
1566
|
+
|
|
1567
|
+
@checkpoint
|
|
1568
|
+
@step
|
|
1569
|
+
def start(self):
|
|
1570
|
+
with open("my_file.txt", "w") as f:
|
|
1571
|
+
f.write("Hello, World!")
|
|
1572
|
+
self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
|
|
1573
|
+
self.next(self.end)
|
|
1574
|
+
|
|
1575
|
+
```
|
|
1576
|
+
|
|
1577
|
+
- Accessing objects stored in external datastores after task execution.
|
|
1578
|
+
|
|
1579
|
+
```python
|
|
1580
|
+
run = Run("CheckpointsTestsFlow/8992")
|
|
1581
|
+
with artifact_store_from(run=run, config={
|
|
1582
|
+
"client_params": {
|
|
1583
|
+
"aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
|
|
1584
|
+
"aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
|
|
1585
|
+
},
|
|
1586
|
+
}):
|
|
1587
|
+
with Checkpoint() as cp:
|
|
1588
|
+
latest = cp.list(
|
|
1589
|
+
task=run["start"].task
|
|
1590
|
+
)[0]
|
|
1591
|
+
print(latest)
|
|
1592
|
+
cp.load(
|
|
1593
|
+
latest,
|
|
1594
|
+
"test-checkpoints"
|
|
1595
|
+
)
|
|
1596
|
+
|
|
1597
|
+
task = Task("TorchTuneFlow/8484/train/53673")
|
|
1598
|
+
with artifact_store_from(run=run, config={
|
|
1599
|
+
"client_params": {
|
|
1600
|
+
"aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
|
|
1601
|
+
"aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
|
|
1602
|
+
},
|
|
1603
|
+
}):
|
|
1604
|
+
load_model(
|
|
1605
|
+
task.data.model_ref,
|
|
1606
|
+
"test-models"
|
|
1607
|
+
)
|
|
1608
|
+
```
|
|
1609
|
+
Parameters:
|
|
1552
1610
|
----------
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1611
|
+
|
|
1612
|
+
type: str
|
|
1613
|
+
The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
|
|
1614
|
+
|
|
1615
|
+
config: dict or Callable
|
|
1616
|
+
Dictionary of configuration options for the datastore. The following keys are required:
|
|
1617
|
+
- root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
|
|
1618
|
+
- example: 's3://bucket-name/path/to/root'
|
|
1619
|
+
- example: 'gs://bucket-name/path/to/root'
|
|
1620
|
+
- example: 'https://myblockacc.blob.core.windows.net/metaflow/'
|
|
1621
|
+
- role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
|
|
1622
|
+
- session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
|
|
1623
|
+
- client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
|
|
1559
1624
|
"""
|
|
1560
1625
|
...
|
|
1561
1626
|
|
|
1562
|
-
|
|
1563
|
-
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1564
|
-
...
|
|
1565
|
-
|
|
1566
|
-
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
|
1627
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1567
1628
|
"""
|
|
1568
|
-
|
|
1629
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
|
1630
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
|
1569
1631
|
|
|
1570
|
-
```
|
|
1571
|
-
@trigger_on_finish(flow='FooFlow')
|
|
1572
|
-
```
|
|
1573
|
-
or
|
|
1574
|
-
```
|
|
1575
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
|
1576
|
-
```
|
|
1577
|
-
This decorator respects the @project decorator and triggers the flow
|
|
1578
|
-
when upstream runs within the same namespace complete successfully
|
|
1579
1632
|
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1633
|
+
Parameters
|
|
1634
|
+
----------
|
|
1635
|
+
timeout : int
|
|
1636
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
|
1637
|
+
poke_interval : int
|
|
1638
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
|
1639
|
+
mode : str
|
|
1640
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
|
1641
|
+
exponential_backoff : bool
|
|
1642
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
|
1643
|
+
pool : str
|
|
1644
|
+
the slot pool this task should run in,
|
|
1645
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
|
1646
|
+
soft_fail : bool
|
|
1647
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
|
1648
|
+
name : str
|
|
1649
|
+
Name of the sensor on Airflow
|
|
1650
|
+
description : str
|
|
1651
|
+
Description of sensor in the Airflow UI
|
|
1652
|
+
external_dag_id : str
|
|
1653
|
+
The dag_id that contains the task you want to wait for.
|
|
1654
|
+
external_task_ids : List[str]
|
|
1655
|
+
The list of task_ids that you want to wait for.
|
|
1656
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
|
1657
|
+
allowed_states : List[str]
|
|
1658
|
+
Iterable of allowed states, (Default: ['success'])
|
|
1659
|
+
failed_states : List[str]
|
|
1660
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
|
1661
|
+
execution_delta : datetime.timedelta
|
|
1662
|
+
time difference with the previous execution to look at,
|
|
1663
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
|
1664
|
+
check_existence: bool
|
|
1665
|
+
Set to True to check if the external task exists or check if
|
|
1666
|
+
the DAG to wait for exists. (Default: True)
|
|
1667
|
+
"""
|
|
1668
|
+
...
|
|
1669
|
+
|
|
1670
|
+
@typing.overload
|
|
1671
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1672
|
+
"""
|
|
1673
|
+
Specifies the PyPI packages for all steps of the flow.
|
|
1589
1674
|
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
```
|
|
1593
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
|
1594
|
-
```
|
|
1675
|
+
Use `@pypi_base` to set common packages required by all
|
|
1676
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
1595
1677
|
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1678
|
+
Parameters
|
|
1679
|
+
----------
|
|
1680
|
+
packages : Dict[str, str], default: {}
|
|
1681
|
+
Packages to use for this flow. The key is the name of the package
|
|
1682
|
+
and the value is the version to use.
|
|
1683
|
+
python : str, optional, default: None
|
|
1684
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1685
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1686
|
+
"""
|
|
1687
|
+
...
|
|
1688
|
+
|
|
1689
|
+
@typing.overload
|
|
1690
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1691
|
+
...
|
|
1692
|
+
|
|
1693
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
|
1694
|
+
"""
|
|
1695
|
+
Specifies the PyPI packages for all steps of the flow.
|
|
1601
1696
|
|
|
1697
|
+
Use `@pypi_base` to set common packages required by all
|
|
1698
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
1602
1699
|
|
|
1603
1700
|
Parameters
|
|
1604
1701
|
----------
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
|
|
1702
|
+
packages : Dict[str, str], default: {}
|
|
1703
|
+
Packages to use for this flow. The key is the name of the package
|
|
1704
|
+
and the value is the version to use.
|
|
1705
|
+
python : str, optional, default: None
|
|
1706
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1707
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1611
1708
|
"""
|
|
1612
1709
|
...
|
|
1613
1710
|
|
|
@@ -1690,53 +1787,53 @@ def project(*, name: str, branch: typing.Optional[str] = None, production: bool
|
|
|
1690
1787
|
...
|
|
1691
1788
|
|
|
1692
1789
|
@typing.overload
|
|
1693
|
-
def
|
|
1790
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1694
1791
|
"""
|
|
1695
|
-
Specifies the
|
|
1696
|
-
|
|
1697
|
-
Use `@conda_base` to set common libraries required by all
|
|
1698
|
-
steps and use `@conda` to specify step-specific additions.
|
|
1792
|
+
Specifies the times when the flow should be run when running on a
|
|
1793
|
+
production scheduler.
|
|
1699
1794
|
|
|
1700
1795
|
|
|
1701
1796
|
Parameters
|
|
1702
1797
|
----------
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1798
|
+
hourly : bool, default False
|
|
1799
|
+
Run the workflow hourly.
|
|
1800
|
+
daily : bool, default True
|
|
1801
|
+
Run the workflow daily.
|
|
1802
|
+
weekly : bool, default False
|
|
1803
|
+
Run the workflow weekly.
|
|
1804
|
+
cron : str, optional, default None
|
|
1805
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
|
1806
|
+
specified by this expression.
|
|
1807
|
+
timezone : str, optional, default None
|
|
1808
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
|
1809
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
|
1713
1810
|
"""
|
|
1714
1811
|
...
|
|
1715
1812
|
|
|
1716
1813
|
@typing.overload
|
|
1717
|
-
def
|
|
1814
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1718
1815
|
...
|
|
1719
1816
|
|
|
1720
|
-
def
|
|
1817
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
|
1721
1818
|
"""
|
|
1722
|
-
Specifies the
|
|
1723
|
-
|
|
1724
|
-
Use `@conda_base` to set common libraries required by all
|
|
1725
|
-
steps and use `@conda` to specify step-specific additions.
|
|
1819
|
+
Specifies the times when the flow should be run when running on a
|
|
1820
|
+
production scheduler.
|
|
1726
1821
|
|
|
1727
1822
|
|
|
1728
1823
|
Parameters
|
|
1729
1824
|
----------
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1825
|
+
hourly : bool, default False
|
|
1826
|
+
Run the workflow hourly.
|
|
1827
|
+
daily : bool, default True
|
|
1828
|
+
Run the workflow daily.
|
|
1829
|
+
weekly : bool, default False
|
|
1830
|
+
Run the workflow weekly.
|
|
1831
|
+
cron : str, optional, default None
|
|
1832
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
|
1833
|
+
specified by this expression.
|
|
1834
|
+
timezone : str, optional, default None
|
|
1835
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
|
1836
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
|
1740
1837
|
"""
|
|
1741
1838
|
...
|
|
1742
1839
|
|
|
@@ -1766,319 +1863,222 @@ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = No
|
|
|
1766
1863
|
|
|
1767
1864
|
'parameters' can also be a list of strings and tuples like so:
|
|
1768
1865
|
```
|
|
1769
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
1770
|
-
```
|
|
1771
|
-
This is equivalent to:
|
|
1772
|
-
```
|
|
1773
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
|
1774
|
-
```
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
Parameters
|
|
1778
|
-
----------
|
|
1779
|
-
event : Union[str, Dict[str, Any]], optional, default None
|
|
1780
|
-
Event dependency for this flow.
|
|
1781
|
-
events : List[Union[str, Dict[str, Any]]], default []
|
|
1782
|
-
Events dependency for this flow.
|
|
1783
|
-
options : Dict[str, Any], default {}
|
|
1784
|
-
Backend-specific configuration for tuning eventing behavior.
|
|
1785
|
-
"""
|
|
1786
|
-
...
|
|
1787
|
-
|
|
1788
|
-
@typing.overload
|
|
1789
|
-
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1790
|
-
...
|
|
1791
|
-
|
|
1792
|
-
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
|
1793
|
-
"""
|
|
1794
|
-
Specifies the event(s) that this flow depends on.
|
|
1795
|
-
|
|
1796
|
-
```
|
|
1797
|
-
@trigger(event='foo')
|
|
1798
|
-
```
|
|
1799
|
-
or
|
|
1800
|
-
```
|
|
1801
|
-
@trigger(events=['foo', 'bar'])
|
|
1802
|
-
```
|
|
1803
|
-
|
|
1804
|
-
Additionally, you can specify the parameter mappings
|
|
1805
|
-
to map event payload to Metaflow parameters for the flow.
|
|
1806
|
-
```
|
|
1807
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
|
1808
|
-
```
|
|
1809
|
-
or
|
|
1810
|
-
```
|
|
1811
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
|
1812
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
|
1813
|
-
```
|
|
1814
|
-
|
|
1815
|
-
'parameters' can also be a list of strings and tuples like so:
|
|
1816
|
-
```
|
|
1817
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
1818
|
-
```
|
|
1819
|
-
This is equivalent to:
|
|
1820
|
-
```
|
|
1821
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
|
1822
|
-
```
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
Parameters
|
|
1826
|
-
----------
|
|
1827
|
-
event : Union[str, Dict[str, Any]], optional, default None
|
|
1828
|
-
Event dependency for this flow.
|
|
1829
|
-
events : List[Union[str, Dict[str, Any]]], default []
|
|
1830
|
-
Events dependency for this flow.
|
|
1831
|
-
options : Dict[str, Any], default {}
|
|
1832
|
-
Backend-specific configuration for tuning eventing behavior.
|
|
1833
|
-
"""
|
|
1834
|
-
...
|
|
1835
|
-
|
|
1836
|
-
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1837
|
-
"""
|
|
1838
|
-
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
|
1839
|
-
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
Parameters
|
|
1843
|
-
----------
|
|
1844
|
-
timeout : int
|
|
1845
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
|
1846
|
-
poke_interval : int
|
|
1847
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
|
1848
|
-
mode : str
|
|
1849
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
|
1850
|
-
exponential_backoff : bool
|
|
1851
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
|
1852
|
-
pool : str
|
|
1853
|
-
the slot pool this task should run in,
|
|
1854
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
|
1855
|
-
soft_fail : bool
|
|
1856
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
|
1857
|
-
name : str
|
|
1858
|
-
Name of the sensor on Airflow
|
|
1859
|
-
description : str
|
|
1860
|
-
Description of sensor in the Airflow UI
|
|
1861
|
-
external_dag_id : str
|
|
1862
|
-
The dag_id that contains the task you want to wait for.
|
|
1863
|
-
external_task_ids : List[str]
|
|
1864
|
-
The list of task_ids that you want to wait for.
|
|
1865
|
-
If None (default value) the sensor waits for the DAG. (Default: None)
|
|
1866
|
-
allowed_states : List[str]
|
|
1867
|
-
Iterable of allowed states, (Default: ['success'])
|
|
1868
|
-
failed_states : List[str]
|
|
1869
|
-
Iterable of failed or dis-allowed states. (Default: None)
|
|
1870
|
-
execution_delta : datetime.timedelta
|
|
1871
|
-
time difference with the previous execution to look at,
|
|
1872
|
-
the default is the same logical date as the current task or DAG. (Default: None)
|
|
1873
|
-
check_existence: bool
|
|
1874
|
-
Set to True to check if the external task exists or check if
|
|
1875
|
-
the DAG to wait for exists. (Default: True)
|
|
1876
|
-
"""
|
|
1877
|
-
...
|
|
1878
|
-
|
|
1879
|
-
def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
|
|
1880
|
-
"""
|
|
1881
|
-
Allows setting external datastores to save data for the
|
|
1882
|
-
`@checkpoint`/`@model`/`@huggingface_hub` decorators.
|
|
1883
|
-
|
|
1884
|
-
This decorator is useful when users wish to save data to a different datastore
|
|
1885
|
-
than what is configured in Metaflow. This can be for variety of reasons:
|
|
1886
|
-
|
|
1887
|
-
1. Data security: The objects needs to be stored in a bucket (object storage) that is not accessible by other flows.
|
|
1888
|
-
2. Data Locality: The location where the task is executing is not located in the same region as the datastore.
|
|
1889
|
-
- Example: Metaflow datastore lives in US East, but the task is executing in Finland datacenters.
|
|
1890
|
-
3. Data Lifecycle Policies: The objects need to be archived / managed separately from the Metaflow managed objects.
|
|
1891
|
-
- Example: Flow is training very large models that need to be stored separately and will be deleted more aggressively than the Metaflow managed objects.
|
|
1892
|
-
|
|
1893
|
-
Usage:
|
|
1894
|
-
----------
|
|
1895
|
-
|
|
1896
|
-
- Using a custom IAM role to access the datastore.
|
|
1897
|
-
|
|
1898
|
-
```python
|
|
1899
|
-
@with_artifact_store(
|
|
1900
|
-
type="s3",
|
|
1901
|
-
config=lambda: {
|
|
1902
|
-
"root": "s3://my-bucket-foo/path/to/root",
|
|
1903
|
-
"role_arn": ROLE,
|
|
1904
|
-
},
|
|
1905
|
-
)
|
|
1906
|
-
class MyFlow(FlowSpec):
|
|
1907
|
-
|
|
1908
|
-
@checkpoint
|
|
1909
|
-
@step
|
|
1910
|
-
def start(self):
|
|
1911
|
-
with open("my_file.txt", "w") as f:
|
|
1912
|
-
f.write("Hello, World!")
|
|
1913
|
-
self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
|
|
1914
|
-
self.next(self.end)
|
|
1915
|
-
|
|
1916
|
-
```
|
|
1917
|
-
|
|
1918
|
-
- Using credentials to access the s3-compatible datastore.
|
|
1919
|
-
|
|
1920
|
-
```python
|
|
1921
|
-
@with_artifact_store(
|
|
1922
|
-
type="s3",
|
|
1923
|
-
config=lambda: {
|
|
1924
|
-
"root": "s3://my-bucket-foo/path/to/root",
|
|
1925
|
-
"client_params": {
|
|
1926
|
-
"aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
|
|
1927
|
-
"aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
|
|
1928
|
-
},
|
|
1929
|
-
},
|
|
1930
|
-
)
|
|
1931
|
-
class MyFlow(FlowSpec):
|
|
1866
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
1867
|
+
```
|
|
1868
|
+
This is equivalent to:
|
|
1869
|
+
```
|
|
1870
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
|
1871
|
+
```
|
|
1932
1872
|
|
|
1933
|
-
@checkpoint
|
|
1934
|
-
@step
|
|
1935
|
-
def start(self):
|
|
1936
|
-
with open("my_file.txt", "w") as f:
|
|
1937
|
-
f.write("Hello, World!")
|
|
1938
|
-
self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
|
|
1939
|
-
self.next(self.end)
|
|
1940
1873
|
|
|
1941
|
-
|
|
1874
|
+
Parameters
|
|
1875
|
+
----------
|
|
1876
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
|
1877
|
+
Event dependency for this flow.
|
|
1878
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
|
1879
|
+
Events dependency for this flow.
|
|
1880
|
+
options : Dict[str, Any], default {}
|
|
1881
|
+
Backend-specific configuration for tuning eventing behavior.
|
|
1882
|
+
"""
|
|
1883
|
+
...
|
|
1884
|
+
|
|
1885
|
+
@typing.overload
|
|
1886
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1887
|
+
...
|
|
1888
|
+
|
|
1889
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
|
1890
|
+
"""
|
|
1891
|
+
Specifies the event(s) that this flow depends on.
|
|
1942
1892
|
|
|
1943
|
-
|
|
1893
|
+
```
|
|
1894
|
+
@trigger(event='foo')
|
|
1895
|
+
```
|
|
1896
|
+
or
|
|
1897
|
+
```
|
|
1898
|
+
@trigger(events=['foo', 'bar'])
|
|
1899
|
+
```
|
|
1944
1900
|
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
-
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
task=run["start"].task
|
|
1956
|
-
)[0]
|
|
1957
|
-
print(latest)
|
|
1958
|
-
cp.load(
|
|
1959
|
-
latest,
|
|
1960
|
-
"test-checkpoints"
|
|
1961
|
-
)
|
|
1901
|
+
Additionally, you can specify the parameter mappings
|
|
1902
|
+
to map event payload to Metaflow parameters for the flow.
|
|
1903
|
+
```
|
|
1904
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
|
1905
|
+
```
|
|
1906
|
+
or
|
|
1907
|
+
```
|
|
1908
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
|
1909
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
|
1910
|
+
```
|
|
1962
1911
|
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
task.data.model_ref,
|
|
1972
|
-
"test-models"
|
|
1973
|
-
)
|
|
1974
|
-
```
|
|
1975
|
-
Parameters:
|
|
1976
|
-
----------
|
|
1912
|
+
'parameters' can also be a list of strings and tuples like so:
|
|
1913
|
+
```
|
|
1914
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
1915
|
+
```
|
|
1916
|
+
This is equivalent to:
|
|
1917
|
+
```
|
|
1918
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
|
1919
|
+
```
|
|
1977
1920
|
|
|
1978
|
-
type: str
|
|
1979
|
-
The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
|
|
1980
1921
|
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
-
|
|
1989
|
-
- client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
|
|
1922
|
+
Parameters
|
|
1923
|
+
----------
|
|
1924
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
|
1925
|
+
Event dependency for this flow.
|
|
1926
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
|
1927
|
+
Events dependency for this flow.
|
|
1928
|
+
options : Dict[str, Any], default {}
|
|
1929
|
+
Backend-specific configuration for tuning eventing behavior.
|
|
1990
1930
|
"""
|
|
1991
1931
|
...
|
|
1992
1932
|
|
|
1993
1933
|
@typing.overload
|
|
1994
|
-
def
|
|
1934
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1995
1935
|
"""
|
|
1996
|
-
Specifies the
|
|
1936
|
+
Specifies the Conda environment for all steps of the flow.
|
|
1937
|
+
|
|
1938
|
+
Use `@conda_base` to set common libraries required by all
|
|
1939
|
+
steps and use `@conda` to specify step-specific additions.
|
|
1997
1940
|
|
|
1998
|
-
Use `@pypi_base` to set common packages required by all
|
|
1999
|
-
steps and use `@pypi` to specify step-specific overrides.
|
|
2000
1941
|
|
|
2001
1942
|
Parameters
|
|
2002
1943
|
----------
|
|
2003
|
-
packages : Dict[str, str], default
|
|
1944
|
+
packages : Dict[str, str], default {}
|
|
2004
1945
|
Packages to use for this flow. The key is the name of the package
|
|
2005
1946
|
and the value is the version to use.
|
|
2006
|
-
|
|
1947
|
+
libraries : Dict[str, str], default {}
|
|
1948
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1949
|
+
python : str, optional, default None
|
|
2007
1950
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
2008
1951
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1952
|
+
disabled : bool, default False
|
|
1953
|
+
If set to True, disables Conda.
|
|
2009
1954
|
"""
|
|
2010
1955
|
...
|
|
2011
1956
|
|
|
2012
1957
|
@typing.overload
|
|
2013
|
-
def
|
|
1958
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
2014
1959
|
...
|
|
2015
1960
|
|
|
2016
|
-
def
|
|
1961
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
|
2017
1962
|
"""
|
|
2018
|
-
Specifies the
|
|
1963
|
+
Specifies the Conda environment for all steps of the flow.
|
|
1964
|
+
|
|
1965
|
+
Use `@conda_base` to set common libraries required by all
|
|
1966
|
+
steps and use `@conda` to specify step-specific additions.
|
|
2019
1967
|
|
|
2020
|
-
Use `@pypi_base` to set common packages required by all
|
|
2021
|
-
steps and use `@pypi` to specify step-specific overrides.
|
|
2022
1968
|
|
|
2023
1969
|
Parameters
|
|
2024
1970
|
----------
|
|
2025
|
-
packages : Dict[str, str], default
|
|
1971
|
+
packages : Dict[str, str], default {}
|
|
2026
1972
|
Packages to use for this flow. The key is the name of the package
|
|
2027
1973
|
and the value is the version to use.
|
|
2028
|
-
|
|
1974
|
+
libraries : Dict[str, str], default {}
|
|
1975
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1976
|
+
python : str, optional, default None
|
|
2029
1977
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
2030
1978
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1979
|
+
disabled : bool, default False
|
|
1980
|
+
If set to True, disables Conda.
|
|
2031
1981
|
"""
|
|
2032
1982
|
...
|
|
2033
1983
|
|
|
2034
1984
|
@typing.overload
|
|
2035
|
-
def
|
|
1985
|
+
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
2036
1986
|
"""
|
|
2037
|
-
Specifies the
|
|
2038
|
-
|
|
1987
|
+
Specifies the flow(s) that this flow depends on.
|
|
1988
|
+
|
|
1989
|
+
```
|
|
1990
|
+
@trigger_on_finish(flow='FooFlow')
|
|
1991
|
+
```
|
|
1992
|
+
or
|
|
1993
|
+
```
|
|
1994
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
|
1995
|
+
```
|
|
1996
|
+
This decorator respects the @project decorator and triggers the flow
|
|
1997
|
+
when upstream runs within the same namespace complete successfully
|
|
1998
|
+
|
|
1999
|
+
Additionally, you can specify project aware upstream flow dependencies
|
|
2000
|
+
by specifying the fully qualified project_flow_name.
|
|
2001
|
+
```
|
|
2002
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
|
2003
|
+
```
|
|
2004
|
+
or
|
|
2005
|
+
```
|
|
2006
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
|
2007
|
+
```
|
|
2008
|
+
|
|
2009
|
+
You can also specify just the project or project branch (other values will be
|
|
2010
|
+
inferred from the current project or project branch):
|
|
2011
|
+
```
|
|
2012
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
|
2013
|
+
```
|
|
2014
|
+
|
|
2015
|
+
Note that `branch` is typically one of:
|
|
2016
|
+
- `prod`
|
|
2017
|
+
- `user.bob`
|
|
2018
|
+
- `test.my_experiment`
|
|
2019
|
+
- `prod.staging`
|
|
2039
2020
|
|
|
2040
2021
|
|
|
2041
2022
|
Parameters
|
|
2042
2023
|
----------
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
cron : str, optional, default None
|
|
2050
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
|
2051
|
-
specified by this expression.
|
|
2052
|
-
timezone : str, optional, default None
|
|
2053
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
|
2054
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
|
2024
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
|
2025
|
+
Upstream flow dependency for this flow.
|
|
2026
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
|
2027
|
+
Upstream flow dependencies for this flow.
|
|
2028
|
+
options : Dict[str, Any], default {}
|
|
2029
|
+
Backend-specific configuration for tuning eventing behavior.
|
|
2055
2030
|
"""
|
|
2056
2031
|
...
|
|
2057
2032
|
|
|
2058
2033
|
@typing.overload
|
|
2059
|
-
def
|
|
2034
|
+
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
2060
2035
|
...
|
|
2061
2036
|
|
|
2062
|
-
def
|
|
2037
|
+
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
|
2063
2038
|
"""
|
|
2064
|
-
Specifies the
|
|
2065
|
-
|
|
2039
|
+
Specifies the flow(s) that this flow depends on.
|
|
2040
|
+
|
|
2041
|
+
```
|
|
2042
|
+
@trigger_on_finish(flow='FooFlow')
|
|
2043
|
+
```
|
|
2044
|
+
or
|
|
2045
|
+
```
|
|
2046
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
|
2047
|
+
```
|
|
2048
|
+
This decorator respects the @project decorator and triggers the flow
|
|
2049
|
+
when upstream runs within the same namespace complete successfully
|
|
2050
|
+
|
|
2051
|
+
Additionally, you can specify project aware upstream flow dependencies
|
|
2052
|
+
by specifying the fully qualified project_flow_name.
|
|
2053
|
+
```
|
|
2054
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
|
2055
|
+
```
|
|
2056
|
+
or
|
|
2057
|
+
```
|
|
2058
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
|
2059
|
+
```
|
|
2060
|
+
|
|
2061
|
+
You can also specify just the project or project branch (other values will be
|
|
2062
|
+
inferred from the current project or project branch):
|
|
2063
|
+
```
|
|
2064
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
|
2065
|
+
```
|
|
2066
|
+
|
|
2067
|
+
Note that `branch` is typically one of:
|
|
2068
|
+
- `prod`
|
|
2069
|
+
- `user.bob`
|
|
2070
|
+
- `test.my_experiment`
|
|
2071
|
+
- `prod.staging`
|
|
2066
2072
|
|
|
2067
2073
|
|
|
2068
2074
|
Parameters
|
|
2069
2075
|
----------
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
cron : str, optional, default None
|
|
2077
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
|
2078
|
-
specified by this expression.
|
|
2079
|
-
timezone : str, optional, default None
|
|
2080
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
|
2081
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
|
2076
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
|
2077
|
+
Upstream flow dependency for this flow.
|
|
2078
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
|
2079
|
+
Upstream flow dependencies for this flow.
|
|
2080
|
+
options : Dict[str, Any], default {}
|
|
2081
|
+
Backend-specific configuration for tuning eventing behavior.
|
|
2082
2082
|
"""
|
|
2083
2083
|
...
|
|
2084
2084
|
|