ob-metaflow-stubs 6.0.10.11__py2.py3-none-any.whl → 6.0.10.13__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +1006 -1005
- metaflow-stubs/cards.pyi +5 -2
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/cli_components/__init__.pyi +2 -2
- metaflow-stubs/cli_components/utils.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +2 -2
- metaflow-stubs/client/core.pyi +7 -7
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +6 -6
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +12 -5
- metaflow-stubs/meta_files.pyi +2 -2
- metaflow-stubs/metadata_provider/__init__.pyi +2 -2
- metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
- metaflow-stubs/metadata_provider/metadata.pyi +3 -3
- metaflow-stubs/metadata_provider/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +52 -52
- metaflow-stubs/metaflow_git.pyi +2 -2
- metaflow-stubs/mf_extensions/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +5 -5
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +5 -5
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/cards/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/cards/hf_hub_card.pyi +93 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +6 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +5 -5
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +6 -6
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/ob_internal.pyi +2 -2
- metaflow-stubs/packaging_sys/__init__.pyi +8 -8
- metaflow-stubs/packaging_sys/backend.pyi +2 -2
- metaflow-stubs/packaging_sys/distribution_support.pyi +4 -4
- metaflow-stubs/packaging_sys/tar_backend.pyi +5 -5
- metaflow-stubs/packaging_sys/utils.pyi +2 -2
- metaflow-stubs/packaging_sys/v1.pyi +4 -4
- metaflow-stubs/parameters.pyi +3 -3
- metaflow-stubs/plugins/__init__.pyi +14 -13
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
- metaflow-stubs/plugins/argo/exit_hooks.pyi +3 -3
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +6 -6
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +3 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +168 -4
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/json_viewer.pyi +119 -0
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +3 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +2 -2
- metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
- metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
- metaflow-stubs/plugins/ollama/__init__.pyi +3 -3
- metaflow-stubs/plugins/optuna/__init__.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/parsers.pyi +25 -0
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
- metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
- metaflow-stubs/plugins/secrets/utils.pyi +2 -2
- metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
- metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
- metaflow-stubs/plugins/uv/__init__.pyi +2 -2
- metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +5 -5
- metaflow-stubs/runner/deployer_impl.pyi +2 -2
- metaflow-stubs/runner/metaflow_runner.pyi +4 -4
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +3 -3
- metaflow-stubs/system/__init__.pyi +2 -2
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/user_configs/__init__.pyi +2 -2
- metaflow-stubs/user_configs/config_options.pyi +3 -3
- metaflow-stubs/user_configs/config_parameters.pyi +6 -6
- metaflow-stubs/user_decorators/__init__.pyi +2 -2
- metaflow-stubs/user_decorators/common.pyi +2 -2
- metaflow-stubs/user_decorators/mutable_flow.pyi +4 -4
- metaflow-stubs/user_decorators/mutable_step.pyi +5 -5
- metaflow-stubs/user_decorators/user_flow_decorator.pyi +5 -5
- metaflow-stubs/user_decorators/user_step_decorator.pyi +5 -5
- {ob_metaflow_stubs-6.0.10.11.dist-info → ob_metaflow_stubs-6.0.10.13.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.10.13.dist-info/RECORD +266 -0
- ob_metaflow_stubs-6.0.10.11.dist-info/RECORD +0 -262
- {ob_metaflow_stubs-6.0.10.11.dist-info → ob_metaflow_stubs-6.0.10.13.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.10.11.dist-info → ob_metaflow_stubs-6.0.10.13.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
######################################################################################################
|
|
2
2
|
# Auto-generated Metaflow stub file #
|
|
3
|
-
# MF version: 2.18.
|
|
4
|
-
# Generated on 2025-
|
|
3
|
+
# MF version: 2.18.10.1+obcheckpoint(0.2.8);ob(v1) #
|
|
4
|
+
# Generated on 2025-10-02T16:55:23.853861 #
|
|
5
5
|
######################################################################################################
|
|
6
6
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import typing
|
|
10
10
|
if typing.TYPE_CHECKING:
|
|
11
|
-
import typing
|
|
12
11
|
import datetime
|
|
12
|
+
import typing
|
|
13
13
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
|
14
14
|
StepFlag = typing.NewType("StepFlag", bool)
|
|
15
15
|
|
|
@@ -39,18 +39,19 @@ from .user_decorators.user_step_decorator import UserStepDecorator as UserStepDe
|
|
|
39
39
|
from .user_decorators.user_step_decorator import StepMutator as StepMutator
|
|
40
40
|
from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
|
|
41
41
|
from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
|
|
42
|
+
from . import tuple_util as tuple_util
|
|
42
43
|
from . import cards as cards
|
|
43
44
|
from . import metaflow_git as metaflow_git
|
|
44
|
-
from . import tuple_util as tuple_util
|
|
45
45
|
from . import events as events
|
|
46
46
|
from . import runner as runner
|
|
47
47
|
from . import plugins as plugins
|
|
48
48
|
from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
|
|
49
49
|
from . import includefile as includefile
|
|
50
50
|
from .includefile import IncludeFile as IncludeFile
|
|
51
|
-
from .plugins.
|
|
52
|
-
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
|
51
|
+
from .plugins.parsers import yaml_parser as yaml_parser
|
|
53
52
|
from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
|
|
53
|
+
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
|
54
|
+
from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
|
|
54
55
|
from . import client as client
|
|
55
56
|
from .client.core import namespace as namespace
|
|
56
57
|
from .client.core import get_namespace as get_namespace
|
|
@@ -224,308 +225,286 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
|
224
225
|
...
|
|
225
226
|
|
|
226
227
|
@typing.overload
|
|
227
|
-
def
|
|
228
|
+
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
228
229
|
"""
|
|
229
|
-
|
|
230
|
-
|
|
230
|
+
Enables loading / saving of models within a step.
|
|
231
|
+
|
|
232
|
+
> Examples
|
|
233
|
+
- Saving Models
|
|
234
|
+
```python
|
|
235
|
+
@model
|
|
236
|
+
@step
|
|
237
|
+
def train(self):
|
|
238
|
+
# current.model.save returns a dictionary reference to the model saved
|
|
239
|
+
self.my_model = current.model.save(
|
|
240
|
+
path_to_my_model,
|
|
241
|
+
label="my_model",
|
|
242
|
+
metadata={
|
|
243
|
+
"epochs": 10,
|
|
244
|
+
"batch-size": 32,
|
|
245
|
+
"learning-rate": 0.001,
|
|
246
|
+
}
|
|
247
|
+
)
|
|
248
|
+
self.next(self.test)
|
|
249
|
+
|
|
250
|
+
@model(load="my_model")
|
|
251
|
+
@step
|
|
252
|
+
def test(self):
|
|
253
|
+
# `current.model.loaded` returns a dictionary of the loaded models
|
|
254
|
+
# where the key is the name of the artifact and the value is the path to the model
|
|
255
|
+
print(os.listdir(current.model.loaded["my_model"]))
|
|
256
|
+
self.next(self.end)
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
- Loading models
|
|
260
|
+
```python
|
|
261
|
+
@step
|
|
262
|
+
def train(self):
|
|
263
|
+
# current.model.load returns the path to the model loaded
|
|
264
|
+
checkpoint_path = current.model.load(
|
|
265
|
+
self.checkpoint_key,
|
|
266
|
+
)
|
|
267
|
+
model_path = current.model.load(
|
|
268
|
+
self.model,
|
|
269
|
+
)
|
|
270
|
+
self.next(self.test)
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
Parameters
|
|
275
|
+
----------
|
|
276
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
|
277
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
|
278
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
|
279
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
|
280
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
|
281
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
|
282
|
+
|
|
283
|
+
temp_dir_root : str, default: None
|
|
284
|
+
The root directory under which `current.model.loaded` will store loaded models
|
|
231
285
|
"""
|
|
232
286
|
...
|
|
233
287
|
|
|
234
288
|
@typing.overload
|
|
235
|
-
def
|
|
289
|
+
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
236
290
|
...
|
|
237
291
|
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
A simple decorator that demonstrates using CardDecoratorInjector
|
|
241
|
-
to inject a card and render simple markdown content.
|
|
242
|
-
"""
|
|
292
|
+
@typing.overload
|
|
293
|
+
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
243
294
|
...
|
|
244
295
|
|
|
245
|
-
|
|
246
|
-
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
296
|
+
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
|
247
297
|
"""
|
|
248
|
-
|
|
298
|
+
Enables loading / saving of models within a step.
|
|
249
299
|
|
|
250
|
-
|
|
300
|
+
> Examples
|
|
301
|
+
- Saving Models
|
|
302
|
+
```python
|
|
303
|
+
@model
|
|
304
|
+
@step
|
|
305
|
+
def train(self):
|
|
306
|
+
# current.model.save returns a dictionary reference to the model saved
|
|
307
|
+
self.my_model = current.model.save(
|
|
308
|
+
path_to_my_model,
|
|
309
|
+
label="my_model",
|
|
310
|
+
metadata={
|
|
311
|
+
"epochs": 10,
|
|
312
|
+
"batch-size": 32,
|
|
313
|
+
"learning-rate": 0.001,
|
|
314
|
+
}
|
|
315
|
+
)
|
|
316
|
+
self.next(self.test)
|
|
317
|
+
|
|
318
|
+
@model(load="my_model")
|
|
319
|
+
@step
|
|
320
|
+
def test(self):
|
|
321
|
+
# `current.model.loaded` returns a dictionary of the loaded models
|
|
322
|
+
# where the key is the name of the artifact and the value is the path to the model
|
|
323
|
+
print(os.listdir(current.model.loaded["my_model"]))
|
|
324
|
+
self.next(self.end)
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
- Loading models
|
|
328
|
+
```python
|
|
329
|
+
@step
|
|
330
|
+
def train(self):
|
|
331
|
+
# current.model.load returns the path to the model loaded
|
|
332
|
+
checkpoint_path = current.model.load(
|
|
333
|
+
self.checkpoint_key,
|
|
334
|
+
)
|
|
335
|
+
model_path = current.model.load(
|
|
336
|
+
self.model,
|
|
337
|
+
)
|
|
338
|
+
self.next(self.test)
|
|
339
|
+
```
|
|
251
340
|
|
|
252
341
|
|
|
253
342
|
Parameters
|
|
254
343
|
----------
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
If
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
344
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
|
345
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
|
346
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
|
347
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
|
348
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
|
349
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
|
350
|
+
|
|
351
|
+
temp_dir_root : str, default: None
|
|
352
|
+
The root directory under which `current.model.loaded` will store loaded models
|
|
263
353
|
"""
|
|
264
354
|
...
|
|
265
355
|
|
|
266
356
|
@typing.overload
|
|
267
|
-
def
|
|
357
|
+
def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
358
|
+
"""
|
|
359
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
360
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
361
|
+
"""
|
|
268
362
|
...
|
|
269
363
|
|
|
270
364
|
@typing.overload
|
|
271
|
-
def
|
|
365
|
+
def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
272
366
|
...
|
|
273
367
|
|
|
274
|
-
def
|
|
368
|
+
def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
275
369
|
"""
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
370
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
371
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
372
|
+
"""
|
|
373
|
+
...
|
|
374
|
+
|
|
375
|
+
@typing.overload
|
|
376
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
377
|
+
"""
|
|
378
|
+
Specifies environment variables to be set prior to the execution of a step.
|
|
279
379
|
|
|
280
380
|
|
|
281
381
|
Parameters
|
|
282
382
|
----------
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
id : str, optional, default None
|
|
286
|
-
If multiple cards are present, use this id to identify this card.
|
|
287
|
-
options : Dict[str, Any], default {}
|
|
288
|
-
Options passed to the card. The contents depend on the card type.
|
|
289
|
-
timeout : int, default 45
|
|
290
|
-
Interrupt reporting if it takes more than this many seconds.
|
|
383
|
+
vars : Dict[str, str], default {}
|
|
384
|
+
Dictionary of environment variables to set.
|
|
291
385
|
"""
|
|
292
386
|
...
|
|
293
387
|
|
|
294
|
-
|
|
388
|
+
@typing.overload
|
|
389
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
390
|
+
...
|
|
391
|
+
|
|
392
|
+
@typing.overload
|
|
393
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
394
|
+
...
|
|
395
|
+
|
|
396
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
|
295
397
|
"""
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
User code call
|
|
299
|
-
--------------
|
|
300
|
-
@vllm(
|
|
301
|
-
model="...",
|
|
302
|
-
...
|
|
303
|
-
)
|
|
304
|
-
|
|
305
|
-
Valid backend options
|
|
306
|
-
---------------------
|
|
307
|
-
- 'local': Run as a separate process on the local task machine.
|
|
308
|
-
|
|
309
|
-
Valid model options
|
|
310
|
-
-------------------
|
|
311
|
-
Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
|
|
312
|
-
|
|
313
|
-
NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
|
|
314
|
-
If you need multiple models, you must create multiple @vllm decorators.
|
|
398
|
+
Specifies environment variables to be set prior to the execution of a step.
|
|
315
399
|
|
|
316
400
|
|
|
317
401
|
Parameters
|
|
318
402
|
----------
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
backend: str
|
|
322
|
-
Determines where and how to run the vLLM process.
|
|
323
|
-
openai_api_server: bool
|
|
324
|
-
Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
|
|
325
|
-
Default is False (uses native engine).
|
|
326
|
-
Set to True for backward compatibility with existing code.
|
|
327
|
-
debug: bool
|
|
328
|
-
Whether to turn on verbose debugging logs.
|
|
329
|
-
card_refresh_interval: int
|
|
330
|
-
Interval in seconds for refreshing the vLLM status card.
|
|
331
|
-
Only used when openai_api_server=True.
|
|
332
|
-
max_retries: int
|
|
333
|
-
Maximum number of retries checking for vLLM server startup.
|
|
334
|
-
Only used when openai_api_server=True.
|
|
335
|
-
retry_alert_frequency: int
|
|
336
|
-
Frequency of alert logs for vLLM server startup retries.
|
|
337
|
-
Only used when openai_api_server=True.
|
|
338
|
-
engine_args : dict
|
|
339
|
-
Additional keyword arguments to pass to the vLLM engine.
|
|
340
|
-
For example, `tensor_parallel_size=2`.
|
|
403
|
+
vars : Dict[str, str], default {}
|
|
404
|
+
Dictionary of environment variables to set.
|
|
341
405
|
"""
|
|
342
406
|
...
|
|
343
407
|
|
|
344
|
-
|
|
408
|
+
@typing.overload
|
|
409
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
345
410
|
"""
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
User code call
|
|
349
|
-
--------------
|
|
350
|
-
@ollama(
|
|
351
|
-
models=[...],
|
|
352
|
-
...
|
|
353
|
-
)
|
|
354
|
-
|
|
355
|
-
Valid backend options
|
|
356
|
-
---------------------
|
|
357
|
-
- 'local': Run as a separate process on the local task machine.
|
|
358
|
-
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
|
359
|
-
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
|
411
|
+
Specifies the Conda environment for the step.
|
|
360
412
|
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
413
|
+
Information in this decorator will augment any
|
|
414
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
|
415
|
+
you can use `@conda_base` to set packages required by all
|
|
416
|
+
steps and use `@conda` to specify step-specific overrides.
|
|
364
417
|
|
|
365
418
|
|
|
366
419
|
Parameters
|
|
367
420
|
----------
|
|
368
|
-
|
|
369
|
-
List of Ollama containers running models in sidecars.
|
|
370
|
-
backend: str
|
|
371
|
-
Determines where and how to run the Ollama process.
|
|
372
|
-
force_pull: bool
|
|
373
|
-
Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
|
|
374
|
-
cache_update_policy: str
|
|
375
|
-
Cache update policy: "auto", "force", or "never".
|
|
376
|
-
force_cache_update: bool
|
|
377
|
-
Simple override for "force" cache update policy.
|
|
378
|
-
debug: bool
|
|
379
|
-
Whether to turn on verbose debugging logs.
|
|
380
|
-
circuit_breaker_config: dict
|
|
381
|
-
Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
|
|
382
|
-
timeout_config: dict
|
|
383
|
-
Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
|
|
384
|
-
"""
|
|
385
|
-
...
|
|
386
|
-
|
|
387
|
-
@typing.overload
|
|
388
|
-
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
389
|
-
"""
|
|
390
|
-
Specifies the PyPI packages for the step.
|
|
391
|
-
|
|
392
|
-
Information in this decorator will augment any
|
|
393
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
|
394
|
-
you can use `@pypi_base` to set packages required by all
|
|
395
|
-
steps and use `@pypi` to specify step-specific overrides.
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
Parameters
|
|
399
|
-
----------
|
|
400
|
-
packages : Dict[str, str], default: {}
|
|
421
|
+
packages : Dict[str, str], default {}
|
|
401
422
|
Packages to use for this step. The key is the name of the package
|
|
402
423
|
and the value is the version to use.
|
|
403
|
-
|
|
424
|
+
libraries : Dict[str, str], default {}
|
|
425
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
426
|
+
python : str, optional, default None
|
|
404
427
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
405
428
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
429
|
+
disabled : bool, default False
|
|
430
|
+
If set to True, disables @conda.
|
|
406
431
|
"""
|
|
407
432
|
...
|
|
408
433
|
|
|
409
434
|
@typing.overload
|
|
410
|
-
def
|
|
435
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
411
436
|
...
|
|
412
437
|
|
|
413
438
|
@typing.overload
|
|
414
|
-
def
|
|
439
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
415
440
|
...
|
|
416
441
|
|
|
417
|
-
def
|
|
442
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
|
418
443
|
"""
|
|
419
|
-
Specifies the
|
|
444
|
+
Specifies the Conda environment for the step.
|
|
420
445
|
|
|
421
446
|
Information in this decorator will augment any
|
|
422
|
-
attributes set in the `@
|
|
423
|
-
you can use `@
|
|
424
|
-
steps and use `@
|
|
447
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
|
448
|
+
you can use `@conda_base` to set packages required by all
|
|
449
|
+
steps and use `@conda` to specify step-specific overrides.
|
|
425
450
|
|
|
426
451
|
|
|
427
452
|
Parameters
|
|
428
453
|
----------
|
|
429
|
-
packages : Dict[str, str], default
|
|
454
|
+
packages : Dict[str, str], default {}
|
|
430
455
|
Packages to use for this step. The key is the name of the package
|
|
431
456
|
and the value is the version to use.
|
|
432
|
-
|
|
457
|
+
libraries : Dict[str, str], default {}
|
|
458
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
459
|
+
python : str, optional, default None
|
|
433
460
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
434
461
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
462
|
+
disabled : bool, default False
|
|
463
|
+
If set to True, disables @conda.
|
|
435
464
|
"""
|
|
436
465
|
...
|
|
437
466
|
|
|
438
|
-
|
|
467
|
+
@typing.overload
|
|
468
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
439
469
|
"""
|
|
440
|
-
Specifies
|
|
470
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
|
471
|
+
the execution of a step.
|
|
441
472
|
|
|
442
473
|
|
|
443
474
|
Parameters
|
|
444
475
|
----------
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
Kubernetes image pull secrets to use when pulling container images
|
|
465
|
-
in Kubernetes.
|
|
466
|
-
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
|
467
|
-
Kubernetes service account to use when launching pod in Kubernetes.
|
|
468
|
-
secrets : List[str], optional, default None
|
|
469
|
-
Kubernetes secrets to use when launching pod in Kubernetes. These
|
|
470
|
-
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
|
471
|
-
in Metaflow configuration.
|
|
472
|
-
node_selector: Union[Dict[str,str], str], optional, default None
|
|
473
|
-
Kubernetes node selector(s) to apply to the pod running the task.
|
|
474
|
-
Can be passed in as a comma separated string of values e.g.
|
|
475
|
-
'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
|
|
476
|
-
{'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
|
|
477
|
-
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
|
478
|
-
Kubernetes namespace to use when launching pod in Kubernetes.
|
|
479
|
-
gpu : int, optional, default None
|
|
480
|
-
Number of GPUs required for this step. A value of zero implies that
|
|
481
|
-
the scheduled node should not have GPUs.
|
|
482
|
-
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
|
483
|
-
The vendor of the GPUs to be used for this step.
|
|
484
|
-
tolerations : List[Dict[str,str]], default []
|
|
485
|
-
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
|
486
|
-
Kubernetes tolerations to use when launching pod in Kubernetes.
|
|
487
|
-
labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
|
|
488
|
-
Kubernetes labels to use when launching pod in Kubernetes.
|
|
489
|
-
annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
|
|
490
|
-
Kubernetes annotations to use when launching pod in Kubernetes.
|
|
491
|
-
use_tmpfs : bool, default False
|
|
492
|
-
This enables an explicit tmpfs mount for this step.
|
|
493
|
-
tmpfs_tempdir : bool, default True
|
|
494
|
-
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
|
495
|
-
tmpfs_size : int, optional, default: None
|
|
496
|
-
The value for the size (in MiB) of the tmpfs mount for this step.
|
|
497
|
-
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
|
498
|
-
memory allocated for this step.
|
|
499
|
-
tmpfs_path : str, optional, default /metaflow_temp
|
|
500
|
-
Path to tmpfs mount for this step.
|
|
501
|
-
persistent_volume_claims : Dict[str, str], optional, default None
|
|
502
|
-
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
|
503
|
-
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
|
504
|
-
shared_memory: int, optional
|
|
505
|
-
Shared memory size (in MiB) required for this step
|
|
506
|
-
port: int, optional
|
|
507
|
-
Port number to specify in the Kubernetes job object
|
|
508
|
-
compute_pool : str, optional, default None
|
|
509
|
-
Compute pool to be used for for this step.
|
|
510
|
-
If not specified, any accessible compute pool within the perimeter is used.
|
|
511
|
-
hostname_resolution_timeout: int, default 10 * 60
|
|
512
|
-
Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
|
|
513
|
-
Only applicable when @parallel is used.
|
|
514
|
-
qos: str, default: Burstable
|
|
515
|
-
Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
|
|
476
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
|
477
|
+
List of secret specs, defining how the secrets are to be retrieved
|
|
478
|
+
role : str, optional, default: None
|
|
479
|
+
Role to use for fetching secrets
|
|
480
|
+
"""
|
|
481
|
+
...
|
|
482
|
+
|
|
483
|
+
@typing.overload
|
|
484
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
485
|
+
...
|
|
486
|
+
|
|
487
|
+
@typing.overload
|
|
488
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
489
|
+
...
|
|
490
|
+
|
|
491
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
|
|
492
|
+
"""
|
|
493
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
|
494
|
+
the execution of a step.
|
|
516
495
|
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
496
|
+
|
|
497
|
+
Parameters
|
|
498
|
+
----------
|
|
499
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
|
500
|
+
List of secret specs, defining how the secrets are to be retrieved
|
|
501
|
+
role : str, optional, default: None
|
|
502
|
+
Role to use for fetching secrets
|
|
524
503
|
"""
|
|
525
504
|
...
|
|
526
505
|
|
|
527
506
|
@typing.overload
|
|
528
|
-
def
|
|
507
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
529
508
|
"""
|
|
530
509
|
Decorator prototype for all step decorators. This function gets specialized
|
|
531
510
|
and imported for all decorators types by _import_plugin_decorators().
|
|
@@ -533,401 +512,256 @@ def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.
|
|
|
533
512
|
...
|
|
534
513
|
|
|
535
514
|
@typing.overload
|
|
536
|
-
def
|
|
515
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
537
516
|
...
|
|
538
517
|
|
|
539
|
-
def
|
|
518
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
540
519
|
"""
|
|
541
520
|
Decorator prototype for all step decorators. This function gets specialized
|
|
542
521
|
and imported for all decorators types by _import_plugin_decorators().
|
|
543
522
|
"""
|
|
544
523
|
...
|
|
545
524
|
|
|
546
|
-
|
|
547
|
-
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
525
|
+
def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
548
526
|
"""
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
> Examples
|
|
552
|
-
|
|
553
|
-
- Saving Checkpoints
|
|
527
|
+
This decorator is used to run vllm APIs as Metaflow task sidecars.
|
|
554
528
|
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
@
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
# some training logic
|
|
562
|
-
loss = model.train(self.dataset)
|
|
563
|
-
if i % 10 == 0:
|
|
564
|
-
model.save(
|
|
565
|
-
current.checkpoint.directory,
|
|
566
|
-
)
|
|
567
|
-
# saves the contents of the `current.checkpoint.directory` as a checkpoint
|
|
568
|
-
# and returns a reference dictionary to the checkpoint saved in the datastore
|
|
569
|
-
self.latest_checkpoint = current.checkpoint.save(
|
|
570
|
-
name="epoch_checkpoint",
|
|
571
|
-
metadata={
|
|
572
|
-
"epoch": i,
|
|
573
|
-
"loss": loss,
|
|
574
|
-
}
|
|
575
|
-
)
|
|
576
|
-
```
|
|
529
|
+
User code call
|
|
530
|
+
--------------
|
|
531
|
+
@vllm(
|
|
532
|
+
model="...",
|
|
533
|
+
...
|
|
534
|
+
)
|
|
577
535
|
|
|
578
|
-
|
|
536
|
+
Valid backend options
|
|
537
|
+
---------------------
|
|
538
|
+
- 'local': Run as a separate process on the local task machine.
|
|
579
539
|
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
@step
|
|
584
|
-
def train(self):
|
|
585
|
-
# Assume that the task has restarted and the previous attempt of the task
|
|
586
|
-
# saved a checkpoint
|
|
587
|
-
checkpoint_path = None
|
|
588
|
-
if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
|
|
589
|
-
print("Loaded checkpoint from the previous attempt")
|
|
590
|
-
checkpoint_path = current.checkpoint.directory
|
|
540
|
+
Valid model options
|
|
541
|
+
-------------------
|
|
542
|
+
Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
|
|
591
543
|
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
...
|
|
595
|
-
```
|
|
544
|
+
NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
|
|
545
|
+
If you need multiple models, you must create multiple @vllm decorators.
|
|
596
546
|
|
|
597
547
|
|
|
598
548
|
Parameters
|
|
599
549
|
----------
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
550
|
+
model: str
|
|
551
|
+
HuggingFace model identifier to be served by vLLM.
|
|
552
|
+
backend: str
|
|
553
|
+
Determines where and how to run the vLLM process.
|
|
554
|
+
openai_api_server: bool
|
|
555
|
+
Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
|
|
556
|
+
Default is False (uses native engine).
|
|
557
|
+
Set to True for backward compatibility with existing code.
|
|
558
|
+
debug: bool
|
|
559
|
+
Whether to turn on verbose debugging logs.
|
|
560
|
+
card_refresh_interval: int
|
|
561
|
+
Interval in seconds for refreshing the vLLM status card.
|
|
562
|
+
Only used when openai_api_server=True.
|
|
563
|
+
max_retries: int
|
|
564
|
+
Maximum number of retries checking for vLLM server startup.
|
|
565
|
+
Only used when openai_api_server=True.
|
|
566
|
+
retry_alert_frequency: int
|
|
567
|
+
Frequency of alert logs for vLLM server startup retries.
|
|
568
|
+
Only used when openai_api_server=True.
|
|
569
|
+
engine_args : dict
|
|
570
|
+
Additional keyword arguments to pass to the vLLM engine.
|
|
571
|
+
For example, `tensor_parallel_size=2`.
|
|
572
|
+
"""
|
|
573
|
+
...
|
|
623
574
|
|
|
624
|
-
def
|
|
575
|
+
def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
625
576
|
"""
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
> Examples
|
|
629
|
-
|
|
630
|
-
- Saving Checkpoints
|
|
631
|
-
|
|
632
|
-
```python
|
|
633
|
-
@checkpoint
|
|
634
|
-
@step
|
|
635
|
-
def train(self):
|
|
636
|
-
model = create_model(self.parameters, checkpoint_path = None)
|
|
637
|
-
for i in range(self.epochs):
|
|
638
|
-
# some training logic
|
|
639
|
-
loss = model.train(self.dataset)
|
|
640
|
-
if i % 10 == 0:
|
|
641
|
-
model.save(
|
|
642
|
-
current.checkpoint.directory,
|
|
643
|
-
)
|
|
644
|
-
# saves the contents of the `current.checkpoint.directory` as a checkpoint
|
|
645
|
-
# and returns a reference dictionary to the checkpoint saved in the datastore
|
|
646
|
-
self.latest_checkpoint = current.checkpoint.save(
|
|
647
|
-
name="epoch_checkpoint",
|
|
648
|
-
metadata={
|
|
649
|
-
"epoch": i,
|
|
650
|
-
"loss": loss,
|
|
651
|
-
}
|
|
652
|
-
)
|
|
653
|
-
```
|
|
577
|
+
Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
|
|
578
|
+
for S3 read and write requests.
|
|
654
579
|
|
|
655
|
-
|
|
580
|
+
This decorator requires an integration in the Outerbounds platform that
|
|
581
|
+
points to an external bucket. It affects S3 operations performed via
|
|
582
|
+
Metaflow's `get_aws_client` and `S3` within a `@step`.
|
|
656
583
|
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
# saved a checkpoint
|
|
664
|
-
checkpoint_path = None
|
|
665
|
-
if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
|
|
666
|
-
print("Loaded checkpoint from the previous attempt")
|
|
667
|
-
checkpoint_path = current.checkpoint.directory
|
|
584
|
+
Read operations
|
|
585
|
+
---------------
|
|
586
|
+
All read operations pass through the proxy. If an object does not already
|
|
587
|
+
exist in the external bucket, it is cached there. For example, if code reads
|
|
588
|
+
from buckets `FOO` and `BAR` using the `S3` interface, objects from both
|
|
589
|
+
buckets are cached in the external bucket.
|
|
668
590
|
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
591
|
+
During task execution, all S3‑related read requests are routed through the
|
|
592
|
+
proxy:
|
|
593
|
+
- If the object is present in the external object store, the proxy
|
|
594
|
+
streams it directly from there without accessing the requested origin
|
|
595
|
+
bucket.
|
|
596
|
+
- If the object is not present in the external storage, the proxy
|
|
597
|
+
fetches it from the requested bucket, caches it in the external
|
|
598
|
+
storage, and streams the response from the origin bucket.
|
|
673
599
|
|
|
600
|
+
Warning
|
|
601
|
+
-------
|
|
602
|
+
All READ operations (e.g., GetObject, HeadObject) pass through the external
|
|
603
|
+
bucket regardless of the bucket specified in user code. Even
|
|
604
|
+
`S3(run=self)` and `S3(s3root="mybucketfoo")` requests go through the
|
|
605
|
+
external bucket cache.
|
|
674
606
|
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
- "eager": Loads the the latest available checkpoint within the namespace.
|
|
680
|
-
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
|
681
|
-
will be loaded at the start of the task.
|
|
682
|
-
- "none": Do not load any checkpoint
|
|
683
|
-
- "fresh": Loads the lastest checkpoint created within the running Task.
|
|
684
|
-
This mode helps loading checkpoints across various retry attempts of the same task.
|
|
685
|
-
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
|
686
|
-
created within the task will be loaded when the task is retries execution on failure.
|
|
607
|
+
Write operations
|
|
608
|
+
----------------
|
|
609
|
+
Write behavior is controlled by the `write_mode` parameter, which determines
|
|
610
|
+
whether writes also persist objects in the cache.
|
|
687
611
|
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
694
|
-
"""
|
|
695
|
-
Specifies that this step should execute on DGX cloud.
|
|
612
|
+
`write_mode` values:
|
|
613
|
+
- `origin-and-cache`: objects are written both to the cache and to their
|
|
614
|
+
intended origin bucket.
|
|
615
|
+
- `origin`: objects are written only to their intended origin bucket.
|
|
696
616
|
|
|
697
617
|
|
|
698
618
|
Parameters
|
|
699
619
|
----------
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
620
|
+
integration_name : str, optional
|
|
621
|
+
[Outerbounds integration name](https://docs.outerbounds.com/outerbounds/configuring-secrets/#integrations-view)
|
|
622
|
+
that holds the configuration for the external, S3‑compatible object
|
|
623
|
+
storage bucket. If not specified, the only available S3 proxy
|
|
624
|
+
integration in the namespace is used (fails if multiple exist).
|
|
625
|
+
write_mode : str, optional
|
|
626
|
+
Controls whether writes also go to the external bucket.
|
|
627
|
+
- `origin` (default)
|
|
628
|
+
- `origin-and-cache`
|
|
629
|
+
debug : bool, optional
|
|
630
|
+
Enables debug logging for proxy operations.
|
|
704
631
|
"""
|
|
705
632
|
...
|
|
706
633
|
|
|
707
634
|
@typing.overload
|
|
708
|
-
def
|
|
635
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
709
636
|
"""
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
The decorator will create an optional artifact, specified by `var`, which
|
|
713
|
-
contains the exception raised. You can use it to detect the presence
|
|
714
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
|
715
|
-
are missing.
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
Parameters
|
|
719
|
-
----------
|
|
720
|
-
var : str, optional, default None
|
|
721
|
-
Name of the artifact in which to store the caught exception.
|
|
722
|
-
If not specified, the exception is not stored.
|
|
723
|
-
print_exception : bool, default True
|
|
724
|
-
Determines whether or not the exception is printed to
|
|
725
|
-
stdout when caught.
|
|
637
|
+
Internal decorator to support Fast bakery
|
|
726
638
|
"""
|
|
727
639
|
...
|
|
728
640
|
|
|
729
641
|
@typing.overload
|
|
730
|
-
def
|
|
731
|
-
...
|
|
732
|
-
|
|
733
|
-
@typing.overload
|
|
734
|
-
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
642
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
735
643
|
...
|
|
736
644
|
|
|
737
|
-
def
|
|
645
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
738
646
|
"""
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
The decorator will create an optional artifact, specified by `var`, which
|
|
742
|
-
contains the exception raised. You can use it to detect the presence
|
|
743
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
|
744
|
-
are missing.
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
Parameters
|
|
748
|
-
----------
|
|
749
|
-
var : str, optional, default None
|
|
750
|
-
Name of the artifact in which to store the caught exception.
|
|
751
|
-
If not specified, the exception is not stored.
|
|
752
|
-
print_exception : bool, default True
|
|
753
|
-
Determines whether or not the exception is printed to
|
|
754
|
-
stdout when caught.
|
|
647
|
+
Internal decorator to support Fast bakery
|
|
755
648
|
"""
|
|
756
649
|
...
|
|
757
650
|
|
|
758
|
-
|
|
759
|
-
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
651
|
+
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, cache_scope: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
760
652
|
"""
|
|
761
|
-
|
|
653
|
+
Decorator that helps cache, version, and store models/datasets from the Hugging Face Hub.
|
|
762
654
|
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
you can use `@conda_base` to set packages required by all
|
|
766
|
-
steps and use `@conda` to specify step-specific overrides.
|
|
655
|
+
Examples
|
|
656
|
+
--------
|
|
767
657
|
|
|
658
|
+
```python
|
|
659
|
+
# **Usage: creating references to models from the Hugging Face Hub that may be loaded in downstream steps**
|
|
660
|
+
@huggingface_hub
|
|
661
|
+
@step
|
|
662
|
+
def pull_model_from_huggingface(self):
|
|
663
|
+
# `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
|
|
664
|
+
# and saves it in the backend storage based on the model's `repo_id`. If there exists a model
|
|
665
|
+
# with the same `repo_id` in the backend storage, it will not download the model again. The return
|
|
666
|
+
# value of the function is a reference to the model in the backend storage.
|
|
667
|
+
# This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
|
|
768
668
|
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
776
|
-
python : str, optional, default None
|
|
777
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
778
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
779
|
-
disabled : bool, default False
|
|
780
|
-
If set to True, disables @conda.
|
|
781
|
-
"""
|
|
782
|
-
...
|
|
783
|
-
|
|
784
|
-
@typing.overload
|
|
785
|
-
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
786
|
-
...
|
|
787
|
-
|
|
788
|
-
@typing.overload
|
|
789
|
-
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
790
|
-
...
|
|
791
|
-
|
|
792
|
-
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
|
793
|
-
"""
|
|
794
|
-
Specifies the Conda environment for the step.
|
|
669
|
+
self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
|
|
670
|
+
self.llama_model = current.huggingface_hub.snapshot_download(
|
|
671
|
+
repo_id=self.model_id,
|
|
672
|
+
allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
|
|
673
|
+
)
|
|
674
|
+
self.next(self.train)
|
|
795
675
|
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
676
|
+
# **Usage: explicitly loading models at runtime from the Hugging Face Hub or from cache (from Metaflow's datastore)**
|
|
677
|
+
@huggingface_hub
|
|
678
|
+
@step
|
|
679
|
+
def run_training(self):
|
|
680
|
+
# Temporary directory (auto-cleaned on exit)
|
|
681
|
+
with current.huggingface_hub.load(
|
|
682
|
+
repo_id="google-bert/bert-base-uncased",
|
|
683
|
+
allow_patterns=["*.bin"],
|
|
684
|
+
) as local_path:
|
|
685
|
+
# Use files under local_path
|
|
686
|
+
train_model(local_path)
|
|
687
|
+
...
|
|
800
688
|
|
|
689
|
+
# **Usage: loading models directly from the Hugging Face Hub or from cache (from Metaflow's datastore)**
|
|
801
690
|
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
and the value is the version to use.
|
|
807
|
-
libraries : Dict[str, str], default {}
|
|
808
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
809
|
-
python : str, optional, default None
|
|
810
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
811
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
812
|
-
disabled : bool, default False
|
|
813
|
-
If set to True, disables @conda.
|
|
814
|
-
"""
|
|
815
|
-
...
|
|
816
|
-
|
|
817
|
-
@typing.overload
|
|
818
|
-
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
819
|
-
"""
|
|
820
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
|
821
|
-
the execution of a step.
|
|
691
|
+
@huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
|
|
692
|
+
@step
|
|
693
|
+
def pull_model_from_huggingface(self):
|
|
694
|
+
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
822
695
|
|
|
696
|
+
@huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora", "/my-lora-directory")])
|
|
697
|
+
@step
|
|
698
|
+
def finetune_model(self):
|
|
699
|
+
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
700
|
+
# path_to_model will be /my-directory
|
|
823
701
|
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
|
|
842
|
-
"""
|
|
843
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
|
844
|
-
the execution of a step.
|
|
702
|
+
|
|
703
|
+
# Takes all the arguments passed to `snapshot_download`
|
|
704
|
+
# except for `local_dir`
|
|
705
|
+
@huggingface_hub(load=[
|
|
706
|
+
{
|
|
707
|
+
"repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
|
|
708
|
+
},
|
|
709
|
+
{
|
|
710
|
+
"repo_id": "myorg/mistral-lora",
|
|
711
|
+
"repo_type": "model",
|
|
712
|
+
},
|
|
713
|
+
])
|
|
714
|
+
@step
|
|
715
|
+
def finetune_model(self):
|
|
716
|
+
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
717
|
+
# path_to_model will be /my-directory
|
|
718
|
+
```
|
|
845
719
|
|
|
846
720
|
|
|
847
721
|
Parameters
|
|
848
722
|
----------
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
role : str, optional, default: None
|
|
852
|
-
Role to use for fetching secrets
|
|
853
|
-
"""
|
|
854
|
-
...
|
|
855
|
-
|
|
856
|
-
@typing.overload
|
|
857
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
858
|
-
"""
|
|
859
|
-
Decorator prototype for all step decorators. This function gets specialized
|
|
860
|
-
and imported for all decorators types by _import_plugin_decorators().
|
|
861
|
-
"""
|
|
862
|
-
...
|
|
863
|
-
|
|
864
|
-
@typing.overload
|
|
865
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
866
|
-
...
|
|
867
|
-
|
|
868
|
-
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
869
|
-
"""
|
|
870
|
-
Decorator prototype for all step decorators. This function gets specialized
|
|
871
|
-
and imported for all decorators types by _import_plugin_decorators().
|
|
872
|
-
"""
|
|
873
|
-
...
|
|
874
|
-
|
|
875
|
-
def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
876
|
-
"""
|
|
877
|
-
Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
|
|
878
|
-
for S3 read and write requests.
|
|
879
|
-
|
|
880
|
-
This decorator requires an integration in the Outerbounds platform that
|
|
881
|
-
points to an external bucket. It affects S3 operations performed via
|
|
882
|
-
Metaflow's `get_aws_client` and `S3` within a `@step`.
|
|
723
|
+
temp_dir_root : str, optional
|
|
724
|
+
The root directory that will hold the temporary directory where objects will be downloaded.
|
|
883
725
|
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
buckets are cached in the external bucket.
|
|
726
|
+
cache_scope : str, optional
|
|
727
|
+
The scope of the cache. Can be `checkpoint` / `flow` / `global`.
|
|
728
|
+
- `checkpoint` (default): All repos are stored like objects saved by `@checkpoint`.
|
|
729
|
+
i.e., the cached path is derived from the namespace, flow, step, and Metaflow foreach iteration.
|
|
730
|
+
Any repo downloaded under this scope will only be retrieved from the cache when the step runs under the same namespace in the same flow (at the same foreach index).
|
|
890
731
|
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
streams it directly from there without accessing the requested origin
|
|
895
|
-
bucket.
|
|
896
|
-
- If the object is not present in the external storage, the proxy
|
|
897
|
-
fetches it from the requested bucket, caches it in the external
|
|
898
|
-
storage, and streams the response from the origin bucket.
|
|
732
|
+
- `flow`: All repos are cached under the flow, regardless of namespace.
|
|
733
|
+
i.e., the cached path is derived solely from the flow name.
|
|
734
|
+
When to use this mode: (1) Multiple users are executing the same flow and want shared access to the repos cached by the decorator. (2) Multiple versions of a flow are deployed, all needing access to the same repos cached by the decorator.
|
|
899
735
|
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
736
|
+
- `global`: All repos are cached under a globally static path.
|
|
737
|
+
i.e., the base path of the cache is static and all repos are stored under it.
|
|
738
|
+
When to use this mode:
|
|
739
|
+
- All repos from the Hugging Face Hub need to be shared by users across all flow executions.
|
|
740
|
+
- Each caching scope comes with its own trade-offs:
|
|
741
|
+
- `checkpoint`:
|
|
742
|
+
- Has explicit control over when caches are populated (controlled by the same flow that has the `@huggingface_hub` decorator) but ends up hitting the Hugging Face Hub more often if there are many users/namespaces/steps.
|
|
743
|
+
- Since objects are written on a `namespace/flow/step` basis, the blast radius of a bad checkpoint is limited to a particular flow in a namespace.
|
|
744
|
+
- `flow`:
|
|
745
|
+
- Has less control over when caches are populated (can be written by any execution instance of a flow from any namespace) but results in more cache hits.
|
|
746
|
+
- The blast radius of a bad checkpoint is limited to all runs of a particular flow.
|
|
747
|
+
- It doesn't promote cache reuse across flows.
|
|
748
|
+
- `global`:
|
|
749
|
+
- Has no control over when caches are populated (can be written by any flow execution) but has the highest cache hit rate.
|
|
750
|
+
- It promotes cache reuse across flows.
|
|
751
|
+
- The blast radius of a bad checkpoint spans every flow that could be using a particular repo.
|
|
906
752
|
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
Write behavior is controlled by the `write_mode` parameter, which determines
|
|
910
|
-
whether writes also persist objects in the cache.
|
|
753
|
+
load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
|
|
754
|
+
The list of repos (models/datasets) to load.
|
|
911
755
|
|
|
912
|
-
|
|
913
|
-
- `origin-and-cache`: objects are written both to the cache and to their
|
|
914
|
-
intended origin bucket.
|
|
915
|
-
- `origin`: objects are written only to their intended origin bucket.
|
|
756
|
+
Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
|
|
916
757
|
|
|
758
|
+
- If repo (model/dataset) is not found in the datastore:
|
|
759
|
+
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
|
760
|
+
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
|
761
|
+
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
|
917
762
|
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
integration_name : str, optional
|
|
921
|
-
[Outerbounds integration name](https://docs.outerbounds.com/outerbounds/configuring-secrets/#integrations-view)
|
|
922
|
-
that holds the configuration for the external, S3‑compatible object
|
|
923
|
-
storage bucket. If not specified, the only available S3 proxy
|
|
924
|
-
integration in the namespace is used (fails if multiple exist).
|
|
925
|
-
write_mode : str, optional
|
|
926
|
-
Controls whether writes also go to the external bucket.
|
|
927
|
-
- `origin` (default)
|
|
928
|
-
- `origin-and-cache`
|
|
929
|
-
debug : bool, optional
|
|
930
|
-
Enables debug logging for proxy operations.
|
|
763
|
+
- If repo is found in the datastore:
|
|
764
|
+
- Loads it directly from datastore to local path (can be temporary directory or specified path)
|
|
931
765
|
"""
|
|
932
766
|
...
|
|
933
767
|
|
|
@@ -1010,165 +844,111 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
|
1010
844
|
"""
|
|
1011
845
|
...
|
|
1012
846
|
|
|
1013
|
-
|
|
1014
|
-
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
847
|
+
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1015
848
|
"""
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
> Examples
|
|
1019
|
-
- Saving Models
|
|
1020
|
-
```python
|
|
1021
|
-
@model
|
|
1022
|
-
@step
|
|
1023
|
-
def train(self):
|
|
1024
|
-
# current.model.save returns a dictionary reference to the model saved
|
|
1025
|
-
self.my_model = current.model.save(
|
|
1026
|
-
path_to_my_model,
|
|
1027
|
-
label="my_model",
|
|
1028
|
-
metadata={
|
|
1029
|
-
"epochs": 10,
|
|
1030
|
-
"batch-size": 32,
|
|
1031
|
-
"learning-rate": 0.001,
|
|
1032
|
-
}
|
|
1033
|
-
)
|
|
1034
|
-
self.next(self.test)
|
|
1035
|
-
|
|
1036
|
-
@model(load="my_model")
|
|
1037
|
-
@step
|
|
1038
|
-
def test(self):
|
|
1039
|
-
# `current.model.loaded` returns a dictionary of the loaded models
|
|
1040
|
-
# where the key is the name of the artifact and the value is the path to the model
|
|
1041
|
-
print(os.listdir(current.model.loaded["my_model"]))
|
|
1042
|
-
self.next(self.end)
|
|
1043
|
-
```
|
|
1044
|
-
|
|
1045
|
-
- Loading models
|
|
1046
|
-
```python
|
|
1047
|
-
@step
|
|
1048
|
-
def train(self):
|
|
1049
|
-
# current.model.load returns the path to the model loaded
|
|
1050
|
-
checkpoint_path = current.model.load(
|
|
1051
|
-
self.checkpoint_key,
|
|
1052
|
-
)
|
|
1053
|
-
model_path = current.model.load(
|
|
1054
|
-
self.model,
|
|
1055
|
-
)
|
|
1056
|
-
self.next(self.test)
|
|
1057
|
-
```
|
|
849
|
+
Specifies that this step should execute on Kubernetes.
|
|
1058
850
|
|
|
1059
851
|
|
|
1060
852
|
Parameters
|
|
1061
853
|
----------
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
|
1134
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
|
1135
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
|
854
|
+
cpu : int, default 1
|
|
855
|
+
Number of CPUs required for this step. If `@resources` is
|
|
856
|
+
also present, the maximum value from all decorators is used.
|
|
857
|
+
memory : int, default 4096
|
|
858
|
+
Memory size (in MB) required for this step. If
|
|
859
|
+
`@resources` is also present, the maximum value from all decorators is
|
|
860
|
+
used.
|
|
861
|
+
disk : int, default 10240
|
|
862
|
+
Disk size (in MB) required for this step. If
|
|
863
|
+
`@resources` is also present, the maximum value from all decorators is
|
|
864
|
+
used.
|
|
865
|
+
image : str, optional, default None
|
|
866
|
+
Docker image to use when launching on Kubernetes. If not specified, and
|
|
867
|
+
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
|
868
|
+
not, a default Docker image mapping to the current version of Python is used.
|
|
869
|
+
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
|
870
|
+
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
|
871
|
+
image_pull_secrets: List[str], default []
|
|
872
|
+
The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
|
|
873
|
+
Kubernetes image pull secrets to use when pulling container images
|
|
874
|
+
in Kubernetes.
|
|
875
|
+
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
|
876
|
+
Kubernetes service account to use when launching pod in Kubernetes.
|
|
877
|
+
secrets : List[str], optional, default None
|
|
878
|
+
Kubernetes secrets to use when launching pod in Kubernetes. These
|
|
879
|
+
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
|
880
|
+
in Metaflow configuration.
|
|
881
|
+
node_selector: Union[Dict[str,str], str], optional, default None
|
|
882
|
+
Kubernetes node selector(s) to apply to the pod running the task.
|
|
883
|
+
Can be passed in as a comma separated string of values e.g.
|
|
884
|
+
'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
|
|
885
|
+
{'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
|
|
886
|
+
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
|
887
|
+
Kubernetes namespace to use when launching pod in Kubernetes.
|
|
888
|
+
gpu : int, optional, default None
|
|
889
|
+
Number of GPUs required for this step. A value of zero implies that
|
|
890
|
+
the scheduled node should not have GPUs.
|
|
891
|
+
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
|
892
|
+
The vendor of the GPUs to be used for this step.
|
|
893
|
+
tolerations : List[Dict[str,str]], default []
|
|
894
|
+
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
|
895
|
+
Kubernetes tolerations to use when launching pod in Kubernetes.
|
|
896
|
+
labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
|
|
897
|
+
Kubernetes labels to use when launching pod in Kubernetes.
|
|
898
|
+
annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
|
|
899
|
+
Kubernetes annotations to use when launching pod in Kubernetes.
|
|
900
|
+
use_tmpfs : bool, default False
|
|
901
|
+
This enables an explicit tmpfs mount for this step.
|
|
902
|
+
tmpfs_tempdir : bool, default True
|
|
903
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
|
904
|
+
tmpfs_size : int, optional, default: None
|
|
905
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
|
906
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
|
907
|
+
memory allocated for this step.
|
|
908
|
+
tmpfs_path : str, optional, default /metaflow_temp
|
|
909
|
+
Path to tmpfs mount for this step.
|
|
910
|
+
persistent_volume_claims : Dict[str, str], optional, default None
|
|
911
|
+
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
|
912
|
+
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
|
913
|
+
shared_memory: int, optional
|
|
914
|
+
Shared memory size (in MiB) required for this step
|
|
915
|
+
port: int, optional
|
|
916
|
+
Port number to specify in the Kubernetes job object
|
|
917
|
+
compute_pool : str, optional, default None
|
|
918
|
+
Compute pool to be used for for this step.
|
|
919
|
+
If not specified, any accessible compute pool within the perimeter is used.
|
|
920
|
+
hostname_resolution_timeout: int, default 10 * 60
|
|
921
|
+
Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
|
|
922
|
+
Only applicable when @parallel is used.
|
|
923
|
+
qos: str, default: Burstable
|
|
924
|
+
Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
|
|
1136
925
|
|
|
1137
|
-
|
|
1138
|
-
|
|
926
|
+
security_context: Dict[str, Any], optional, default None
|
|
927
|
+
Container security context. Applies to the task container. Allows the following keys:
|
|
928
|
+
- privileged: bool, optional, default None
|
|
929
|
+
- allow_privilege_escalation: bool, optional, default None
|
|
930
|
+
- run_as_user: int, optional, default None
|
|
931
|
+
- run_as_group: int, optional, default None
|
|
932
|
+
- run_as_non_root: bool, optional, default None
|
|
1139
933
|
"""
|
|
1140
934
|
...
|
|
1141
935
|
|
|
1142
936
|
@typing.overload
|
|
1143
|
-
def
|
|
937
|
+
def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1144
938
|
"""
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
Parameters
|
|
1149
|
-
----------
|
|
1150
|
-
vars : Dict[str, str], default {}
|
|
1151
|
-
Dictionary of environment variables to set.
|
|
939
|
+
A simple decorator that demonstrates using CardDecoratorInjector
|
|
940
|
+
to inject a card and render simple markdown content.
|
|
1152
941
|
"""
|
|
1153
942
|
...
|
|
1154
943
|
|
|
1155
944
|
@typing.overload
|
|
1156
|
-
def
|
|
1157
|
-
...
|
|
1158
|
-
|
|
1159
|
-
@typing.overload
|
|
1160
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
945
|
+
def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1161
946
|
...
|
|
1162
947
|
|
|
1163
|
-
def
|
|
948
|
+
def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
1164
949
|
"""
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
Parameters
|
|
1169
|
-
----------
|
|
1170
|
-
vars : Dict[str, str], default {}
|
|
1171
|
-
Dictionary of environment variables to set.
|
|
950
|
+
A simple decorator that demonstrates using CardDecoratorInjector
|
|
951
|
+
to inject a card and render simple markdown content.
|
|
1172
952
|
"""
|
|
1173
953
|
...
|
|
1174
954
|
|
|
@@ -1295,120 +1075,167 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
|
1295
1075
|
"""
|
|
1296
1076
|
...
|
|
1297
1077
|
|
|
1298
|
-
|
|
1078
|
+
@typing.overload
|
|
1079
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1299
1080
|
"""
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
Examples
|
|
1303
|
-
--------
|
|
1081
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
|
1304
1082
|
|
|
1305
|
-
|
|
1306
|
-
# **Usage: creating references to models from the Hugging Face Hub that may be loaded in downstream steps**
|
|
1307
|
-
@huggingface_hub
|
|
1308
|
-
@step
|
|
1309
|
-
def pull_model_from_huggingface(self):
|
|
1310
|
-
# `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
|
|
1311
|
-
# and saves it in the backend storage based on the model's `repo_id`. If there exists a model
|
|
1312
|
-
# with the same `repo_id` in the backend storage, it will not download the model again. The return
|
|
1313
|
-
# value of the function is a reference to the model in the backend storage.
|
|
1314
|
-
# This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
|
|
1083
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
|
1315
1084
|
|
|
1316
|
-
self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
|
|
1317
|
-
self.llama_model = current.huggingface_hub.snapshot_download(
|
|
1318
|
-
repo_id=self.model_id,
|
|
1319
|
-
allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
|
|
1320
|
-
)
|
|
1321
|
-
self.next(self.train)
|
|
1322
1085
|
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1086
|
+
Parameters
|
|
1087
|
+
----------
|
|
1088
|
+
type : str, default 'default'
|
|
1089
|
+
Card type.
|
|
1090
|
+
id : str, optional, default None
|
|
1091
|
+
If multiple cards are present, use this id to identify this card.
|
|
1092
|
+
options : Dict[str, Any], default {}
|
|
1093
|
+
Options passed to the card. The contents depend on the card type.
|
|
1094
|
+
timeout : int, default 45
|
|
1095
|
+
Interrupt reporting if it takes more than this many seconds.
|
|
1096
|
+
"""
|
|
1097
|
+
...
|
|
1098
|
+
|
|
1099
|
+
@typing.overload
|
|
1100
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1101
|
+
...
|
|
1102
|
+
|
|
1103
|
+
@typing.overload
|
|
1104
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1105
|
+
...
|
|
1106
|
+
|
|
1107
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
|
1108
|
+
"""
|
|
1109
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
|
1335
1110
|
|
|
1336
|
-
|
|
1111
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
|
1337
1112
|
|
|
1338
|
-
@huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
|
|
1339
|
-
@step
|
|
1340
|
-
def pull_model_from_huggingface(self):
|
|
1341
|
-
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
1342
1113
|
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1114
|
+
Parameters
|
|
1115
|
+
----------
|
|
1116
|
+
type : str, default 'default'
|
|
1117
|
+
Card type.
|
|
1118
|
+
id : str, optional, default None
|
|
1119
|
+
If multiple cards are present, use this id to identify this card.
|
|
1120
|
+
options : Dict[str, Any], default {}
|
|
1121
|
+
Options passed to the card. The contents depend on the card type.
|
|
1122
|
+
timeout : int, default 45
|
|
1123
|
+
Interrupt reporting if it takes more than this many seconds.
|
|
1124
|
+
"""
|
|
1125
|
+
...
|
|
1126
|
+
|
|
1127
|
+
def nebius_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1128
|
+
"""
|
|
1129
|
+
`@nebius_s3_proxy` is a Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
1130
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
|
1131
|
+
a Neo Cloud like Nebius. The underlying mechanics of the decorator is the same as the `@s3_proxy`:
|
|
1348
1132
|
|
|
1349
1133
|
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1134
|
+
Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
|
|
1135
|
+
for S3 read and write requests.
|
|
1136
|
+
|
|
1137
|
+
This decorator requires an integration in the Outerbounds platform that
|
|
1138
|
+
points to an external bucket. It affects S3 operations performed via
|
|
1139
|
+
Metaflow's `get_aws_client` and `S3` within a `@step`.
|
|
1140
|
+
|
|
1141
|
+
Read operations
|
|
1142
|
+
---------------
|
|
1143
|
+
All read operations pass through the proxy. If an object does not already
|
|
1144
|
+
exist in the external bucket, it is cached there. For example, if code reads
|
|
1145
|
+
from buckets `FOO` and `BAR` using the `S3` interface, objects from both
|
|
1146
|
+
buckets are cached in the external bucket.
|
|
1147
|
+
|
|
1148
|
+
During task execution, all S3‑related read requests are routed through the
|
|
1149
|
+
proxy:
|
|
1150
|
+
- If the object is present in the external object store, the proxy
|
|
1151
|
+
streams it directly from there without accessing the requested origin
|
|
1152
|
+
bucket.
|
|
1153
|
+
- If the object is not present in the external storage, the proxy
|
|
1154
|
+
fetches it from the requested bucket, caches it in the external
|
|
1155
|
+
storage, and streams the response from the origin bucket.
|
|
1156
|
+
|
|
1157
|
+
Warning
|
|
1158
|
+
-------
|
|
1159
|
+
All READ operations (e.g., GetObject, HeadObject) pass through the external
|
|
1160
|
+
bucket regardless of the bucket specified in user code. Even
|
|
1161
|
+
`S3(run=self)` and `S3(s3root="mybucketfoo")` requests go through the
|
|
1162
|
+
external bucket cache.
|
|
1163
|
+
|
|
1164
|
+
Write operations
|
|
1165
|
+
----------------
|
|
1166
|
+
Write behavior is controlled by the `write_mode` parameter, which determines
|
|
1167
|
+
whether writes also persist objects in the cache.
|
|
1168
|
+
|
|
1169
|
+
`write_mode` values:
|
|
1170
|
+
- `origin-and-cache`: objects are written both to the cache and to their
|
|
1171
|
+
intended origin bucket.
|
|
1172
|
+
- `origin`: objects are written only to their intended origin bucket.
|
|
1366
1173
|
|
|
1367
1174
|
|
|
1368
1175
|
Parameters
|
|
1369
1176
|
----------
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1177
|
+
integration_name : str, optional
|
|
1178
|
+
[Outerbounds integration name](https://docs.outerbounds.com/outerbounds/configuring-secrets/#integrations-view)
|
|
1179
|
+
that holds the configuration for the external, S3‑compatible object
|
|
1180
|
+
storage bucket. If not specified, the only available S3 proxy
|
|
1181
|
+
integration in the namespace is used (fails if multiple exist).
|
|
1182
|
+
write_mode : str, optional
|
|
1183
|
+
Controls whether writes also go to the external bucket.
|
|
1184
|
+
- `origin` (default)
|
|
1185
|
+
- `origin-and-cache`
|
|
1186
|
+
debug : bool, optional
|
|
1187
|
+
Enables debug logging for proxy operations.
|
|
1188
|
+
"""
|
|
1189
|
+
...
|
|
1190
|
+
|
|
1191
|
+
@typing.overload
|
|
1192
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1193
|
+
"""
|
|
1194
|
+
Specifies that the step will success under all circumstances.
|
|
1378
1195
|
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1196
|
+
The decorator will create an optional artifact, specified by `var`, which
|
|
1197
|
+
contains the exception raised. You can use it to detect the presence
|
|
1198
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
|
1199
|
+
are missing.
|
|
1382
1200
|
|
|
1383
|
-
- `global`: All repos are cached under a globally static path.
|
|
1384
|
-
i.e., the base path of the cache is static and all repos are stored under it.
|
|
1385
|
-
When to use this mode:
|
|
1386
|
-
- All repos from the Hugging Face Hub need to be shared by users across all flow executions.
|
|
1387
|
-
- Each caching scope comes with its own trade-offs:
|
|
1388
|
-
- `checkpoint`:
|
|
1389
|
-
- Has explicit control over when caches are populated (controlled by the same flow that has the `@huggingface_hub` decorator) but ends up hitting the Hugging Face Hub more often if there are many users/namespaces/steps.
|
|
1390
|
-
- Since objects are written on a `namespace/flow/step` basis, the blast radius of a bad checkpoint is limited to a particular flow in a namespace.
|
|
1391
|
-
- `flow`:
|
|
1392
|
-
- Has less control over when caches are populated (can be written by any execution instance of a flow from any namespace) but results in more cache hits.
|
|
1393
|
-
- The blast radius of a bad checkpoint is limited to all runs of a particular flow.
|
|
1394
|
-
- It doesn't promote cache reuse across flows.
|
|
1395
|
-
- `global`:
|
|
1396
|
-
- Has no control over when caches are populated (can be written by any flow execution) but has the highest cache hit rate.
|
|
1397
|
-
- It promotes cache reuse across flows.
|
|
1398
|
-
- The blast radius of a bad checkpoint spans every flow that could be using a particular repo.
|
|
1399
1201
|
|
|
1400
|
-
|
|
1401
|
-
|
|
1202
|
+
Parameters
|
|
1203
|
+
----------
|
|
1204
|
+
var : str, optional, default None
|
|
1205
|
+
Name of the artifact in which to store the caught exception.
|
|
1206
|
+
If not specified, the exception is not stored.
|
|
1207
|
+
print_exception : bool, default True
|
|
1208
|
+
Determines whether or not the exception is printed to
|
|
1209
|
+
stdout when caught.
|
|
1210
|
+
"""
|
|
1211
|
+
...
|
|
1212
|
+
|
|
1213
|
+
@typing.overload
|
|
1214
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1215
|
+
...
|
|
1216
|
+
|
|
1217
|
+
@typing.overload
|
|
1218
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1219
|
+
...
|
|
1220
|
+
|
|
1221
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
|
1222
|
+
"""
|
|
1223
|
+
Specifies that the step will success under all circumstances.
|
|
1402
1224
|
|
|
1403
|
-
|
|
1225
|
+
The decorator will create an optional artifact, specified by `var`, which
|
|
1226
|
+
contains the exception raised. You can use it to detect the presence
|
|
1227
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
|
1228
|
+
are missing.
|
|
1404
1229
|
|
|
1405
|
-
- If repo (model/dataset) is not found in the datastore:
|
|
1406
|
-
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
|
1407
|
-
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
|
1408
|
-
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
|
1409
1230
|
|
|
1410
|
-
|
|
1411
|
-
|
|
1231
|
+
Parameters
|
|
1232
|
+
----------
|
|
1233
|
+
var : str, optional, default None
|
|
1234
|
+
Name of the artifact in which to store the caught exception.
|
|
1235
|
+
If not specified, the exception is not stored.
|
|
1236
|
+
print_exception : bool, default True
|
|
1237
|
+
Determines whether or not the exception is printed to
|
|
1238
|
+
stdout when caught.
|
|
1412
1239
|
"""
|
|
1413
1240
|
...
|
|
1414
1241
|
|
|
@@ -1428,84 +1255,336 @@ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[t
|
|
|
1428
1255
|
"""
|
|
1429
1256
|
...
|
|
1430
1257
|
|
|
1258
|
+
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1259
|
+
"""
|
|
1260
|
+
Specifies that this step should execute on DGX cloud.
|
|
1261
|
+
|
|
1262
|
+
|
|
1263
|
+
Parameters
|
|
1264
|
+
----------
|
|
1265
|
+
gpu : int
|
|
1266
|
+
Number of GPUs to use.
|
|
1267
|
+
gpu_type : str
|
|
1268
|
+
Type of Nvidia GPU to use.
|
|
1269
|
+
"""
|
|
1270
|
+
...
|
|
1271
|
+
|
|
1272
|
+
def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1273
|
+
"""
|
|
1274
|
+
This decorator is used to run Ollama APIs as Metaflow task sidecars.
|
|
1275
|
+
|
|
1276
|
+
User code call
|
|
1277
|
+
--------------
|
|
1278
|
+
@ollama(
|
|
1279
|
+
models=[...],
|
|
1280
|
+
...
|
|
1281
|
+
)
|
|
1282
|
+
|
|
1283
|
+
Valid backend options
|
|
1284
|
+
---------------------
|
|
1285
|
+
- 'local': Run as a separate process on the local task machine.
|
|
1286
|
+
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
|
1287
|
+
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
|
1288
|
+
|
|
1289
|
+
Valid model options
|
|
1290
|
+
-------------------
|
|
1291
|
+
Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
|
|
1292
|
+
|
|
1293
|
+
|
|
1294
|
+
Parameters
|
|
1295
|
+
----------
|
|
1296
|
+
models: list[str]
|
|
1297
|
+
List of Ollama containers running models in sidecars.
|
|
1298
|
+
backend: str
|
|
1299
|
+
Determines where and how to run the Ollama process.
|
|
1300
|
+
force_pull: bool
|
|
1301
|
+
Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
|
|
1302
|
+
cache_update_policy: str
|
|
1303
|
+
Cache update policy: "auto", "force", or "never".
|
|
1304
|
+
force_cache_update: bool
|
|
1305
|
+
Simple override for "force" cache update policy.
|
|
1306
|
+
debug: bool
|
|
1307
|
+
Whether to turn on verbose debugging logs.
|
|
1308
|
+
circuit_breaker_config: dict
|
|
1309
|
+
Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
|
|
1310
|
+
timeout_config: dict
|
|
1311
|
+
Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
|
|
1312
|
+
"""
|
|
1313
|
+
...
|
|
1314
|
+
|
|
1431
1315
|
@typing.overload
|
|
1432
|
-
def
|
|
1316
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1433
1317
|
"""
|
|
1434
|
-
|
|
1318
|
+
Specifies the PyPI packages for the step.
|
|
1319
|
+
|
|
1320
|
+
Information in this decorator will augment any
|
|
1321
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
|
1322
|
+
you can use `@pypi_base` to set packages required by all
|
|
1323
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
1324
|
+
|
|
1325
|
+
|
|
1326
|
+
Parameters
|
|
1327
|
+
----------
|
|
1328
|
+
packages : Dict[str, str], default: {}
|
|
1329
|
+
Packages to use for this step. The key is the name of the package
|
|
1330
|
+
and the value is the version to use.
|
|
1331
|
+
python : str, optional, default: None
|
|
1332
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1333
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1435
1334
|
"""
|
|
1436
1335
|
...
|
|
1437
1336
|
|
|
1438
1337
|
@typing.overload
|
|
1439
|
-
def
|
|
1338
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1440
1339
|
...
|
|
1441
1340
|
|
|
1442
|
-
|
|
1341
|
+
@typing.overload
|
|
1342
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1343
|
+
...
|
|
1344
|
+
|
|
1345
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
|
1443
1346
|
"""
|
|
1444
|
-
|
|
1347
|
+
Specifies the PyPI packages for the step.
|
|
1348
|
+
|
|
1349
|
+
Information in this decorator will augment any
|
|
1350
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
|
1351
|
+
you can use `@pypi_base` to set packages required by all
|
|
1352
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
1353
|
+
|
|
1354
|
+
|
|
1355
|
+
Parameters
|
|
1356
|
+
----------
|
|
1357
|
+
packages : Dict[str, str], default: {}
|
|
1358
|
+
Packages to use for this step. The key is the name of the package
|
|
1359
|
+
and the value is the version to use.
|
|
1360
|
+
python : str, optional, default: None
|
|
1361
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1362
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1445
1363
|
"""
|
|
1446
1364
|
...
|
|
1447
1365
|
|
|
1448
|
-
|
|
1366
|
+
@typing.overload
|
|
1367
|
+
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1449
1368
|
"""
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1369
|
+
Enables checkpointing for a step.
|
|
1370
|
+
|
|
1371
|
+
> Examples
|
|
1453
1372
|
|
|
1373
|
+
- Saving Checkpoints
|
|
1454
1374
|
|
|
1455
|
-
|
|
1456
|
-
|
|
1375
|
+
```python
|
|
1376
|
+
@checkpoint
|
|
1377
|
+
@step
|
|
1378
|
+
def train(self):
|
|
1379
|
+
model = create_model(self.parameters, checkpoint_path = None)
|
|
1380
|
+
for i in range(self.epochs):
|
|
1381
|
+
# some training logic
|
|
1382
|
+
loss = model.train(self.dataset)
|
|
1383
|
+
if i % 10 == 0:
|
|
1384
|
+
model.save(
|
|
1385
|
+
current.checkpoint.directory,
|
|
1386
|
+
)
|
|
1387
|
+
# saves the contents of the `current.checkpoint.directory` as a checkpoint
|
|
1388
|
+
# and returns a reference dictionary to the checkpoint saved in the datastore
|
|
1389
|
+
self.latest_checkpoint = current.checkpoint.save(
|
|
1390
|
+
name="epoch_checkpoint",
|
|
1391
|
+
metadata={
|
|
1392
|
+
"epoch": i,
|
|
1393
|
+
"loss": loss,
|
|
1394
|
+
}
|
|
1395
|
+
)
|
|
1396
|
+
```
|
|
1457
1397
|
|
|
1458
|
-
|
|
1459
|
-
points to an external bucket. It affects S3 operations performed via
|
|
1460
|
-
Metaflow's `get_aws_client` and `S3` within a `@step`.
|
|
1398
|
+
- Using Loaded Checkpoints
|
|
1461
1399
|
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1400
|
+
```python
|
|
1401
|
+
@retry(times=3)
|
|
1402
|
+
@checkpoint
|
|
1403
|
+
@step
|
|
1404
|
+
def train(self):
|
|
1405
|
+
# Assume that the task has restarted and the previous attempt of the task
|
|
1406
|
+
# saved a checkpoint
|
|
1407
|
+
checkpoint_path = None
|
|
1408
|
+
if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
|
|
1409
|
+
print("Loaded checkpoint from the previous attempt")
|
|
1410
|
+
checkpoint_path = current.checkpoint.directory
|
|
1468
1411
|
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
bucket.
|
|
1474
|
-
- If the object is not present in the external storage, the proxy
|
|
1475
|
-
fetches it from the requested bucket, caches it in the external
|
|
1476
|
-
storage, and streams the response from the origin bucket.
|
|
1412
|
+
model = create_model(self.parameters, checkpoint_path = checkpoint_path)
|
|
1413
|
+
for i in range(self.epochs):
|
|
1414
|
+
...
|
|
1415
|
+
```
|
|
1477
1416
|
|
|
1478
|
-
Warning
|
|
1479
|
-
-------
|
|
1480
|
-
All READ operations (e.g., GetObject, HeadObject) pass through the external
|
|
1481
|
-
bucket regardless of the bucket specified in user code. Even
|
|
1482
|
-
`S3(run=self)` and `S3(s3root="mybucketfoo")` requests go through the
|
|
1483
|
-
external bucket cache.
|
|
1484
1417
|
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1418
|
+
Parameters
|
|
1419
|
+
----------
|
|
1420
|
+
load_policy : str, default: "fresh"
|
|
1421
|
+
The policy for loading the checkpoint. The following policies are supported:
|
|
1422
|
+
- "eager": Loads the the latest available checkpoint within the namespace.
|
|
1423
|
+
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
|
1424
|
+
will be loaded at the start of the task.
|
|
1425
|
+
- "none": Do not load any checkpoint
|
|
1426
|
+
- "fresh": Loads the lastest checkpoint created within the running Task.
|
|
1427
|
+
This mode helps loading checkpoints across various retry attempts of the same task.
|
|
1428
|
+
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
|
1429
|
+
created within the task will be loaded when the task is retries execution on failure.
|
|
1489
1430
|
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1431
|
+
temp_dir_root : str, default: None
|
|
1432
|
+
The root directory under which `current.checkpoint.directory` will be created.
|
|
1433
|
+
"""
|
|
1434
|
+
...
|
|
1435
|
+
|
|
1436
|
+
@typing.overload
|
|
1437
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1438
|
+
...
|
|
1439
|
+
|
|
1440
|
+
@typing.overload
|
|
1441
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1442
|
+
...
|
|
1443
|
+
|
|
1444
|
+
def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
|
|
1445
|
+
"""
|
|
1446
|
+
Enables checkpointing for a step.
|
|
1447
|
+
|
|
1448
|
+
> Examples
|
|
1449
|
+
|
|
1450
|
+
- Saving Checkpoints
|
|
1451
|
+
|
|
1452
|
+
```python
|
|
1453
|
+
@checkpoint
|
|
1454
|
+
@step
|
|
1455
|
+
def train(self):
|
|
1456
|
+
model = create_model(self.parameters, checkpoint_path = None)
|
|
1457
|
+
for i in range(self.epochs):
|
|
1458
|
+
# some training logic
|
|
1459
|
+
loss = model.train(self.dataset)
|
|
1460
|
+
if i % 10 == 0:
|
|
1461
|
+
model.save(
|
|
1462
|
+
current.checkpoint.directory,
|
|
1463
|
+
)
|
|
1464
|
+
# saves the contents of the `current.checkpoint.directory` as a checkpoint
|
|
1465
|
+
# and returns a reference dictionary to the checkpoint saved in the datastore
|
|
1466
|
+
self.latest_checkpoint = current.checkpoint.save(
|
|
1467
|
+
name="epoch_checkpoint",
|
|
1468
|
+
metadata={
|
|
1469
|
+
"epoch": i,
|
|
1470
|
+
"loss": loss,
|
|
1471
|
+
}
|
|
1472
|
+
)
|
|
1473
|
+
```
|
|
1474
|
+
|
|
1475
|
+
- Using Loaded Checkpoints
|
|
1476
|
+
|
|
1477
|
+
```python
|
|
1478
|
+
@retry(times=3)
|
|
1479
|
+
@checkpoint
|
|
1480
|
+
@step
|
|
1481
|
+
def train(self):
|
|
1482
|
+
# Assume that the task has restarted and the previous attempt of the task
|
|
1483
|
+
# saved a checkpoint
|
|
1484
|
+
checkpoint_path = None
|
|
1485
|
+
if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
|
|
1486
|
+
print("Loaded checkpoint from the previous attempt")
|
|
1487
|
+
checkpoint_path = current.checkpoint.directory
|
|
1488
|
+
|
|
1489
|
+
model = create_model(self.parameters, checkpoint_path = checkpoint_path)
|
|
1490
|
+
for i in range(self.epochs):
|
|
1491
|
+
...
|
|
1492
|
+
```
|
|
1494
1493
|
|
|
1495
1494
|
|
|
1496
1495
|
Parameters
|
|
1497
1496
|
----------
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1497
|
+
load_policy : str, default: "fresh"
|
|
1498
|
+
The policy for loading the checkpoint. The following policies are supported:
|
|
1499
|
+
- "eager": Loads the the latest available checkpoint within the namespace.
|
|
1500
|
+
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
|
1501
|
+
will be loaded at the start of the task.
|
|
1502
|
+
- "none": Do not load any checkpoint
|
|
1503
|
+
- "fresh": Loads the lastest checkpoint created within the running Task.
|
|
1504
|
+
This mode helps loading checkpoints across various retry attempts of the same task.
|
|
1505
|
+
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
|
1506
|
+
created within the task will be loaded when the task is retries execution on failure.
|
|
1507
|
+
|
|
1508
|
+
temp_dir_root : str, default: None
|
|
1509
|
+
The root directory under which `current.checkpoint.directory` will be created.
|
|
1510
|
+
"""
|
|
1511
|
+
...
|
|
1512
|
+
|
|
1513
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1514
|
+
"""
|
|
1515
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
|
1516
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
|
1517
|
+
|
|
1518
|
+
|
|
1519
|
+
Parameters
|
|
1520
|
+
----------
|
|
1521
|
+
timeout : int
|
|
1522
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
|
1523
|
+
poke_interval : int
|
|
1524
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
|
1525
|
+
mode : str
|
|
1526
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
|
1527
|
+
exponential_backoff : bool
|
|
1528
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
|
1529
|
+
pool : str
|
|
1530
|
+
the slot pool this task should run in,
|
|
1531
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
|
1532
|
+
soft_fail : bool
|
|
1533
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
|
1534
|
+
name : str
|
|
1535
|
+
Name of the sensor on Airflow
|
|
1536
|
+
description : str
|
|
1537
|
+
Description of sensor in the Airflow UI
|
|
1538
|
+
external_dag_id : str
|
|
1539
|
+
The dag_id that contains the task you want to wait for.
|
|
1540
|
+
external_task_ids : List[str]
|
|
1541
|
+
The list of task_ids that you want to wait for.
|
|
1542
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
|
1543
|
+
allowed_states : List[str]
|
|
1544
|
+
Iterable of allowed states, (Default: ['success'])
|
|
1545
|
+
failed_states : List[str]
|
|
1546
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
|
1547
|
+
execution_delta : datetime.timedelta
|
|
1548
|
+
time difference with the previous execution to look at,
|
|
1549
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
|
1550
|
+
check_existence: bool
|
|
1551
|
+
Set to True to check if the external task exists or check if
|
|
1552
|
+
the DAG to wait for exists. (Default: True)
|
|
1553
|
+
"""
|
|
1554
|
+
...
|
|
1555
|
+
|
|
1556
|
+
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1557
|
+
"""
|
|
1558
|
+
Specifies what flows belong to the same project.
|
|
1559
|
+
|
|
1560
|
+
A project-specific namespace is created for all flows that
|
|
1561
|
+
use the same `@project(name)`.
|
|
1562
|
+
|
|
1563
|
+
|
|
1564
|
+
Parameters
|
|
1565
|
+
----------
|
|
1566
|
+
name : str
|
|
1567
|
+
Project name. Make sure that the name is unique amongst all
|
|
1568
|
+
projects that use the same production scheduler. The name may
|
|
1569
|
+
contain only lowercase alphanumeric characters and underscores.
|
|
1570
|
+
|
|
1571
|
+
branch : Optional[str], default None
|
|
1572
|
+
The branch to use. If not specified, the branch is set to
|
|
1573
|
+
`user.<username>` unless `production` is set to `True`. This can
|
|
1574
|
+
also be set on the command line using `--branch` as a top-level option.
|
|
1575
|
+
It is an error to specify `branch` in the decorator and on the command line.
|
|
1576
|
+
|
|
1577
|
+
production : bool, default False
|
|
1578
|
+
Whether or not the branch is the production branch. This can also be set on the
|
|
1579
|
+
command line using `--production` as a top-level option. It is an error to specify
|
|
1580
|
+
`production` in the decorator and on the command line.
|
|
1581
|
+
The project branch name will be:
|
|
1582
|
+
- if `branch` is specified:
|
|
1583
|
+
- if `production` is True: `prod.<branch>`
|
|
1584
|
+
- if `production` is False: `test.<branch>`
|
|
1585
|
+
- if `branch` is not specified:
|
|
1586
|
+
- if `production` is True: `prod`
|
|
1587
|
+
- if `production` is False: `user.<username>`
|
|
1509
1588
|
"""
|
|
1510
1589
|
...
|
|
1511
1590
|
|
|
@@ -1601,38 +1680,96 @@ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packag
|
|
|
1601
1680
|
"""
|
|
1602
1681
|
...
|
|
1603
1682
|
|
|
1604
|
-
|
|
1683
|
+
@typing.overload
|
|
1684
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1605
1685
|
"""
|
|
1606
|
-
Specifies
|
|
1686
|
+
Specifies the event(s) that this flow depends on.
|
|
1607
1687
|
|
|
1608
|
-
|
|
1609
|
-
|
|
1688
|
+
```
|
|
1689
|
+
@trigger(event='foo')
|
|
1690
|
+
```
|
|
1691
|
+
or
|
|
1692
|
+
```
|
|
1693
|
+
@trigger(events=['foo', 'bar'])
|
|
1694
|
+
```
|
|
1695
|
+
|
|
1696
|
+
Additionally, you can specify the parameter mappings
|
|
1697
|
+
to map event payload to Metaflow parameters for the flow.
|
|
1698
|
+
```
|
|
1699
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
|
1700
|
+
```
|
|
1701
|
+
or
|
|
1702
|
+
```
|
|
1703
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
|
1704
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
|
1705
|
+
```
|
|
1706
|
+
|
|
1707
|
+
'parameters' can also be a list of strings and tuples like so:
|
|
1708
|
+
```
|
|
1709
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
1710
|
+
```
|
|
1711
|
+
This is equivalent to:
|
|
1712
|
+
```
|
|
1713
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
|
1714
|
+
```
|
|
1610
1715
|
|
|
1611
1716
|
|
|
1612
1717
|
Parameters
|
|
1613
1718
|
----------
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1719
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
|
1720
|
+
Event dependency for this flow.
|
|
1721
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
|
1722
|
+
Events dependency for this flow.
|
|
1723
|
+
options : Dict[str, Any], default {}
|
|
1724
|
+
Backend-specific configuration for tuning eventing behavior.
|
|
1725
|
+
"""
|
|
1726
|
+
...
|
|
1727
|
+
|
|
1728
|
+
@typing.overload
|
|
1729
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1730
|
+
...
|
|
1731
|
+
|
|
1732
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
|
1733
|
+
"""
|
|
1734
|
+
Specifies the event(s) that this flow depends on.
|
|
1618
1735
|
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1736
|
+
```
|
|
1737
|
+
@trigger(event='foo')
|
|
1738
|
+
```
|
|
1739
|
+
or
|
|
1740
|
+
```
|
|
1741
|
+
@trigger(events=['foo', 'bar'])
|
|
1742
|
+
```
|
|
1624
1743
|
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1744
|
+
Additionally, you can specify the parameter mappings
|
|
1745
|
+
to map event payload to Metaflow parameters for the flow.
|
|
1746
|
+
```
|
|
1747
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
|
1748
|
+
```
|
|
1749
|
+
or
|
|
1750
|
+
```
|
|
1751
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
|
1752
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
|
1753
|
+
```
|
|
1754
|
+
|
|
1755
|
+
'parameters' can also be a list of strings and tuples like so:
|
|
1756
|
+
```
|
|
1757
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
1758
|
+
```
|
|
1759
|
+
This is equivalent to:
|
|
1760
|
+
```
|
|
1761
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
|
1762
|
+
```
|
|
1763
|
+
|
|
1764
|
+
|
|
1765
|
+
Parameters
|
|
1766
|
+
----------
|
|
1767
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
|
1768
|
+
Event dependency for this flow.
|
|
1769
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
|
1770
|
+
Events dependency for this flow.
|
|
1771
|
+
options : Dict[str, Any], default {}
|
|
1772
|
+
Backend-specific configuration for tuning eventing behavior.
|
|
1636
1773
|
"""
|
|
1637
1774
|
...
|
|
1638
1775
|
|
|
@@ -1844,49 +1981,6 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
|
|
|
1844
1981
|
"""
|
|
1845
1982
|
...
|
|
1846
1983
|
|
|
1847
|
-
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1848
|
-
"""
|
|
1849
|
-
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
|
1850
|
-
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
Parameters
|
|
1854
|
-
----------
|
|
1855
|
-
timeout : int
|
|
1856
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
|
1857
|
-
poke_interval : int
|
|
1858
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
|
1859
|
-
mode : str
|
|
1860
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
|
1861
|
-
exponential_backoff : bool
|
|
1862
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
|
1863
|
-
pool : str
|
|
1864
|
-
the slot pool this task should run in,
|
|
1865
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
|
1866
|
-
soft_fail : bool
|
|
1867
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
|
1868
|
-
name : str
|
|
1869
|
-
Name of the sensor on Airflow
|
|
1870
|
-
description : str
|
|
1871
|
-
Description of sensor in the Airflow UI
|
|
1872
|
-
external_dag_id : str
|
|
1873
|
-
The dag_id that contains the task you want to wait for.
|
|
1874
|
-
external_task_ids : List[str]
|
|
1875
|
-
The list of task_ids that you want to wait for.
|
|
1876
|
-
If None (default value) the sensor waits for the DAG. (Default: None)
|
|
1877
|
-
allowed_states : List[str]
|
|
1878
|
-
Iterable of allowed states, (Default: ['success'])
|
|
1879
|
-
failed_states : List[str]
|
|
1880
|
-
Iterable of failed or dis-allowed states. (Default: None)
|
|
1881
|
-
execution_delta : datetime.timedelta
|
|
1882
|
-
time difference with the previous execution to look at,
|
|
1883
|
-
the default is the same logical date as the current task or DAG. (Default: None)
|
|
1884
|
-
check_existence: bool
|
|
1885
|
-
Set to True to check if the external task exists or check if
|
|
1886
|
-
the DAG to wait for exists. (Default: True)
|
|
1887
|
-
"""
|
|
1888
|
-
...
|
|
1889
|
-
|
|
1890
1984
|
@typing.overload
|
|
1891
1985
|
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1892
1986
|
"""
|
|
@@ -1988,98 +2082,5 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
|
1988
2082
|
"""
|
|
1989
2083
|
...
|
|
1990
2084
|
|
|
1991
|
-
@typing.overload
|
|
1992
|
-
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1993
|
-
"""
|
|
1994
|
-
Specifies the event(s) that this flow depends on.
|
|
1995
|
-
|
|
1996
|
-
```
|
|
1997
|
-
@trigger(event='foo')
|
|
1998
|
-
```
|
|
1999
|
-
or
|
|
2000
|
-
```
|
|
2001
|
-
@trigger(events=['foo', 'bar'])
|
|
2002
|
-
```
|
|
2003
|
-
|
|
2004
|
-
Additionally, you can specify the parameter mappings
|
|
2005
|
-
to map event payload to Metaflow parameters for the flow.
|
|
2006
|
-
```
|
|
2007
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
|
2008
|
-
```
|
|
2009
|
-
or
|
|
2010
|
-
```
|
|
2011
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
|
2012
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
|
2013
|
-
```
|
|
2014
|
-
|
|
2015
|
-
'parameters' can also be a list of strings and tuples like so:
|
|
2016
|
-
```
|
|
2017
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
2018
|
-
```
|
|
2019
|
-
This is equivalent to:
|
|
2020
|
-
```
|
|
2021
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
|
2022
|
-
```
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
Parameters
|
|
2026
|
-
----------
|
|
2027
|
-
event : Union[str, Dict[str, Any]], optional, default None
|
|
2028
|
-
Event dependency for this flow.
|
|
2029
|
-
events : List[Union[str, Dict[str, Any]]], default []
|
|
2030
|
-
Events dependency for this flow.
|
|
2031
|
-
options : Dict[str, Any], default {}
|
|
2032
|
-
Backend-specific configuration for tuning eventing behavior.
|
|
2033
|
-
"""
|
|
2034
|
-
...
|
|
2035
|
-
|
|
2036
|
-
@typing.overload
|
|
2037
|
-
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
2038
|
-
...
|
|
2039
|
-
|
|
2040
|
-
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
|
2041
|
-
"""
|
|
2042
|
-
Specifies the event(s) that this flow depends on.
|
|
2043
|
-
|
|
2044
|
-
```
|
|
2045
|
-
@trigger(event='foo')
|
|
2046
|
-
```
|
|
2047
|
-
or
|
|
2048
|
-
```
|
|
2049
|
-
@trigger(events=['foo', 'bar'])
|
|
2050
|
-
```
|
|
2051
|
-
|
|
2052
|
-
Additionally, you can specify the parameter mappings
|
|
2053
|
-
to map event payload to Metaflow parameters for the flow.
|
|
2054
|
-
```
|
|
2055
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
|
2056
|
-
```
|
|
2057
|
-
or
|
|
2058
|
-
```
|
|
2059
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
|
2060
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
|
2061
|
-
```
|
|
2062
|
-
|
|
2063
|
-
'parameters' can also be a list of strings and tuples like so:
|
|
2064
|
-
```
|
|
2065
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
2066
|
-
```
|
|
2067
|
-
This is equivalent to:
|
|
2068
|
-
```
|
|
2069
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
|
2070
|
-
```
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
Parameters
|
|
2074
|
-
----------
|
|
2075
|
-
event : Union[str, Dict[str, Any]], optional, default None
|
|
2076
|
-
Event dependency for this flow.
|
|
2077
|
-
events : List[Union[str, Dict[str, Any]]], default []
|
|
2078
|
-
Events dependency for this flow.
|
|
2079
|
-
options : Dict[str, Any], default {}
|
|
2080
|
-
Backend-specific configuration for tuning eventing behavior.
|
|
2081
|
-
"""
|
|
2082
|
-
...
|
|
2083
|
-
|
|
2084
2085
|
pkg_name: str
|
|
2085
2086
|
|