ob-metaflow-stubs 6.0.5.2__py2.py3-none-any.whl → 6.0.6.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +911 -911
- metaflow-stubs/cards.pyi +2 -2
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/cli_components/__init__.pyi +2 -2
- metaflow-stubs/cli_components/utils.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +2 -2
- metaflow-stubs/client/core.pyi +5 -5
- metaflow-stubs/client/filecache.pyi +3 -3
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +4 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +3 -3
- metaflow-stubs/meta_files.pyi +2 -2
- metaflow-stubs/metadata_provider/__init__.pyi +2 -2
- metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
- metaflow-stubs/metadata_provider/metadata.pyi +2 -2
- metaflow-stubs/metadata_provider/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +38 -38
- metaflow-stubs/metaflow_git.pyi +2 -2
- metaflow-stubs/mf_extensions/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/ob_internal.pyi +2 -2
- metaflow-stubs/packaging_sys/__init__.pyi +5 -5
- metaflow-stubs/packaging_sys/backend.pyi +3 -3
- metaflow-stubs/packaging_sys/distribution_support.pyi +4 -4
- metaflow-stubs/packaging_sys/tar_backend.pyi +5 -5
- metaflow-stubs/packaging_sys/utils.pyi +2 -2
- metaflow-stubs/packaging_sys/v1.pyi +2 -2
- metaflow-stubs/parameters.pyi +3 -3
- metaflow-stubs/plugins/__init__.pyi +13 -13
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
- metaflow-stubs/plugins/argo/exit_hooks.pyi +2 -2
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +2 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +2 -2
- metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
- metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
- metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
- metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
- metaflow-stubs/plugins/secrets/utils.pyi +2 -2
- metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
- metaflow-stubs/plugins/uv/__init__.pyi +2 -2
- metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +32 -32
- metaflow-stubs/runner/deployer_impl.pyi +2 -2
- metaflow-stubs/runner/metaflow_runner.pyi +3 -3
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +3 -3
- metaflow-stubs/system/__init__.pyi +2 -2
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/user_configs/__init__.pyi +2 -2
- metaflow-stubs/user_configs/config_options.pyi +2 -2
- metaflow-stubs/user_configs/config_parameters.pyi +5 -5
- metaflow-stubs/user_decorators/__init__.pyi +2 -2
- metaflow-stubs/user_decorators/common.pyi +2 -2
- metaflow-stubs/user_decorators/mutable_flow.pyi +5 -5
- metaflow-stubs/user_decorators/mutable_step.pyi +6 -6
- metaflow-stubs/user_decorators/user_flow_decorator.pyi +4 -4
- metaflow-stubs/user_decorators/user_step_decorator.pyi +4 -4
- {ob_metaflow_stubs-6.0.5.2.dist-info → ob_metaflow_stubs-6.0.6.0.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.6.0.dist-info/RECORD +261 -0
- ob_metaflow_stubs-6.0.5.2.dist-info/RECORD +0 -261
- {ob_metaflow_stubs-6.0.5.2.dist-info → ob_metaflow_stubs-6.0.6.0.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.5.2.dist-info → ob_metaflow_stubs-6.0.6.0.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
######################################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.
|
4
|
-
# Generated on 2025-08-
|
3
|
+
# MF version: 2.17.0.1+obcheckpoint(0.2.4);ob(v1) #
|
4
|
+
# Generated on 2025-08-05T23:30:10.180789 #
|
5
5
|
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -39,9 +39,9 @@ from .user_decorators.user_step_decorator import UserStepDecorator as UserStepDe
|
|
39
39
|
from .user_decorators.user_step_decorator import StepMutator as StepMutator
|
40
40
|
from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
|
41
41
|
from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
|
42
|
+
from . import metaflow_git as metaflow_git
|
42
43
|
from . import cards as cards
|
43
44
|
from . import tuple_util as tuple_util
|
44
|
-
from . import metaflow_git as metaflow_git
|
45
45
|
from . import events as events
|
46
46
|
from . import runner as runner
|
47
47
|
from . import plugins as plugins
|
@@ -49,8 +49,8 @@ from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package imp
|
|
49
49
|
from . import includefile as includefile
|
50
50
|
from .includefile import IncludeFile as IncludeFile
|
51
51
|
from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
|
52
|
-
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
53
52
|
from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
|
53
|
+
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
54
54
|
from . import client as client
|
55
55
|
from .client.core import namespace as namespace
|
56
56
|
from .client.core import get_namespace as get_namespace
|
@@ -167,504 +167,386 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
167
167
|
"""
|
168
168
|
...
|
169
169
|
|
170
|
-
|
170
|
+
@typing.overload
|
171
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
171
172
|
"""
|
172
|
-
|
173
|
-
|
174
|
-
User code call
|
175
|
-
--------------
|
176
|
-
@ollama(
|
177
|
-
models=[...],
|
178
|
-
...
|
179
|
-
)
|
180
|
-
|
181
|
-
Valid backend options
|
182
|
-
---------------------
|
183
|
-
- 'local': Run as a separate process on the local task machine.
|
184
|
-
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
185
|
-
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
186
|
-
|
187
|
-
Valid model options
|
188
|
-
-------------------
|
189
|
-
Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
|
173
|
+
Specifies environment variables to be set prior to the execution of a step.
|
190
174
|
|
191
175
|
|
192
176
|
Parameters
|
193
177
|
----------
|
194
|
-
|
195
|
-
|
196
|
-
backend: str
|
197
|
-
Determines where and how to run the Ollama process.
|
198
|
-
force_pull: bool
|
199
|
-
Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
|
200
|
-
cache_update_policy: str
|
201
|
-
Cache update policy: "auto", "force", or "never".
|
202
|
-
force_cache_update: bool
|
203
|
-
Simple override for "force" cache update policy.
|
204
|
-
debug: bool
|
205
|
-
Whether to turn on verbose debugging logs.
|
206
|
-
circuit_breaker_config: dict
|
207
|
-
Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
|
208
|
-
timeout_config: dict
|
209
|
-
Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
|
178
|
+
vars : Dict[str, str], default {}
|
179
|
+
Dictionary of environment variables to set.
|
210
180
|
"""
|
211
181
|
...
|
212
182
|
|
213
|
-
|
183
|
+
@typing.overload
|
184
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
185
|
+
...
|
186
|
+
|
187
|
+
@typing.overload
|
188
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
189
|
+
...
|
190
|
+
|
191
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
214
192
|
"""
|
215
|
-
|
193
|
+
Specifies environment variables to be set prior to the execution of a step.
|
216
194
|
|
217
195
|
|
218
196
|
Parameters
|
219
197
|
----------
|
220
|
-
|
221
|
-
|
222
|
-
available S3 proxy integration in the namespace (fails if multiple exist).
|
223
|
-
write_mode : str, optional
|
224
|
-
The desired behavior during write operations to target (origin) S3 bucket.
|
225
|
-
allowed options are:
|
226
|
-
"origin-and-cache" -> write to both the target S3 bucket and local object
|
227
|
-
storage
|
228
|
-
"origin" -> only write to the target S3 bucket
|
229
|
-
"cache" -> only write to the object storage service used for caching
|
230
|
-
debug : bool, optional
|
231
|
-
Enable debug logging for proxy operations.
|
198
|
+
vars : Dict[str, str], default {}
|
199
|
+
Dictionary of environment variables to set.
|
232
200
|
"""
|
233
201
|
...
|
234
202
|
|
235
203
|
@typing.overload
|
236
|
-
def
|
204
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
237
205
|
"""
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
# and returns a reference dictionary to the checkpoint saved in the datastore
|
258
|
-
self.latest_checkpoint = current.checkpoint.save(
|
259
|
-
name="epoch_checkpoint",
|
260
|
-
metadata={
|
261
|
-
"epoch": i,
|
262
|
-
"loss": loss,
|
263
|
-
}
|
264
|
-
)
|
265
|
-
```
|
266
|
-
|
267
|
-
- Using Loaded Checkpoints
|
206
|
+
Internal decorator to support Fast bakery
|
207
|
+
"""
|
208
|
+
...
|
209
|
+
|
210
|
+
@typing.overload
|
211
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
212
|
+
...
|
213
|
+
|
214
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
215
|
+
"""
|
216
|
+
Internal decorator to support Fast bakery
|
217
|
+
"""
|
218
|
+
...
|
219
|
+
|
220
|
+
@typing.overload
|
221
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
222
|
+
"""
|
223
|
+
Specifies the number of times the task corresponding
|
224
|
+
to a step needs to be retried.
|
268
225
|
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
@step
|
273
|
-
def train(self):
|
274
|
-
# Assume that the task has restarted and the previous attempt of the task
|
275
|
-
# saved a checkpoint
|
276
|
-
checkpoint_path = None
|
277
|
-
if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
|
278
|
-
print("Loaded checkpoint from the previous attempt")
|
279
|
-
checkpoint_path = current.checkpoint.directory
|
226
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
227
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
228
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
280
229
|
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
```
|
230
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
231
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
232
|
+
ensuring that the flow execution can continue.
|
285
233
|
|
286
234
|
|
287
235
|
Parameters
|
288
236
|
----------
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
will be loaded at the start of the task.
|
294
|
-
- "none": Do not load any checkpoint
|
295
|
-
- "fresh": Loads the lastest checkpoint created within the running Task.
|
296
|
-
This mode helps loading checkpoints across various retry attempts of the same task.
|
297
|
-
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
298
|
-
created within the task will be loaded when the task is retries execution on failure.
|
299
|
-
|
300
|
-
temp_dir_root : str, default: None
|
301
|
-
The root directory under which `current.checkpoint.directory` will be created.
|
237
|
+
times : int, default 3
|
238
|
+
Number of times to retry this task.
|
239
|
+
minutes_between_retries : int, default 2
|
240
|
+
Number of minutes between retries.
|
302
241
|
"""
|
303
242
|
...
|
304
243
|
|
305
244
|
@typing.overload
|
306
|
-
def
|
245
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
307
246
|
...
|
308
247
|
|
309
248
|
@typing.overload
|
310
|
-
def
|
249
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
311
250
|
...
|
312
251
|
|
313
|
-
def
|
252
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
314
253
|
"""
|
315
|
-
|
316
|
-
|
317
|
-
> Examples
|
318
|
-
|
319
|
-
- Saving Checkpoints
|
320
|
-
|
321
|
-
```python
|
322
|
-
@checkpoint
|
323
|
-
@step
|
324
|
-
def train(self):
|
325
|
-
model = create_model(self.parameters, checkpoint_path = None)
|
326
|
-
for i in range(self.epochs):
|
327
|
-
# some training logic
|
328
|
-
loss = model.train(self.dataset)
|
329
|
-
if i % 10 == 0:
|
330
|
-
model.save(
|
331
|
-
current.checkpoint.directory,
|
332
|
-
)
|
333
|
-
# saves the contents of the `current.checkpoint.directory` as a checkpoint
|
334
|
-
# and returns a reference dictionary to the checkpoint saved in the datastore
|
335
|
-
self.latest_checkpoint = current.checkpoint.save(
|
336
|
-
name="epoch_checkpoint",
|
337
|
-
metadata={
|
338
|
-
"epoch": i,
|
339
|
-
"loss": loss,
|
340
|
-
}
|
341
|
-
)
|
342
|
-
```
|
343
|
-
|
344
|
-
- Using Loaded Checkpoints
|
254
|
+
Specifies the number of times the task corresponding
|
255
|
+
to a step needs to be retried.
|
345
256
|
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
@step
|
350
|
-
def train(self):
|
351
|
-
# Assume that the task has restarted and the previous attempt of the task
|
352
|
-
# saved a checkpoint
|
353
|
-
checkpoint_path = None
|
354
|
-
if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
|
355
|
-
print("Loaded checkpoint from the previous attempt")
|
356
|
-
checkpoint_path = current.checkpoint.directory
|
257
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
258
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
259
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
357
260
|
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
```
|
261
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
262
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
263
|
+
ensuring that the flow execution can continue.
|
362
264
|
|
363
265
|
|
364
266
|
Parameters
|
365
267
|
----------
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
will be loaded at the start of the task.
|
371
|
-
- "none": Do not load any checkpoint
|
372
|
-
- "fresh": Loads the lastest checkpoint created within the running Task.
|
373
|
-
This mode helps loading checkpoints across various retry attempts of the same task.
|
374
|
-
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
375
|
-
created within the task will be loaded when the task is retries execution on failure.
|
376
|
-
|
377
|
-
temp_dir_root : str, default: None
|
378
|
-
The root directory under which `current.checkpoint.directory` will be created.
|
268
|
+
times : int, default 3
|
269
|
+
Number of times to retry this task.
|
270
|
+
minutes_between_retries : int, default 2
|
271
|
+
Number of minutes between retries.
|
379
272
|
"""
|
380
273
|
...
|
381
274
|
|
382
275
|
@typing.overload
|
383
|
-
def
|
276
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
384
277
|
"""
|
385
|
-
Specifies
|
278
|
+
Specifies the PyPI packages for the step.
|
279
|
+
|
280
|
+
Information in this decorator will augment any
|
281
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
282
|
+
you can use `@pypi_base` to set packages required by all
|
283
|
+
steps and use `@pypi` to specify step-specific overrides.
|
386
284
|
|
387
285
|
|
388
286
|
Parameters
|
389
287
|
----------
|
390
|
-
|
391
|
-
|
288
|
+
packages : Dict[str, str], default: {}
|
289
|
+
Packages to use for this step. The key is the name of the package
|
290
|
+
and the value is the version to use.
|
291
|
+
python : str, optional, default: None
|
292
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
293
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
392
294
|
"""
|
393
295
|
...
|
394
296
|
|
395
297
|
@typing.overload
|
396
|
-
def
|
298
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
397
299
|
...
|
398
300
|
|
399
301
|
@typing.overload
|
400
|
-
def
|
302
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
401
303
|
...
|
402
304
|
|
403
|
-
def
|
404
|
-
"""
|
405
|
-
Specifies environment variables to be set prior to the execution of a step.
|
406
|
-
|
407
|
-
|
408
|
-
Parameters
|
409
|
-
----------
|
410
|
-
vars : Dict[str, str], default {}
|
411
|
-
Dictionary of environment variables to set.
|
412
|
-
"""
|
413
|
-
...
|
414
|
-
|
415
|
-
@typing.overload
|
416
|
-
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
305
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
417
306
|
"""
|
418
|
-
Specifies
|
307
|
+
Specifies the PyPI packages for the step.
|
419
308
|
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
309
|
+
Information in this decorator will augment any
|
310
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
311
|
+
you can use `@pypi_base` to set packages required by all
|
312
|
+
steps and use `@pypi` to specify step-specific overrides.
|
424
313
|
|
425
314
|
|
426
315
|
Parameters
|
427
316
|
----------
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
317
|
+
packages : Dict[str, str], default: {}
|
318
|
+
Packages to use for this step. The key is the name of the package
|
319
|
+
and the value is the version to use.
|
320
|
+
python : str, optional, default: None
|
321
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
322
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
434
323
|
"""
|
435
324
|
...
|
436
325
|
|
437
326
|
@typing.overload
|
438
|
-
def
|
439
|
-
...
|
440
|
-
|
441
|
-
@typing.overload
|
442
|
-
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
443
|
-
...
|
444
|
-
|
445
|
-
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
446
|
-
"""
|
447
|
-
Specifies that the step will success under all circumstances.
|
448
|
-
|
449
|
-
The decorator will create an optional artifact, specified by `var`, which
|
450
|
-
contains the exception raised. You can use it to detect the presence
|
451
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
452
|
-
are missing.
|
453
|
-
|
454
|
-
|
455
|
-
Parameters
|
456
|
-
----------
|
457
|
-
var : str, optional, default None
|
458
|
-
Name of the artifact in which to store the caught exception.
|
459
|
-
If not specified, the exception is not stored.
|
460
|
-
print_exception : bool, default True
|
461
|
-
Determines whether or not the exception is printed to
|
462
|
-
stdout when caught.
|
463
|
-
"""
|
464
|
-
...
|
465
|
-
|
466
|
-
def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
327
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
467
328
|
"""
|
468
|
-
Specifies
|
329
|
+
Specifies a timeout for your step.
|
469
330
|
|
331
|
+
This decorator is useful if this step may hang indefinitely.
|
470
332
|
|
471
|
-
|
472
|
-
|
473
|
-
|
474
|
-
Number of GPUs to use.
|
475
|
-
gpu_type : str
|
476
|
-
Type of Nvidia GPU to use.
|
477
|
-
queue_timeout : int
|
478
|
-
Time to keep the job in NVCF's queue.
|
479
|
-
"""
|
480
|
-
...
|
481
|
-
|
482
|
-
@typing.overload
|
483
|
-
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
484
|
-
"""
|
485
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
333
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
334
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
335
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
486
336
|
|
487
|
-
Note that
|
337
|
+
Note that all the values specified in parameters are added together so if you specify
|
338
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
488
339
|
|
489
340
|
|
490
341
|
Parameters
|
491
342
|
----------
|
492
|
-
|
493
|
-
|
494
|
-
|
495
|
-
|
496
|
-
|
497
|
-
|
498
|
-
timeout : int, default 45
|
499
|
-
Interrupt reporting if it takes more than this many seconds.
|
343
|
+
seconds : int, default 0
|
344
|
+
Number of seconds to wait prior to timing out.
|
345
|
+
minutes : int, default 0
|
346
|
+
Number of minutes to wait prior to timing out.
|
347
|
+
hours : int, default 0
|
348
|
+
Number of hours to wait prior to timing out.
|
500
349
|
"""
|
501
350
|
...
|
502
351
|
|
503
352
|
@typing.overload
|
504
|
-
def
|
353
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
505
354
|
...
|
506
355
|
|
507
356
|
@typing.overload
|
508
|
-
def
|
357
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
509
358
|
...
|
510
359
|
|
511
|
-
def
|
360
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
512
361
|
"""
|
513
|
-
|
362
|
+
Specifies a timeout for your step.
|
514
363
|
|
515
|
-
|
364
|
+
This decorator is useful if this step may hang indefinitely.
|
365
|
+
|
366
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
367
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
368
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
369
|
+
|
370
|
+
Note that all the values specified in parameters are added together so if you specify
|
371
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
516
372
|
|
517
373
|
|
518
374
|
Parameters
|
519
375
|
----------
|
520
|
-
|
521
|
-
|
522
|
-
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
timeout : int, default 45
|
527
|
-
Interrupt reporting if it takes more than this many seconds.
|
376
|
+
seconds : int, default 0
|
377
|
+
Number of seconds to wait prior to timing out.
|
378
|
+
minutes : int, default 0
|
379
|
+
Number of minutes to wait prior to timing out.
|
380
|
+
hours : int, default 0
|
381
|
+
Number of hours to wait prior to timing out.
|
528
382
|
"""
|
529
383
|
...
|
530
384
|
|
531
385
|
@typing.overload
|
532
|
-
def
|
386
|
+
def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
533
387
|
"""
|
534
|
-
|
388
|
+
Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
389
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
390
|
+
a Neo Cloud like Nebius.
|
535
391
|
"""
|
536
392
|
...
|
537
393
|
|
538
394
|
@typing.overload
|
539
|
-
def
|
395
|
+
def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
540
396
|
...
|
541
397
|
|
542
|
-
def
|
398
|
+
def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
543
399
|
"""
|
544
|
-
|
400
|
+
Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
401
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
402
|
+
a Neo Cloud like Nebius.
|
545
403
|
"""
|
546
404
|
...
|
547
405
|
|
548
|
-
|
406
|
+
@typing.overload
|
407
|
+
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
549
408
|
"""
|
550
|
-
|
409
|
+
Enables checkpointing for a step.
|
551
410
|
|
411
|
+
> Examples
|
552
412
|
|
553
|
-
|
554
|
-
----------
|
555
|
-
gpu : int
|
556
|
-
Number of GPUs to use.
|
557
|
-
gpu_type : str
|
558
|
-
Type of Nvidia GPU to use.
|
559
|
-
"""
|
560
|
-
...
|
561
|
-
|
562
|
-
@typing.overload
|
563
|
-
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
564
|
-
"""
|
565
|
-
Specifies the PyPI packages for the step.
|
413
|
+
- Saving Checkpoints
|
566
414
|
|
567
|
-
|
568
|
-
|
569
|
-
|
570
|
-
|
415
|
+
```python
|
416
|
+
@checkpoint
|
417
|
+
@step
|
418
|
+
def train(self):
|
419
|
+
model = create_model(self.parameters, checkpoint_path = None)
|
420
|
+
for i in range(self.epochs):
|
421
|
+
# some training logic
|
422
|
+
loss = model.train(self.dataset)
|
423
|
+
if i % 10 == 0:
|
424
|
+
model.save(
|
425
|
+
current.checkpoint.directory,
|
426
|
+
)
|
427
|
+
# saves the contents of the `current.checkpoint.directory` as a checkpoint
|
428
|
+
# and returns a reference dictionary to the checkpoint saved in the datastore
|
429
|
+
self.latest_checkpoint = current.checkpoint.save(
|
430
|
+
name="epoch_checkpoint",
|
431
|
+
metadata={
|
432
|
+
"epoch": i,
|
433
|
+
"loss": loss,
|
434
|
+
}
|
435
|
+
)
|
436
|
+
```
|
437
|
+
|
438
|
+
- Using Loaded Checkpoints
|
439
|
+
|
440
|
+
```python
|
441
|
+
@retry(times=3)
|
442
|
+
@checkpoint
|
443
|
+
@step
|
444
|
+
def train(self):
|
445
|
+
# Assume that the task has restarted and the previous attempt of the task
|
446
|
+
# saved a checkpoint
|
447
|
+
checkpoint_path = None
|
448
|
+
if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
|
449
|
+
print("Loaded checkpoint from the previous attempt")
|
450
|
+
checkpoint_path = current.checkpoint.directory
|
451
|
+
|
452
|
+
model = create_model(self.parameters, checkpoint_path = checkpoint_path)
|
453
|
+
for i in range(self.epochs):
|
454
|
+
...
|
455
|
+
```
|
571
456
|
|
572
457
|
|
573
458
|
Parameters
|
574
459
|
----------
|
575
|
-
|
576
|
-
|
577
|
-
|
578
|
-
|
579
|
-
|
580
|
-
|
460
|
+
load_policy : str, default: "fresh"
|
461
|
+
The policy for loading the checkpoint. The following policies are supported:
|
462
|
+
- "eager": Loads the the latest available checkpoint within the namespace.
|
463
|
+
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
464
|
+
will be loaded at the start of the task.
|
465
|
+
- "none": Do not load any checkpoint
|
466
|
+
- "fresh": Loads the lastest checkpoint created within the running Task.
|
467
|
+
This mode helps loading checkpoints across various retry attempts of the same task.
|
468
|
+
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
469
|
+
created within the task will be loaded when the task is retries execution on failure.
|
470
|
+
|
471
|
+
temp_dir_root : str, default: None
|
472
|
+
The root directory under which `current.checkpoint.directory` will be created.
|
581
473
|
"""
|
582
474
|
...
|
583
475
|
|
584
476
|
@typing.overload
|
585
|
-
def
|
477
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
586
478
|
...
|
587
479
|
|
588
480
|
@typing.overload
|
589
|
-
def
|
481
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
590
482
|
...
|
591
483
|
|
592
|
-
def
|
484
|
+
def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
|
593
485
|
"""
|
594
|
-
|
486
|
+
Enables checkpointing for a step.
|
595
487
|
|
596
|
-
|
597
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
598
|
-
you can use `@pypi_base` to set packages required by all
|
599
|
-
steps and use `@pypi` to specify step-specific overrides.
|
488
|
+
> Examples
|
600
489
|
|
490
|
+
- Saving Checkpoints
|
601
491
|
|
602
|
-
|
603
|
-
|
604
|
-
|
605
|
-
|
606
|
-
|
607
|
-
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
|
612
|
-
|
613
|
-
|
614
|
-
|
615
|
-
|
616
|
-
|
617
|
-
|
492
|
+
```python
|
493
|
+
@checkpoint
|
494
|
+
@step
|
495
|
+
def train(self):
|
496
|
+
model = create_model(self.parameters, checkpoint_path = None)
|
497
|
+
for i in range(self.epochs):
|
498
|
+
# some training logic
|
499
|
+
loss = model.train(self.dataset)
|
500
|
+
if i % 10 == 0:
|
501
|
+
model.save(
|
502
|
+
current.checkpoint.directory,
|
503
|
+
)
|
504
|
+
# saves the contents of the `current.checkpoint.directory` as a checkpoint
|
505
|
+
# and returns a reference dictionary to the checkpoint saved in the datastore
|
506
|
+
self.latest_checkpoint = current.checkpoint.save(
|
507
|
+
name="epoch_checkpoint",
|
508
|
+
metadata={
|
509
|
+
"epoch": i,
|
510
|
+
"loss": loss,
|
511
|
+
}
|
512
|
+
)
|
513
|
+
```
|
618
514
|
|
515
|
+
- Using Loaded Checkpoints
|
619
516
|
|
620
|
-
|
621
|
-
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
|
633
|
-
|
634
|
-
|
635
|
-
|
636
|
-
|
637
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
|
638
|
-
"""
|
639
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
640
|
-
the execution of a step.
|
517
|
+
```python
|
518
|
+
@retry(times=3)
|
519
|
+
@checkpoint
|
520
|
+
@step
|
521
|
+
def train(self):
|
522
|
+
# Assume that the task has restarted and the previous attempt of the task
|
523
|
+
# saved a checkpoint
|
524
|
+
checkpoint_path = None
|
525
|
+
if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
|
526
|
+
print("Loaded checkpoint from the previous attempt")
|
527
|
+
checkpoint_path = current.checkpoint.directory
|
528
|
+
|
529
|
+
model = create_model(self.parameters, checkpoint_path = checkpoint_path)
|
530
|
+
for i in range(self.epochs):
|
531
|
+
...
|
532
|
+
```
|
641
533
|
|
642
534
|
|
643
535
|
Parameters
|
644
536
|
----------
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
|
651
|
-
|
652
|
-
|
653
|
-
|
654
|
-
|
655
|
-
|
656
|
-
|
657
|
-
|
658
|
-
...
|
659
|
-
|
660
|
-
@typing.overload
|
661
|
-
def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
662
|
-
...
|
663
|
-
|
664
|
-
def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
665
|
-
"""
|
666
|
-
A simple decorator that demonstrates using CardDecoratorInjector
|
667
|
-
to inject a card and render simple markdown content.
|
537
|
+
load_policy : str, default: "fresh"
|
538
|
+
The policy for loading the checkpoint. The following policies are supported:
|
539
|
+
- "eager": Loads the the latest available checkpoint within the namespace.
|
540
|
+
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
541
|
+
will be loaded at the start of the task.
|
542
|
+
- "none": Do not load any checkpoint
|
543
|
+
- "fresh": Loads the lastest checkpoint created within the running Task.
|
544
|
+
This mode helps loading checkpoints across various retry attempts of the same task.
|
545
|
+
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
546
|
+
created within the task will be loaded when the task is retries execution on failure.
|
547
|
+
|
548
|
+
temp_dir_root : str, default: None
|
549
|
+
The root directory under which `current.checkpoint.directory` will be created.
|
668
550
|
"""
|
669
551
|
...
|
670
552
|
|
@@ -757,82 +639,6 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
757
639
|
"""
|
758
640
|
...
|
759
641
|
|
760
|
-
@typing.overload
|
761
|
-
def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
762
|
-
"""
|
763
|
-
Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
764
|
-
It exists to make it easier for users to know that this decorator should only be used with
|
765
|
-
a Neo Cloud like Nebius.
|
766
|
-
"""
|
767
|
-
...
|
768
|
-
|
769
|
-
@typing.overload
|
770
|
-
def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
771
|
-
...
|
772
|
-
|
773
|
-
def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
774
|
-
"""
|
775
|
-
Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
776
|
-
It exists to make it easier for users to know that this decorator should only be used with
|
777
|
-
a Neo Cloud like Nebius.
|
778
|
-
"""
|
779
|
-
...
|
780
|
-
|
781
|
-
@typing.overload
|
782
|
-
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
783
|
-
"""
|
784
|
-
Specifies the number of times the task corresponding
|
785
|
-
to a step needs to be retried.
|
786
|
-
|
787
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
788
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
789
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
790
|
-
|
791
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
792
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
793
|
-
ensuring that the flow execution can continue.
|
794
|
-
|
795
|
-
|
796
|
-
Parameters
|
797
|
-
----------
|
798
|
-
times : int, default 3
|
799
|
-
Number of times to retry this task.
|
800
|
-
minutes_between_retries : int, default 2
|
801
|
-
Number of minutes between retries.
|
802
|
-
"""
|
803
|
-
...
|
804
|
-
|
805
|
-
@typing.overload
|
806
|
-
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
807
|
-
...
|
808
|
-
|
809
|
-
@typing.overload
|
810
|
-
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
811
|
-
...
|
812
|
-
|
813
|
-
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
814
|
-
"""
|
815
|
-
Specifies the number of times the task corresponding
|
816
|
-
to a step needs to be retried.
|
817
|
-
|
818
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
819
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
820
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
821
|
-
|
822
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
823
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
824
|
-
ensuring that the flow execution can continue.
|
825
|
-
|
826
|
-
|
827
|
-
Parameters
|
828
|
-
----------
|
829
|
-
times : int, default 3
|
830
|
-
Number of times to retry this task.
|
831
|
-
minutes_between_retries : int, default 2
|
832
|
-
Number of minutes between retries.
|
833
|
-
"""
|
834
|
-
...
|
835
|
-
|
836
642
|
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
837
643
|
"""
|
838
644
|
Decorator that helps cache, version and store models/datasets from huggingface hub.
|
@@ -914,121 +720,103 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.
|
|
914
720
|
...
|
915
721
|
|
916
722
|
@typing.overload
|
917
|
-
def
|
723
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
918
724
|
"""
|
919
|
-
|
920
|
-
|
921
|
-
a Neo Cloud like CoreWeave.
|
725
|
+
Decorator prototype for all step decorators. This function gets specialized
|
726
|
+
and imported for all decorators types by _import_plugin_decorators().
|
922
727
|
"""
|
923
728
|
...
|
924
729
|
|
925
730
|
@typing.overload
|
926
|
-
def
|
731
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
927
732
|
...
|
928
733
|
|
929
|
-
def
|
734
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
930
735
|
"""
|
931
|
-
|
932
|
-
|
933
|
-
a Neo Cloud like CoreWeave.
|
736
|
+
Decorator prototype for all step decorators. This function gets specialized
|
737
|
+
and imported for all decorators types by _import_plugin_decorators().
|
934
738
|
"""
|
935
739
|
...
|
936
740
|
|
937
741
|
@typing.overload
|
938
|
-
def
|
742
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
939
743
|
"""
|
940
|
-
|
941
|
-
|
942
|
-
Use `@resources` to specify the resource requirements
|
943
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
744
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
944
745
|
|
945
|
-
|
946
|
-
```
|
947
|
-
python myflow.py run --with batch
|
948
|
-
```
|
949
|
-
or
|
950
|
-
```
|
951
|
-
python myflow.py run --with kubernetes
|
952
|
-
```
|
953
|
-
which executes the flow on the desired system using the
|
954
|
-
requirements specified in `@resources`.
|
746
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
955
747
|
|
956
748
|
|
957
749
|
Parameters
|
958
750
|
----------
|
959
|
-
|
960
|
-
|
961
|
-
|
962
|
-
|
963
|
-
|
964
|
-
|
965
|
-
|
966
|
-
|
967
|
-
shared_memory : int, optional, default None
|
968
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
969
|
-
This parameter maps to the `--shm-size` option in Docker.
|
751
|
+
type : str, default 'default'
|
752
|
+
Card type.
|
753
|
+
id : str, optional, default None
|
754
|
+
If multiple cards are present, use this id to identify this card.
|
755
|
+
options : Dict[str, Any], default {}
|
756
|
+
Options passed to the card. The contents depend on the card type.
|
757
|
+
timeout : int, default 45
|
758
|
+
Interrupt reporting if it takes more than this many seconds.
|
970
759
|
"""
|
971
760
|
...
|
972
761
|
|
973
762
|
@typing.overload
|
974
|
-
def
|
763
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
975
764
|
...
|
976
765
|
|
977
766
|
@typing.overload
|
978
|
-
def
|
767
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
979
768
|
...
|
980
769
|
|
981
|
-
def
|
770
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
982
771
|
"""
|
983
|
-
|
984
|
-
|
985
|
-
Use `@resources` to specify the resource requirements
|
986
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
772
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
987
773
|
|
988
|
-
|
989
|
-
```
|
990
|
-
python myflow.py run --with batch
|
991
|
-
```
|
992
|
-
or
|
993
|
-
```
|
994
|
-
python myflow.py run --with kubernetes
|
995
|
-
```
|
996
|
-
which executes the flow on the desired system using the
|
997
|
-
requirements specified in `@resources`.
|
774
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
998
775
|
|
999
776
|
|
1000
777
|
Parameters
|
1001
778
|
----------
|
1002
|
-
|
1003
|
-
|
1004
|
-
|
1005
|
-
|
1006
|
-
|
1007
|
-
|
1008
|
-
|
1009
|
-
|
1010
|
-
shared_memory : int, optional, default None
|
1011
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1012
|
-
This parameter maps to the `--shm-size` option in Docker.
|
779
|
+
type : str, default 'default'
|
780
|
+
Card type.
|
781
|
+
id : str, optional, default None
|
782
|
+
If multiple cards are present, use this id to identify this card.
|
783
|
+
options : Dict[str, Any], default {}
|
784
|
+
Options passed to the card. The contents depend on the card type.
|
785
|
+
timeout : int, default 45
|
786
|
+
Interrupt reporting if it takes more than this many seconds.
|
1013
787
|
"""
|
1014
788
|
...
|
1015
789
|
|
1016
790
|
@typing.overload
|
1017
|
-
def
|
791
|
+
def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1018
792
|
"""
|
1019
|
-
|
1020
|
-
|
793
|
+
A simple decorator that demonstrates using CardDecoratorInjector
|
794
|
+
to inject a card and render simple markdown content.
|
1021
795
|
"""
|
1022
796
|
...
|
1023
797
|
|
1024
798
|
@typing.overload
|
1025
|
-
def
|
799
|
+
def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1026
800
|
...
|
1027
801
|
|
1028
|
-
def
|
802
|
+
def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1029
803
|
"""
|
1030
|
-
|
1031
|
-
|
804
|
+
A simple decorator that demonstrates using CardDecoratorInjector
|
805
|
+
to inject a card and render simple markdown content.
|
806
|
+
"""
|
807
|
+
...
|
808
|
+
|
809
|
+
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
810
|
+
"""
|
811
|
+
Specifies that this step should execute on DGX cloud.
|
812
|
+
|
813
|
+
|
814
|
+
Parameters
|
815
|
+
----------
|
816
|
+
gpu : int
|
817
|
+
Number of GPUs to use.
|
818
|
+
gpu_type : str
|
819
|
+
Type of Nvidia GPU to use.
|
1032
820
|
"""
|
1033
821
|
...
|
1034
822
|
|
@@ -1132,100 +920,583 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
1132
920
|
self.next(self.end)
|
1133
921
|
```
|
1134
922
|
|
1135
|
-
- Loading models
|
1136
|
-
```python
|
1137
|
-
@step
|
1138
|
-
def train(self):
|
1139
|
-
# current.model.load returns the path to the model loaded
|
1140
|
-
checkpoint_path = current.model.load(
|
1141
|
-
self.checkpoint_key,
|
1142
|
-
)
|
1143
|
-
model_path = current.model.load(
|
1144
|
-
self.model,
|
1145
|
-
)
|
1146
|
-
self.next(self.test)
|
923
|
+
- Loading models
|
924
|
+
```python
|
925
|
+
@step
|
926
|
+
def train(self):
|
927
|
+
# current.model.load returns the path to the model loaded
|
928
|
+
checkpoint_path = current.model.load(
|
929
|
+
self.checkpoint_key,
|
930
|
+
)
|
931
|
+
model_path = current.model.load(
|
932
|
+
self.model,
|
933
|
+
)
|
934
|
+
self.next(self.test)
|
935
|
+
```
|
936
|
+
|
937
|
+
|
938
|
+
Parameters
|
939
|
+
----------
|
940
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
941
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
942
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
943
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
944
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
945
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
946
|
+
|
947
|
+
temp_dir_root : str, default: None
|
948
|
+
The root directory under which `current.model.loaded` will store loaded models
|
949
|
+
"""
|
950
|
+
...
|
951
|
+
|
952
|
+
def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
953
|
+
"""
|
954
|
+
Specifies that this step should execute on DGX cloud.
|
955
|
+
|
956
|
+
|
957
|
+
Parameters
|
958
|
+
----------
|
959
|
+
gpu : int
|
960
|
+
Number of GPUs to use.
|
961
|
+
gpu_type : str
|
962
|
+
Type of Nvidia GPU to use.
|
963
|
+
queue_timeout : int
|
964
|
+
Time to keep the job in NVCF's queue.
|
965
|
+
"""
|
966
|
+
...
|
967
|
+
|
968
|
+
def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
969
|
+
"""
|
970
|
+
This decorator is used to run Ollama APIs as Metaflow task sidecars.
|
971
|
+
|
972
|
+
User code call
|
973
|
+
--------------
|
974
|
+
@ollama(
|
975
|
+
models=[...],
|
976
|
+
...
|
977
|
+
)
|
978
|
+
|
979
|
+
Valid backend options
|
980
|
+
---------------------
|
981
|
+
- 'local': Run as a separate process on the local task machine.
|
982
|
+
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
983
|
+
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
984
|
+
|
985
|
+
Valid model options
|
986
|
+
-------------------
|
987
|
+
Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
|
988
|
+
|
989
|
+
|
990
|
+
Parameters
|
991
|
+
----------
|
992
|
+
models: list[str]
|
993
|
+
List of Ollama containers running models in sidecars.
|
994
|
+
backend: str
|
995
|
+
Determines where and how to run the Ollama process.
|
996
|
+
force_pull: bool
|
997
|
+
Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
|
998
|
+
cache_update_policy: str
|
999
|
+
Cache update policy: "auto", "force", or "never".
|
1000
|
+
force_cache_update: bool
|
1001
|
+
Simple override for "force" cache update policy.
|
1002
|
+
debug: bool
|
1003
|
+
Whether to turn on verbose debugging logs.
|
1004
|
+
circuit_breaker_config: dict
|
1005
|
+
Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
|
1006
|
+
timeout_config: dict
|
1007
|
+
Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
|
1008
|
+
"""
|
1009
|
+
...
|
1010
|
+
|
1011
|
+
@typing.overload
|
1012
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1013
|
+
"""
|
1014
|
+
Specifies the Conda environment for the step.
|
1015
|
+
|
1016
|
+
Information in this decorator will augment any
|
1017
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1018
|
+
you can use `@conda_base` to set packages required by all
|
1019
|
+
steps and use `@conda` to specify step-specific overrides.
|
1020
|
+
|
1021
|
+
|
1022
|
+
Parameters
|
1023
|
+
----------
|
1024
|
+
packages : Dict[str, str], default {}
|
1025
|
+
Packages to use for this step. The key is the name of the package
|
1026
|
+
and the value is the version to use.
|
1027
|
+
libraries : Dict[str, str], default {}
|
1028
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1029
|
+
python : str, optional, default None
|
1030
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1031
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1032
|
+
disabled : bool, default False
|
1033
|
+
If set to True, disables @conda.
|
1034
|
+
"""
|
1035
|
+
...
|
1036
|
+
|
1037
|
+
@typing.overload
|
1038
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1039
|
+
...
|
1040
|
+
|
1041
|
+
@typing.overload
|
1042
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1043
|
+
...
|
1044
|
+
|
1045
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1046
|
+
"""
|
1047
|
+
Specifies the Conda environment for the step.
|
1048
|
+
|
1049
|
+
Information in this decorator will augment any
|
1050
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1051
|
+
you can use `@conda_base` to set packages required by all
|
1052
|
+
steps and use `@conda` to specify step-specific overrides.
|
1053
|
+
|
1054
|
+
|
1055
|
+
Parameters
|
1056
|
+
----------
|
1057
|
+
packages : Dict[str, str], default {}
|
1058
|
+
Packages to use for this step. The key is the name of the package
|
1059
|
+
and the value is the version to use.
|
1060
|
+
libraries : Dict[str, str], default {}
|
1061
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1062
|
+
python : str, optional, default None
|
1063
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1064
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1065
|
+
disabled : bool, default False
|
1066
|
+
If set to True, disables @conda.
|
1067
|
+
"""
|
1068
|
+
...
|
1069
|
+
|
1070
|
+
@typing.overload
|
1071
|
+
def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1072
|
+
"""
|
1073
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1074
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1075
|
+
"""
|
1076
|
+
...
|
1077
|
+
|
1078
|
+
@typing.overload
|
1079
|
+
def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1080
|
+
...
|
1081
|
+
|
1082
|
+
def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1083
|
+
"""
|
1084
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1085
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1086
|
+
"""
|
1087
|
+
...
|
1088
|
+
|
1089
|
+
@typing.overload
|
1090
|
+
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1091
|
+
"""
|
1092
|
+
Specifies the resources needed when executing this step.
|
1093
|
+
|
1094
|
+
Use `@resources` to specify the resource requirements
|
1095
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
1096
|
+
|
1097
|
+
You can choose the compute layer on the command line by executing e.g.
|
1098
|
+
```
|
1099
|
+
python myflow.py run --with batch
|
1100
|
+
```
|
1101
|
+
or
|
1102
|
+
```
|
1103
|
+
python myflow.py run --with kubernetes
|
1104
|
+
```
|
1105
|
+
which executes the flow on the desired system using the
|
1106
|
+
requirements specified in `@resources`.
|
1107
|
+
|
1108
|
+
|
1109
|
+
Parameters
|
1110
|
+
----------
|
1111
|
+
cpu : int, default 1
|
1112
|
+
Number of CPUs required for this step.
|
1113
|
+
gpu : int, optional, default None
|
1114
|
+
Number of GPUs required for this step.
|
1115
|
+
disk : int, optional, default None
|
1116
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
1117
|
+
memory : int, default 4096
|
1118
|
+
Memory size (in MB) required for this step.
|
1119
|
+
shared_memory : int, optional, default None
|
1120
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1121
|
+
This parameter maps to the `--shm-size` option in Docker.
|
1122
|
+
"""
|
1123
|
+
...
|
1124
|
+
|
1125
|
+
@typing.overload
|
1126
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1127
|
+
...
|
1128
|
+
|
1129
|
+
@typing.overload
|
1130
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1131
|
+
...
|
1132
|
+
|
1133
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
1134
|
+
"""
|
1135
|
+
Specifies the resources needed when executing this step.
|
1136
|
+
|
1137
|
+
Use `@resources` to specify the resource requirements
|
1138
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
1139
|
+
|
1140
|
+
You can choose the compute layer on the command line by executing e.g.
|
1141
|
+
```
|
1142
|
+
python myflow.py run --with batch
|
1143
|
+
```
|
1144
|
+
or
|
1145
|
+
```
|
1146
|
+
python myflow.py run --with kubernetes
|
1147
|
+
```
|
1148
|
+
which executes the flow on the desired system using the
|
1149
|
+
requirements specified in `@resources`.
|
1150
|
+
|
1151
|
+
|
1152
|
+
Parameters
|
1153
|
+
----------
|
1154
|
+
cpu : int, default 1
|
1155
|
+
Number of CPUs required for this step.
|
1156
|
+
gpu : int, optional, default None
|
1157
|
+
Number of GPUs required for this step.
|
1158
|
+
disk : int, optional, default None
|
1159
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
1160
|
+
memory : int, default 4096
|
1161
|
+
Memory size (in MB) required for this step.
|
1162
|
+
shared_memory : int, optional, default None
|
1163
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1164
|
+
This parameter maps to the `--shm-size` option in Docker.
|
1165
|
+
"""
|
1166
|
+
...
|
1167
|
+
|
1168
|
+
@typing.overload
|
1169
|
+
def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1170
|
+
"""
|
1171
|
+
CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
1172
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
1173
|
+
a Neo Cloud like CoreWeave.
|
1174
|
+
"""
|
1175
|
+
...
|
1176
|
+
|
1177
|
+
@typing.overload
|
1178
|
+
def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1179
|
+
...
|
1180
|
+
|
1181
|
+
def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1182
|
+
"""
|
1183
|
+
CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
1184
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
1185
|
+
a Neo Cloud like CoreWeave.
|
1186
|
+
"""
|
1187
|
+
...
|
1188
|
+
|
1189
|
+
@typing.overload
|
1190
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1191
|
+
"""
|
1192
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1193
|
+
the execution of a step.
|
1194
|
+
|
1195
|
+
|
1196
|
+
Parameters
|
1197
|
+
----------
|
1198
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1199
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1200
|
+
role : str, optional, default: None
|
1201
|
+
Role to use for fetching secrets
|
1202
|
+
"""
|
1203
|
+
...
|
1204
|
+
|
1205
|
+
@typing.overload
|
1206
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1207
|
+
...
|
1208
|
+
|
1209
|
+
@typing.overload
|
1210
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1211
|
+
...
|
1212
|
+
|
1213
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
|
1214
|
+
"""
|
1215
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1216
|
+
the execution of a step.
|
1217
|
+
|
1218
|
+
|
1219
|
+
Parameters
|
1220
|
+
----------
|
1221
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1222
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1223
|
+
role : str, optional, default: None
|
1224
|
+
Role to use for fetching secrets
|
1225
|
+
"""
|
1226
|
+
...
|
1227
|
+
|
1228
|
+
def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1229
|
+
"""
|
1230
|
+
This decorator is used to run vllm APIs as Metaflow task sidecars.
|
1231
|
+
|
1232
|
+
User code call
|
1233
|
+
--------------
|
1234
|
+
@vllm(
|
1235
|
+
model="...",
|
1236
|
+
...
|
1237
|
+
)
|
1238
|
+
|
1239
|
+
Valid backend options
|
1240
|
+
---------------------
|
1241
|
+
- 'local': Run as a separate process on the local task machine.
|
1242
|
+
|
1243
|
+
Valid model options
|
1244
|
+
-------------------
|
1245
|
+
Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
|
1246
|
+
|
1247
|
+
NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
|
1248
|
+
If you need multiple models, you must create multiple @vllm decorators.
|
1249
|
+
|
1250
|
+
|
1251
|
+
Parameters
|
1252
|
+
----------
|
1253
|
+
model: str
|
1254
|
+
HuggingFace model identifier to be served by vLLM.
|
1255
|
+
backend: str
|
1256
|
+
Determines where and how to run the vLLM process.
|
1257
|
+
openai_api_server: bool
|
1258
|
+
Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
|
1259
|
+
Default is False (uses native engine).
|
1260
|
+
Set to True for backward compatibility with existing code.
|
1261
|
+
debug: bool
|
1262
|
+
Whether to turn on verbose debugging logs.
|
1263
|
+
card_refresh_interval: int
|
1264
|
+
Interval in seconds for refreshing the vLLM status card.
|
1265
|
+
Only used when openai_api_server=True.
|
1266
|
+
max_retries: int
|
1267
|
+
Maximum number of retries checking for vLLM server startup.
|
1268
|
+
Only used when openai_api_server=True.
|
1269
|
+
retry_alert_frequency: int
|
1270
|
+
Frequency of alert logs for vLLM server startup retries.
|
1271
|
+
Only used when openai_api_server=True.
|
1272
|
+
engine_args : dict
|
1273
|
+
Additional keyword arguments to pass to the vLLM engine.
|
1274
|
+
For example, `tensor_parallel_size=2`.
|
1275
|
+
"""
|
1276
|
+
...
|
1277
|
+
|
1278
|
+
@typing.overload
|
1279
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1280
|
+
"""
|
1281
|
+
Specifies that the step will success under all circumstances.
|
1282
|
+
|
1283
|
+
The decorator will create an optional artifact, specified by `var`, which
|
1284
|
+
contains the exception raised. You can use it to detect the presence
|
1285
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
1286
|
+
are missing.
|
1287
|
+
|
1288
|
+
|
1289
|
+
Parameters
|
1290
|
+
----------
|
1291
|
+
var : str, optional, default None
|
1292
|
+
Name of the artifact in which to store the caught exception.
|
1293
|
+
If not specified, the exception is not stored.
|
1294
|
+
print_exception : bool, default True
|
1295
|
+
Determines whether or not the exception is printed to
|
1296
|
+
stdout when caught.
|
1297
|
+
"""
|
1298
|
+
...
|
1299
|
+
|
1300
|
+
@typing.overload
|
1301
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1302
|
+
...
|
1303
|
+
|
1304
|
+
@typing.overload
|
1305
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1306
|
+
...
|
1307
|
+
|
1308
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
1309
|
+
"""
|
1310
|
+
Specifies that the step will success under all circumstances.
|
1311
|
+
|
1312
|
+
The decorator will create an optional artifact, specified by `var`, which
|
1313
|
+
contains the exception raised. You can use it to detect the presence
|
1314
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
1315
|
+
are missing.
|
1316
|
+
|
1317
|
+
|
1318
|
+
Parameters
|
1319
|
+
----------
|
1320
|
+
var : str, optional, default None
|
1321
|
+
Name of the artifact in which to store the caught exception.
|
1322
|
+
If not specified, the exception is not stored.
|
1323
|
+
print_exception : bool, default True
|
1324
|
+
Determines whether or not the exception is printed to
|
1325
|
+
stdout when caught.
|
1326
|
+
"""
|
1327
|
+
...
|
1328
|
+
|
1329
|
+
def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1330
|
+
"""
|
1331
|
+
S3 Proxy decorator for routing S3 requests through a local proxy service.
|
1332
|
+
|
1333
|
+
|
1334
|
+
Parameters
|
1335
|
+
----------
|
1336
|
+
integration_name : str, optional
|
1337
|
+
Name of the S3 proxy integration. If not specified, will use the only
|
1338
|
+
available S3 proxy integration in the namespace (fails if multiple exist).
|
1339
|
+
write_mode : str, optional
|
1340
|
+
The desired behavior during write operations to target (origin) S3 bucket.
|
1341
|
+
allowed options are:
|
1342
|
+
"origin-and-cache" -> write to both the target S3 bucket and local object
|
1343
|
+
storage
|
1344
|
+
"origin" -> only write to the target S3 bucket
|
1345
|
+
"cache" -> only write to the object storage service used for caching
|
1346
|
+
debug : bool, optional
|
1347
|
+
Enable debug logging for proxy operations.
|
1348
|
+
"""
|
1349
|
+
...
|
1350
|
+
|
1351
|
+
@typing.overload
|
1352
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1353
|
+
"""
|
1354
|
+
Specifies the event(s) that this flow depends on.
|
1355
|
+
|
1356
|
+
```
|
1357
|
+
@trigger(event='foo')
|
1358
|
+
```
|
1359
|
+
or
|
1360
|
+
```
|
1361
|
+
@trigger(events=['foo', 'bar'])
|
1362
|
+
```
|
1363
|
+
|
1364
|
+
Additionally, you can specify the parameter mappings
|
1365
|
+
to map event payload to Metaflow parameters for the flow.
|
1366
|
+
```
|
1367
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1368
|
+
```
|
1369
|
+
or
|
1370
|
+
```
|
1371
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1372
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1373
|
+
```
|
1374
|
+
|
1375
|
+
'parameters' can also be a list of strings and tuples like so:
|
1376
|
+
```
|
1377
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1378
|
+
```
|
1379
|
+
This is equivalent to:
|
1380
|
+
```
|
1381
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1382
|
+
```
|
1383
|
+
|
1384
|
+
|
1385
|
+
Parameters
|
1386
|
+
----------
|
1387
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1388
|
+
Event dependency for this flow.
|
1389
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1390
|
+
Events dependency for this flow.
|
1391
|
+
options : Dict[str, Any], default {}
|
1392
|
+
Backend-specific configuration for tuning eventing behavior.
|
1393
|
+
"""
|
1394
|
+
...
|
1395
|
+
|
1396
|
+
@typing.overload
|
1397
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1398
|
+
...
|
1399
|
+
|
1400
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1401
|
+
"""
|
1402
|
+
Specifies the event(s) that this flow depends on.
|
1403
|
+
|
1404
|
+
```
|
1405
|
+
@trigger(event='foo')
|
1406
|
+
```
|
1407
|
+
or
|
1408
|
+
```
|
1409
|
+
@trigger(events=['foo', 'bar'])
|
1410
|
+
```
|
1411
|
+
|
1412
|
+
Additionally, you can specify the parameter mappings
|
1413
|
+
to map event payload to Metaflow parameters for the flow.
|
1414
|
+
```
|
1415
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1416
|
+
```
|
1417
|
+
or
|
1418
|
+
```
|
1419
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1420
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1421
|
+
```
|
1422
|
+
|
1423
|
+
'parameters' can also be a list of strings and tuples like so:
|
1424
|
+
```
|
1425
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1426
|
+
```
|
1427
|
+
This is equivalent to:
|
1428
|
+
```
|
1429
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1147
1430
|
```
|
1148
1431
|
|
1149
1432
|
|
1150
1433
|
Parameters
|
1151
1434
|
----------
|
1152
|
-
|
1153
|
-
|
1154
|
-
|
1155
|
-
|
1156
|
-
|
1157
|
-
|
1158
|
-
|
1159
|
-
temp_dir_root : str, default: None
|
1160
|
-
The root directory under which `current.model.loaded` will store loaded models
|
1435
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1436
|
+
Event dependency for this flow.
|
1437
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1438
|
+
Events dependency for this flow.
|
1439
|
+
options : Dict[str, Any], default {}
|
1440
|
+
Backend-specific configuration for tuning eventing behavior.
|
1161
1441
|
"""
|
1162
1442
|
...
|
1163
1443
|
|
1164
|
-
def
|
1444
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1165
1445
|
"""
|
1166
|
-
|
1167
|
-
|
1168
|
-
|
1169
|
-
|
1170
|
-
|
1171
|
-
model="...",
|
1172
|
-
...
|
1173
|
-
)
|
1174
|
-
|
1175
|
-
Valid backend options
|
1176
|
-
---------------------
|
1177
|
-
- 'local': Run as a separate process on the local task machine.
|
1178
|
-
|
1179
|
-
Valid model options
|
1180
|
-
-------------------
|
1181
|
-
Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
|
1182
|
-
|
1183
|
-
NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
|
1184
|
-
If you need multiple models, you must create multiple @vllm decorators.
|
1446
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1447
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1448
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1449
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1450
|
+
starts only after all sensors finish.
|
1185
1451
|
|
1186
1452
|
|
1187
1453
|
Parameters
|
1188
1454
|
----------
|
1189
|
-
|
1190
|
-
|
1191
|
-
|
1192
|
-
|
1193
|
-
|
1194
|
-
|
1195
|
-
|
1196
|
-
|
1197
|
-
|
1198
|
-
|
1199
|
-
|
1200
|
-
|
1201
|
-
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
|
1206
|
-
|
1207
|
-
|
1208
|
-
|
1209
|
-
|
1210
|
-
|
1455
|
+
timeout : int
|
1456
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1457
|
+
poke_interval : int
|
1458
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1459
|
+
mode : str
|
1460
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1461
|
+
exponential_backoff : bool
|
1462
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1463
|
+
pool : str
|
1464
|
+
the slot pool this task should run in,
|
1465
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1466
|
+
soft_fail : bool
|
1467
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1468
|
+
name : str
|
1469
|
+
Name of the sensor on Airflow
|
1470
|
+
description : str
|
1471
|
+
Description of sensor in the Airflow UI
|
1472
|
+
bucket_key : Union[str, List[str]]
|
1473
|
+
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1474
|
+
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1475
|
+
bucket_name : str
|
1476
|
+
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1477
|
+
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1478
|
+
wildcard_match : bool
|
1479
|
+
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1480
|
+
aws_conn_id : str
|
1481
|
+
a reference to the s3 connection on Airflow. (Default: None)
|
1482
|
+
verify : bool
|
1483
|
+
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1211
1484
|
"""
|
1212
1485
|
...
|
1213
1486
|
|
1214
1487
|
@typing.overload
|
1215
|
-
def
|
1488
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1216
1489
|
"""
|
1217
|
-
Specifies the Conda environment for the
|
1490
|
+
Specifies the Conda environment for all steps of the flow.
|
1218
1491
|
|
1219
|
-
|
1220
|
-
|
1221
|
-
you can use `@conda_base` to set packages required by all
|
1222
|
-
steps and use `@conda` to specify step-specific overrides.
|
1492
|
+
Use `@conda_base` to set common libraries required by all
|
1493
|
+
steps and use `@conda` to specify step-specific additions.
|
1223
1494
|
|
1224
1495
|
|
1225
1496
|
Parameters
|
1226
1497
|
----------
|
1227
1498
|
packages : Dict[str, str], default {}
|
1228
|
-
Packages to use for this
|
1499
|
+
Packages to use for this flow. The key is the name of the package
|
1229
1500
|
and the value is the version to use.
|
1230
1501
|
libraries : Dict[str, str], default {}
|
1231
1502
|
Supported for backward compatibility. When used with packages, packages will take precedence.
|
@@ -1233,32 +1504,26 @@ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, s
|
|
1233
1504
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1234
1505
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1235
1506
|
disabled : bool, default False
|
1236
|
-
If set to True, disables
|
1507
|
+
If set to True, disables Conda.
|
1237
1508
|
"""
|
1238
1509
|
...
|
1239
1510
|
|
1240
1511
|
@typing.overload
|
1241
|
-
def
|
1242
|
-
...
|
1243
|
-
|
1244
|
-
@typing.overload
|
1245
|
-
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1512
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1246
1513
|
...
|
1247
1514
|
|
1248
|
-
def
|
1515
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1249
1516
|
"""
|
1250
|
-
Specifies the Conda environment for the
|
1517
|
+
Specifies the Conda environment for all steps of the flow.
|
1251
1518
|
|
1252
|
-
|
1253
|
-
|
1254
|
-
you can use `@conda_base` to set packages required by all
|
1255
|
-
steps and use `@conda` to specify step-specific overrides.
|
1519
|
+
Use `@conda_base` to set common libraries required by all
|
1520
|
+
steps and use `@conda` to specify step-specific additions.
|
1256
1521
|
|
1257
1522
|
|
1258
1523
|
Parameters
|
1259
1524
|
----------
|
1260
1525
|
packages : Dict[str, str], default {}
|
1261
|
-
Packages to use for this
|
1526
|
+
Packages to use for this flow. The key is the name of the package
|
1262
1527
|
and the value is the version to use.
|
1263
1528
|
libraries : Dict[str, str], default {}
|
1264
1529
|
Supported for backward compatibility. When used with packages, packages will take precedence.
|
@@ -1266,90 +1531,39 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
1266
1531
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1267
1532
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1268
1533
|
disabled : bool, default False
|
1269
|
-
If set to True, disables
|
1270
|
-
"""
|
1271
|
-
...
|
1272
|
-
|
1273
|
-
@typing.overload
|
1274
|
-
def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1275
|
-
"""
|
1276
|
-
Decorator prototype for all step decorators. This function gets specialized
|
1277
|
-
and imported for all decorators types by _import_plugin_decorators().
|
1278
|
-
"""
|
1279
|
-
...
|
1280
|
-
|
1281
|
-
@typing.overload
|
1282
|
-
def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1283
|
-
...
|
1284
|
-
|
1285
|
-
def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1286
|
-
"""
|
1287
|
-
Decorator prototype for all step decorators. This function gets specialized
|
1288
|
-
and imported for all decorators types by _import_plugin_decorators().
|
1534
|
+
If set to True, disables Conda.
|
1289
1535
|
"""
|
1290
1536
|
...
|
1291
1537
|
|
1292
1538
|
@typing.overload
|
1293
|
-
def
|
1539
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1294
1540
|
"""
|
1295
|
-
Specifies
|
1296
|
-
|
1297
|
-
This decorator is useful if this step may hang indefinitely.
|
1298
|
-
|
1299
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1300
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1301
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1302
|
-
|
1303
|
-
Note that all the values specified in parameters are added together so if you specify
|
1304
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1541
|
+
Specifies the times when the flow should be run when running on a
|
1542
|
+
production scheduler.
|
1305
1543
|
|
1306
1544
|
|
1307
1545
|
Parameters
|
1308
1546
|
----------
|
1309
|
-
|
1310
|
-
|
1311
|
-
|
1312
|
-
|
1313
|
-
|
1314
|
-
|
1547
|
+
hourly : bool, default False
|
1548
|
+
Run the workflow hourly.
|
1549
|
+
daily : bool, default True
|
1550
|
+
Run the workflow daily.
|
1551
|
+
weekly : bool, default False
|
1552
|
+
Run the workflow weekly.
|
1553
|
+
cron : str, optional, default None
|
1554
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1555
|
+
specified by this expression.
|
1556
|
+
timezone : str, optional, default None
|
1557
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1558
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1315
1559
|
"""
|
1316
1560
|
...
|
1317
1561
|
|
1318
1562
|
@typing.overload
|
1319
|
-
def
|
1320
|
-
...
|
1321
|
-
|
1322
|
-
@typing.overload
|
1323
|
-
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1324
|
-
...
|
1325
|
-
|
1326
|
-
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
1327
|
-
"""
|
1328
|
-
Specifies a timeout for your step.
|
1329
|
-
|
1330
|
-
This decorator is useful if this step may hang indefinitely.
|
1331
|
-
|
1332
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1333
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1334
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1335
|
-
|
1336
|
-
Note that all the values specified in parameters are added together so if you specify
|
1337
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1338
|
-
|
1339
|
-
|
1340
|
-
Parameters
|
1341
|
-
----------
|
1342
|
-
seconds : int, default 0
|
1343
|
-
Number of seconds to wait prior to timing out.
|
1344
|
-
minutes : int, default 0
|
1345
|
-
Number of minutes to wait prior to timing out.
|
1346
|
-
hours : int, default 0
|
1347
|
-
Number of hours to wait prior to timing out.
|
1348
|
-
"""
|
1563
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1349
1564
|
...
|
1350
1565
|
|
1351
|
-
|
1352
|
-
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1566
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1353
1567
|
"""
|
1354
1568
|
Specifies the times when the flow should be run when running on a
|
1355
1569
|
production scheduler.
|
@@ -1373,29 +1587,43 @@ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False,
|
|
1373
1587
|
...
|
1374
1588
|
|
1375
1589
|
@typing.overload
|
1376
|
-
def
|
1590
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1591
|
+
"""
|
1592
|
+
Specifies the PyPI packages for all steps of the flow.
|
1593
|
+
|
1594
|
+
Use `@pypi_base` to set common packages required by all
|
1595
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1596
|
+
|
1597
|
+
Parameters
|
1598
|
+
----------
|
1599
|
+
packages : Dict[str, str], default: {}
|
1600
|
+
Packages to use for this flow. The key is the name of the package
|
1601
|
+
and the value is the version to use.
|
1602
|
+
python : str, optional, default: None
|
1603
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1604
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1605
|
+
"""
|
1606
|
+
...
|
1607
|
+
|
1608
|
+
@typing.overload
|
1609
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1377
1610
|
...
|
1378
1611
|
|
1379
|
-
def
|
1612
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1380
1613
|
"""
|
1381
|
-
Specifies the
|
1382
|
-
production scheduler.
|
1614
|
+
Specifies the PyPI packages for all steps of the flow.
|
1383
1615
|
|
1616
|
+
Use `@pypi_base` to set common packages required by all
|
1617
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1384
1618
|
|
1385
1619
|
Parameters
|
1386
1620
|
----------
|
1387
|
-
|
1388
|
-
|
1389
|
-
|
1390
|
-
|
1391
|
-
|
1392
|
-
|
1393
|
-
cron : str, optional, default None
|
1394
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1395
|
-
specified by this expression.
|
1396
|
-
timezone : str, optional, default None
|
1397
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1398
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1621
|
+
packages : Dict[str, str], default: {}
|
1622
|
+
Packages to use for this flow. The key is the name of the package
|
1623
|
+
and the value is the version to use.
|
1624
|
+
python : str, optional, default: None
|
1625
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1626
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1399
1627
|
"""
|
1400
1628
|
...
|
1401
1629
|
|
@@ -1692,233 +1920,5 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
|
|
1692
1920
|
"""
|
1693
1921
|
...
|
1694
1922
|
|
1695
|
-
@typing.overload
|
1696
|
-
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1697
|
-
"""
|
1698
|
-
Specifies the event(s) that this flow depends on.
|
1699
|
-
|
1700
|
-
```
|
1701
|
-
@trigger(event='foo')
|
1702
|
-
```
|
1703
|
-
or
|
1704
|
-
```
|
1705
|
-
@trigger(events=['foo', 'bar'])
|
1706
|
-
```
|
1707
|
-
|
1708
|
-
Additionally, you can specify the parameter mappings
|
1709
|
-
to map event payload to Metaflow parameters for the flow.
|
1710
|
-
```
|
1711
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1712
|
-
```
|
1713
|
-
or
|
1714
|
-
```
|
1715
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1716
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1717
|
-
```
|
1718
|
-
|
1719
|
-
'parameters' can also be a list of strings and tuples like so:
|
1720
|
-
```
|
1721
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1722
|
-
```
|
1723
|
-
This is equivalent to:
|
1724
|
-
```
|
1725
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1726
|
-
```
|
1727
|
-
|
1728
|
-
|
1729
|
-
Parameters
|
1730
|
-
----------
|
1731
|
-
event : Union[str, Dict[str, Any]], optional, default None
|
1732
|
-
Event dependency for this flow.
|
1733
|
-
events : List[Union[str, Dict[str, Any]]], default []
|
1734
|
-
Events dependency for this flow.
|
1735
|
-
options : Dict[str, Any], default {}
|
1736
|
-
Backend-specific configuration for tuning eventing behavior.
|
1737
|
-
"""
|
1738
|
-
...
|
1739
|
-
|
1740
|
-
@typing.overload
|
1741
|
-
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1742
|
-
...
|
1743
|
-
|
1744
|
-
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1745
|
-
"""
|
1746
|
-
Specifies the event(s) that this flow depends on.
|
1747
|
-
|
1748
|
-
```
|
1749
|
-
@trigger(event='foo')
|
1750
|
-
```
|
1751
|
-
or
|
1752
|
-
```
|
1753
|
-
@trigger(events=['foo', 'bar'])
|
1754
|
-
```
|
1755
|
-
|
1756
|
-
Additionally, you can specify the parameter mappings
|
1757
|
-
to map event payload to Metaflow parameters for the flow.
|
1758
|
-
```
|
1759
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1760
|
-
```
|
1761
|
-
or
|
1762
|
-
```
|
1763
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1764
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1765
|
-
```
|
1766
|
-
|
1767
|
-
'parameters' can also be a list of strings and tuples like so:
|
1768
|
-
```
|
1769
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1770
|
-
```
|
1771
|
-
This is equivalent to:
|
1772
|
-
```
|
1773
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1774
|
-
```
|
1775
|
-
|
1776
|
-
|
1777
|
-
Parameters
|
1778
|
-
----------
|
1779
|
-
event : Union[str, Dict[str, Any]], optional, default None
|
1780
|
-
Event dependency for this flow.
|
1781
|
-
events : List[Union[str, Dict[str, Any]]], default []
|
1782
|
-
Events dependency for this flow.
|
1783
|
-
options : Dict[str, Any], default {}
|
1784
|
-
Backend-specific configuration for tuning eventing behavior.
|
1785
|
-
"""
|
1786
|
-
...
|
1787
|
-
|
1788
|
-
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1789
|
-
"""
|
1790
|
-
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1791
|
-
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1792
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1793
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1794
|
-
starts only after all sensors finish.
|
1795
|
-
|
1796
|
-
|
1797
|
-
Parameters
|
1798
|
-
----------
|
1799
|
-
timeout : int
|
1800
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1801
|
-
poke_interval : int
|
1802
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1803
|
-
mode : str
|
1804
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1805
|
-
exponential_backoff : bool
|
1806
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1807
|
-
pool : str
|
1808
|
-
the slot pool this task should run in,
|
1809
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1810
|
-
soft_fail : bool
|
1811
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1812
|
-
name : str
|
1813
|
-
Name of the sensor on Airflow
|
1814
|
-
description : str
|
1815
|
-
Description of sensor in the Airflow UI
|
1816
|
-
bucket_key : Union[str, List[str]]
|
1817
|
-
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1818
|
-
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1819
|
-
bucket_name : str
|
1820
|
-
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1821
|
-
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1822
|
-
wildcard_match : bool
|
1823
|
-
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1824
|
-
aws_conn_id : str
|
1825
|
-
a reference to the s3 connection on Airflow. (Default: None)
|
1826
|
-
verify : bool
|
1827
|
-
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1828
|
-
"""
|
1829
|
-
...
|
1830
|
-
|
1831
|
-
@typing.overload
|
1832
|
-
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1833
|
-
"""
|
1834
|
-
Specifies the Conda environment for all steps of the flow.
|
1835
|
-
|
1836
|
-
Use `@conda_base` to set common libraries required by all
|
1837
|
-
steps and use `@conda` to specify step-specific additions.
|
1838
|
-
|
1839
|
-
|
1840
|
-
Parameters
|
1841
|
-
----------
|
1842
|
-
packages : Dict[str, str], default {}
|
1843
|
-
Packages to use for this flow. The key is the name of the package
|
1844
|
-
and the value is the version to use.
|
1845
|
-
libraries : Dict[str, str], default {}
|
1846
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1847
|
-
python : str, optional, default None
|
1848
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1849
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1850
|
-
disabled : bool, default False
|
1851
|
-
If set to True, disables Conda.
|
1852
|
-
"""
|
1853
|
-
...
|
1854
|
-
|
1855
|
-
@typing.overload
|
1856
|
-
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1857
|
-
...
|
1858
|
-
|
1859
|
-
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1860
|
-
"""
|
1861
|
-
Specifies the Conda environment for all steps of the flow.
|
1862
|
-
|
1863
|
-
Use `@conda_base` to set common libraries required by all
|
1864
|
-
steps and use `@conda` to specify step-specific additions.
|
1865
|
-
|
1866
|
-
|
1867
|
-
Parameters
|
1868
|
-
----------
|
1869
|
-
packages : Dict[str, str], default {}
|
1870
|
-
Packages to use for this flow. The key is the name of the package
|
1871
|
-
and the value is the version to use.
|
1872
|
-
libraries : Dict[str, str], default {}
|
1873
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1874
|
-
python : str, optional, default None
|
1875
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1876
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1877
|
-
disabled : bool, default False
|
1878
|
-
If set to True, disables Conda.
|
1879
|
-
"""
|
1880
|
-
...
|
1881
|
-
|
1882
|
-
@typing.overload
|
1883
|
-
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1884
|
-
"""
|
1885
|
-
Specifies the PyPI packages for all steps of the flow.
|
1886
|
-
|
1887
|
-
Use `@pypi_base` to set common packages required by all
|
1888
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1889
|
-
|
1890
|
-
Parameters
|
1891
|
-
----------
|
1892
|
-
packages : Dict[str, str], default: {}
|
1893
|
-
Packages to use for this flow. The key is the name of the package
|
1894
|
-
and the value is the version to use.
|
1895
|
-
python : str, optional, default: None
|
1896
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1897
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1898
|
-
"""
|
1899
|
-
...
|
1900
|
-
|
1901
|
-
@typing.overload
|
1902
|
-
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1903
|
-
...
|
1904
|
-
|
1905
|
-
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1906
|
-
"""
|
1907
|
-
Specifies the PyPI packages for all steps of the flow.
|
1908
|
-
|
1909
|
-
Use `@pypi_base` to set common packages required by all
|
1910
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1911
|
-
|
1912
|
-
Parameters
|
1913
|
-
----------
|
1914
|
-
packages : Dict[str, str], default: {}
|
1915
|
-
Packages to use for this flow. The key is the name of the package
|
1916
|
-
and the value is the version to use.
|
1917
|
-
python : str, optional, default: None
|
1918
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1919
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1920
|
-
"""
|
1921
|
-
...
|
1922
|
-
|
1923
1923
|
pkg_name: str
|
1924
1924
|
|