ob-metaflow-stubs 6.0.10.7__py2.py3-none-any.whl → 6.0.10.8__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ob-metaflow-stubs might be problematic. Click here for more details.
- metaflow-stubs/__init__.pyi +1039 -1039
- metaflow-stubs/cards.pyi +3 -2
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/cli_components/__init__.pyi +2 -2
- metaflow-stubs/cli_components/utils.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +2 -2
- metaflow-stubs/client/core.pyi +6 -6
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/events.pyi +2 -2
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +5 -5
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +5 -5
- metaflow-stubs/meta_files.pyi +2 -2
- metaflow-stubs/metadata_provider/__init__.pyi +2 -2
- metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
- metaflow-stubs/metadata_provider/metadata.pyi +2 -2
- metaflow-stubs/metadata_provider/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +35 -35
- metaflow-stubs/metaflow_git.pyi +2 -2
- metaflow-stubs/mf_extensions/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +5 -5
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/ob_internal.pyi +2 -2
- metaflow-stubs/packaging_sys/__init__.pyi +8 -8
- metaflow-stubs/packaging_sys/backend.pyi +4 -4
- metaflow-stubs/packaging_sys/distribution_support.pyi +5 -5
- metaflow-stubs/packaging_sys/tar_backend.pyi +5 -5
- metaflow-stubs/packaging_sys/utils.pyi +2 -2
- metaflow-stubs/packaging_sys/v1.pyi +3 -3
- metaflow-stubs/parameters.pyi +5 -5
- metaflow-stubs/plugins/__init__.pyi +13 -13
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
- metaflow-stubs/plugins/argo/exit_hooks.pyi +3 -3
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_client.pyi +2 -2
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +83 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +2 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +2 -2
- metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
- metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
- metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
- metaflow-stubs/plugins/optuna/__init__.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
- metaflow-stubs/plugins/secrets/utils.pyi +2 -2
- metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
- metaflow-stubs/plugins/uv/__init__.pyi +2 -2
- metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +34 -34
- metaflow-stubs/runner/deployer_impl.pyi +3 -3
- metaflow-stubs/runner/metaflow_runner.pyi +4 -4
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +2 -2
- metaflow-stubs/system/__init__.pyi +2 -2
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/user_configs/__init__.pyi +2 -2
- metaflow-stubs/user_configs/config_options.pyi +3 -3
- metaflow-stubs/user_configs/config_parameters.pyi +7 -7
- metaflow-stubs/user_decorators/__init__.pyi +2 -2
- metaflow-stubs/user_decorators/common.pyi +2 -2
- metaflow-stubs/user_decorators/mutable_flow.pyi +5 -5
- metaflow-stubs/user_decorators/mutable_step.pyi +5 -5
- metaflow-stubs/user_decorators/user_flow_decorator.pyi +4 -4
- metaflow-stubs/user_decorators/user_step_decorator.pyi +6 -6
- {ob_metaflow_stubs-6.0.10.7.dist-info → ob_metaflow_stubs-6.0.10.8.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.10.8.dist-info/RECORD +262 -0
- ob_metaflow_stubs-6.0.10.7.dist-info/RECORD +0 -262
- {ob_metaflow_stubs-6.0.10.7.dist-info → ob_metaflow_stubs-6.0.10.8.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.10.7.dist-info → ob_metaflow_stubs-6.0.10.8.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
######################################################################################################
|
|
2
2
|
# Auto-generated Metaflow stub file #
|
|
3
|
-
# MF version: 2.18.7.
|
|
4
|
-
# Generated on 2025-09-
|
|
3
|
+
# MF version: 2.18.7.5+obcheckpoint(0.2.6);ob(v1) #
|
|
4
|
+
# Generated on 2025-09-19T21:56:58.875223 #
|
|
5
5
|
######################################################################################################
|
|
6
6
|
|
|
7
7
|
from __future__ import annotations
|
|
8
8
|
|
|
9
9
|
import typing
|
|
10
10
|
if typing.TYPE_CHECKING:
|
|
11
|
-
import datetime
|
|
12
11
|
import typing
|
|
12
|
+
import datetime
|
|
13
13
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
|
14
14
|
StepFlag = typing.NewType("StepFlag", bool)
|
|
15
15
|
|
|
@@ -39,10 +39,10 @@ from .user_decorators.user_step_decorator import UserStepDecorator as UserStepDe
|
|
|
39
39
|
from .user_decorators.user_step_decorator import StepMutator as StepMutator
|
|
40
40
|
from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
|
|
41
41
|
from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
|
|
42
|
-
from . import cards as cards
|
|
43
42
|
from . import metaflow_git as metaflow_git
|
|
44
|
-
from . import events as events
|
|
45
43
|
from . import tuple_util as tuple_util
|
|
44
|
+
from . import cards as cards
|
|
45
|
+
from . import events as events
|
|
46
46
|
from . import runner as runner
|
|
47
47
|
from . import plugins as plugins
|
|
48
48
|
from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
|
|
@@ -168,494 +168,158 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
|
168
168
|
...
|
|
169
169
|
|
|
170
170
|
@typing.overload
|
|
171
|
-
def
|
|
171
|
+
def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
172
172
|
"""
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
|
177
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
178
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
|
179
|
-
|
|
180
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
|
181
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
|
182
|
-
ensuring that the flow execution can continue.
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
Parameters
|
|
186
|
-
----------
|
|
187
|
-
times : int, default 3
|
|
188
|
-
Number of times to retry this task.
|
|
189
|
-
minutes_between_retries : int, default 2
|
|
190
|
-
Number of minutes between retries.
|
|
173
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
174
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
191
175
|
"""
|
|
192
176
|
...
|
|
193
177
|
|
|
194
178
|
@typing.overload
|
|
195
|
-
def
|
|
196
|
-
...
|
|
197
|
-
|
|
198
|
-
@typing.overload
|
|
199
|
-
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
179
|
+
def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
200
180
|
...
|
|
201
181
|
|
|
202
|
-
def
|
|
182
|
+
def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
203
183
|
"""
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
|
208
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
209
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
|
210
|
-
|
|
211
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
|
212
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
|
213
|
-
ensuring that the flow execution can continue.
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
Parameters
|
|
217
|
-
----------
|
|
218
|
-
times : int, default 3
|
|
219
|
-
Number of times to retry this task.
|
|
220
|
-
minutes_between_retries : int, default 2
|
|
221
|
-
Number of minutes between retries.
|
|
184
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
185
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
222
186
|
"""
|
|
223
187
|
...
|
|
224
188
|
|
|
225
|
-
|
|
189
|
+
@typing.overload
|
|
190
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
226
191
|
"""
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
User code call
|
|
230
|
-
--------------
|
|
231
|
-
@vllm(
|
|
232
|
-
model="...",
|
|
233
|
-
...
|
|
234
|
-
)
|
|
192
|
+
Specifies a timeout for your step.
|
|
235
193
|
|
|
236
|
-
|
|
237
|
-
---------------------
|
|
238
|
-
- 'local': Run as a separate process on the local task machine.
|
|
194
|
+
This decorator is useful if this step may hang indefinitely.
|
|
239
195
|
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
196
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
|
197
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
|
198
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
|
243
199
|
|
|
244
|
-
|
|
245
|
-
|
|
200
|
+
Note that all the values specified in parameters are added together so if you specify
|
|
201
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
|
246
202
|
|
|
247
203
|
|
|
248
204
|
Parameters
|
|
249
205
|
----------
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
Default is False (uses native engine).
|
|
257
|
-
Set to True for backward compatibility with existing code.
|
|
258
|
-
debug: bool
|
|
259
|
-
Whether to turn on verbose debugging logs.
|
|
260
|
-
card_refresh_interval: int
|
|
261
|
-
Interval in seconds for refreshing the vLLM status card.
|
|
262
|
-
Only used when openai_api_server=True.
|
|
263
|
-
max_retries: int
|
|
264
|
-
Maximum number of retries checking for vLLM server startup.
|
|
265
|
-
Only used when openai_api_server=True.
|
|
266
|
-
retry_alert_frequency: int
|
|
267
|
-
Frequency of alert logs for vLLM server startup retries.
|
|
268
|
-
Only used when openai_api_server=True.
|
|
269
|
-
engine_args : dict
|
|
270
|
-
Additional keyword arguments to pass to the vLLM engine.
|
|
271
|
-
For example, `tensor_parallel_size=2`.
|
|
206
|
+
seconds : int, default 0
|
|
207
|
+
Number of seconds to wait prior to timing out.
|
|
208
|
+
minutes : int, default 0
|
|
209
|
+
Number of minutes to wait prior to timing out.
|
|
210
|
+
hours : int, default 0
|
|
211
|
+
Number of hours to wait prior to timing out.
|
|
272
212
|
"""
|
|
273
213
|
...
|
|
274
214
|
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
This decorator is used to run Ollama APIs as Metaflow task sidecars.
|
|
278
|
-
|
|
279
|
-
User code call
|
|
280
|
-
--------------
|
|
281
|
-
@ollama(
|
|
282
|
-
models=[...],
|
|
283
|
-
...
|
|
284
|
-
)
|
|
285
|
-
|
|
286
|
-
Valid backend options
|
|
287
|
-
---------------------
|
|
288
|
-
- 'local': Run as a separate process on the local task machine.
|
|
289
|
-
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
|
290
|
-
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
|
291
|
-
|
|
292
|
-
Valid model options
|
|
293
|
-
-------------------
|
|
294
|
-
Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
Parameters
|
|
298
|
-
----------
|
|
299
|
-
models: list[str]
|
|
300
|
-
List of Ollama containers running models in sidecars.
|
|
301
|
-
backend: str
|
|
302
|
-
Determines where and how to run the Ollama process.
|
|
303
|
-
force_pull: bool
|
|
304
|
-
Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
|
|
305
|
-
cache_update_policy: str
|
|
306
|
-
Cache update policy: "auto", "force", or "never".
|
|
307
|
-
force_cache_update: bool
|
|
308
|
-
Simple override for "force" cache update policy.
|
|
309
|
-
debug: bool
|
|
310
|
-
Whether to turn on verbose debugging logs.
|
|
311
|
-
circuit_breaker_config: dict
|
|
312
|
-
Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
|
|
313
|
-
timeout_config: dict
|
|
314
|
-
Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
|
|
315
|
-
"""
|
|
215
|
+
@typing.overload
|
|
216
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
316
217
|
...
|
|
317
218
|
|
|
318
219
|
@typing.overload
|
|
319
|
-
def
|
|
220
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
221
|
+
...
|
|
222
|
+
|
|
223
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
|
320
224
|
"""
|
|
321
|
-
Specifies
|
|
322
|
-
|
|
225
|
+
Specifies a timeout for your step.
|
|
226
|
+
|
|
227
|
+
This decorator is useful if this step may hang indefinitely.
|
|
228
|
+
|
|
229
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
|
230
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
|
231
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
|
232
|
+
|
|
233
|
+
Note that all the values specified in parameters are added together so if you specify
|
|
234
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
|
323
235
|
|
|
324
236
|
|
|
325
237
|
Parameters
|
|
326
238
|
----------
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
239
|
+
seconds : int, default 0
|
|
240
|
+
Number of seconds to wait prior to timing out.
|
|
241
|
+
minutes : int, default 0
|
|
242
|
+
Number of minutes to wait prior to timing out.
|
|
243
|
+
hours : int, default 0
|
|
244
|
+
Number of hours to wait prior to timing out.
|
|
331
245
|
"""
|
|
332
246
|
...
|
|
333
247
|
|
|
334
|
-
|
|
335
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
336
|
-
...
|
|
337
|
-
|
|
338
|
-
@typing.overload
|
|
339
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
340
|
-
...
|
|
341
|
-
|
|
342
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
|
|
248
|
+
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
343
249
|
"""
|
|
344
|
-
Specifies
|
|
345
|
-
the execution of a step.
|
|
250
|
+
Specifies that this step should execute on DGX cloud.
|
|
346
251
|
|
|
347
252
|
|
|
348
253
|
Parameters
|
|
349
254
|
----------
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
255
|
+
gpu : int
|
|
256
|
+
Number of GPUs to use.
|
|
257
|
+
gpu_type : str
|
|
258
|
+
Type of Nvidia GPU to use.
|
|
354
259
|
"""
|
|
355
260
|
...
|
|
356
261
|
|
|
357
262
|
@typing.overload
|
|
358
|
-
def
|
|
263
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
359
264
|
"""
|
|
360
|
-
Specifies the
|
|
265
|
+
Specifies the Conda environment for the step.
|
|
361
266
|
|
|
362
267
|
Information in this decorator will augment any
|
|
363
|
-
attributes set in the `@
|
|
364
|
-
you can use `@
|
|
365
|
-
steps and use `@
|
|
268
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
|
269
|
+
you can use `@conda_base` to set packages required by all
|
|
270
|
+
steps and use `@conda` to specify step-specific overrides.
|
|
366
271
|
|
|
367
272
|
|
|
368
273
|
Parameters
|
|
369
274
|
----------
|
|
370
|
-
packages : Dict[str, str], default
|
|
275
|
+
packages : Dict[str, str], default {}
|
|
371
276
|
Packages to use for this step. The key is the name of the package
|
|
372
277
|
and the value is the version to use.
|
|
373
|
-
|
|
278
|
+
libraries : Dict[str, str], default {}
|
|
279
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
280
|
+
python : str, optional, default None
|
|
374
281
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
375
282
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
283
|
+
disabled : bool, default False
|
|
284
|
+
If set to True, disables @conda.
|
|
376
285
|
"""
|
|
377
286
|
...
|
|
378
287
|
|
|
379
288
|
@typing.overload
|
|
380
|
-
def
|
|
289
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
381
290
|
...
|
|
382
291
|
|
|
383
292
|
@typing.overload
|
|
384
|
-
def
|
|
293
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
385
294
|
...
|
|
386
295
|
|
|
387
|
-
def
|
|
296
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
|
388
297
|
"""
|
|
389
|
-
Specifies the
|
|
298
|
+
Specifies the Conda environment for the step.
|
|
390
299
|
|
|
391
300
|
Information in this decorator will augment any
|
|
392
|
-
attributes set in the `@
|
|
393
|
-
you can use `@
|
|
394
|
-
steps and use `@
|
|
301
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
|
302
|
+
you can use `@conda_base` to set packages required by all
|
|
303
|
+
steps and use `@conda` to specify step-specific overrides.
|
|
395
304
|
|
|
396
305
|
|
|
397
306
|
Parameters
|
|
398
307
|
----------
|
|
399
|
-
packages : Dict[str, str], default
|
|
308
|
+
packages : Dict[str, str], default {}
|
|
400
309
|
Packages to use for this step. The key is the name of the package
|
|
401
310
|
and the value is the version to use.
|
|
402
|
-
|
|
311
|
+
libraries : Dict[str, str], default {}
|
|
312
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
313
|
+
python : str, optional, default None
|
|
403
314
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
404
315
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
316
|
+
disabled : bool, default False
|
|
317
|
+
If set to True, disables @conda.
|
|
405
318
|
"""
|
|
406
319
|
...
|
|
407
320
|
|
|
408
321
|
@typing.overload
|
|
409
|
-
def
|
|
410
|
-
"""
|
|
411
|
-
Internal decorator to support Fast bakery
|
|
412
|
-
"""
|
|
413
|
-
...
|
|
414
|
-
|
|
415
|
-
@typing.overload
|
|
416
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
417
|
-
...
|
|
418
|
-
|
|
419
|
-
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
420
|
-
"""
|
|
421
|
-
Internal decorator to support Fast bakery
|
|
422
|
-
"""
|
|
423
|
-
...
|
|
424
|
-
|
|
425
|
-
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
426
|
-
"""
|
|
427
|
-
Specifies that this step should execute on DGX cloud.
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
Parameters
|
|
431
|
-
----------
|
|
432
|
-
gpu : int
|
|
433
|
-
Number of GPUs to use.
|
|
434
|
-
gpu_type : str
|
|
435
|
-
Type of Nvidia GPU to use.
|
|
436
|
-
"""
|
|
437
|
-
...
|
|
438
|
-
|
|
439
|
-
@typing.overload
|
|
440
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
441
|
-
"""
|
|
442
|
-
Decorator prototype for all step decorators. This function gets specialized
|
|
443
|
-
and imported for all decorators types by _import_plugin_decorators().
|
|
444
|
-
"""
|
|
445
|
-
...
|
|
446
|
-
|
|
447
|
-
@typing.overload
|
|
448
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
449
|
-
...
|
|
450
|
-
|
|
451
|
-
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
452
|
-
"""
|
|
453
|
-
Decorator prototype for all step decorators. This function gets specialized
|
|
454
|
-
and imported for all decorators types by _import_plugin_decorators().
|
|
455
|
-
"""
|
|
456
|
-
...
|
|
457
|
-
|
|
458
|
-
@typing.overload
|
|
459
|
-
def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
460
|
-
"""
|
|
461
|
-
A simple decorator that demonstrates using CardDecoratorInjector
|
|
462
|
-
to inject a card and render simple markdown content.
|
|
463
|
-
"""
|
|
464
|
-
...
|
|
465
|
-
|
|
466
|
-
@typing.overload
|
|
467
|
-
def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
468
|
-
...
|
|
469
|
-
|
|
470
|
-
def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
471
|
-
"""
|
|
472
|
-
A simple decorator that demonstrates using CardDecoratorInjector
|
|
473
|
-
to inject a card and render simple markdown content.
|
|
474
|
-
"""
|
|
475
|
-
...
|
|
476
|
-
|
|
477
|
-
@typing.overload
|
|
478
|
-
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
479
|
-
"""
|
|
480
|
-
Enables checkpointing for a step.
|
|
481
|
-
|
|
482
|
-
> Examples
|
|
483
|
-
|
|
484
|
-
- Saving Checkpoints
|
|
485
|
-
|
|
486
|
-
```python
|
|
487
|
-
@checkpoint
|
|
488
|
-
@step
|
|
489
|
-
def train(self):
|
|
490
|
-
model = create_model(self.parameters, checkpoint_path = None)
|
|
491
|
-
for i in range(self.epochs):
|
|
492
|
-
# some training logic
|
|
493
|
-
loss = model.train(self.dataset)
|
|
494
|
-
if i % 10 == 0:
|
|
495
|
-
model.save(
|
|
496
|
-
current.checkpoint.directory,
|
|
497
|
-
)
|
|
498
|
-
# saves the contents of the `current.checkpoint.directory` as a checkpoint
|
|
499
|
-
# and returns a reference dictionary to the checkpoint saved in the datastore
|
|
500
|
-
self.latest_checkpoint = current.checkpoint.save(
|
|
501
|
-
name="epoch_checkpoint",
|
|
502
|
-
metadata={
|
|
503
|
-
"epoch": i,
|
|
504
|
-
"loss": loss,
|
|
505
|
-
}
|
|
506
|
-
)
|
|
507
|
-
```
|
|
508
|
-
|
|
509
|
-
- Using Loaded Checkpoints
|
|
510
|
-
|
|
511
|
-
```python
|
|
512
|
-
@retry(times=3)
|
|
513
|
-
@checkpoint
|
|
514
|
-
@step
|
|
515
|
-
def train(self):
|
|
516
|
-
# Assume that the task has restarted and the previous attempt of the task
|
|
517
|
-
# saved a checkpoint
|
|
518
|
-
checkpoint_path = None
|
|
519
|
-
if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
|
|
520
|
-
print("Loaded checkpoint from the previous attempt")
|
|
521
|
-
checkpoint_path = current.checkpoint.directory
|
|
522
|
-
|
|
523
|
-
model = create_model(self.parameters, checkpoint_path = checkpoint_path)
|
|
524
|
-
for i in range(self.epochs):
|
|
525
|
-
...
|
|
526
|
-
```
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
Parameters
|
|
530
|
-
----------
|
|
531
|
-
load_policy : str, default: "fresh"
|
|
532
|
-
The policy for loading the checkpoint. The following policies are supported:
|
|
533
|
-
- "eager": Loads the the latest available checkpoint within the namespace.
|
|
534
|
-
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
|
535
|
-
will be loaded at the start of the task.
|
|
536
|
-
- "none": Do not load any checkpoint
|
|
537
|
-
- "fresh": Loads the lastest checkpoint created within the running Task.
|
|
538
|
-
This mode helps loading checkpoints across various retry attempts of the same task.
|
|
539
|
-
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
|
540
|
-
created within the task will be loaded when the task is retries execution on failure.
|
|
541
|
-
|
|
542
|
-
temp_dir_root : str, default: None
|
|
543
|
-
The root directory under which `current.checkpoint.directory` will be created.
|
|
544
|
-
"""
|
|
545
|
-
...
|
|
546
|
-
|
|
547
|
-
@typing.overload
|
|
548
|
-
def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
549
|
-
...
|
|
550
|
-
|
|
551
|
-
@typing.overload
|
|
552
|
-
def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
553
|
-
...
|
|
554
|
-
|
|
555
|
-
def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
|
|
556
|
-
"""
|
|
557
|
-
Enables checkpointing for a step.
|
|
558
|
-
|
|
559
|
-
> Examples
|
|
560
|
-
|
|
561
|
-
- Saving Checkpoints
|
|
562
|
-
|
|
563
|
-
```python
|
|
564
|
-
@checkpoint
|
|
565
|
-
@step
|
|
566
|
-
def train(self):
|
|
567
|
-
model = create_model(self.parameters, checkpoint_path = None)
|
|
568
|
-
for i in range(self.epochs):
|
|
569
|
-
# some training logic
|
|
570
|
-
loss = model.train(self.dataset)
|
|
571
|
-
if i % 10 == 0:
|
|
572
|
-
model.save(
|
|
573
|
-
current.checkpoint.directory,
|
|
574
|
-
)
|
|
575
|
-
# saves the contents of the `current.checkpoint.directory` as a checkpoint
|
|
576
|
-
# and returns a reference dictionary to the checkpoint saved in the datastore
|
|
577
|
-
self.latest_checkpoint = current.checkpoint.save(
|
|
578
|
-
name="epoch_checkpoint",
|
|
579
|
-
metadata={
|
|
580
|
-
"epoch": i,
|
|
581
|
-
"loss": loss,
|
|
582
|
-
}
|
|
583
|
-
)
|
|
584
|
-
```
|
|
585
|
-
|
|
586
|
-
- Using Loaded Checkpoints
|
|
587
|
-
|
|
588
|
-
```python
|
|
589
|
-
@retry(times=3)
|
|
590
|
-
@checkpoint
|
|
591
|
-
@step
|
|
592
|
-
def train(self):
|
|
593
|
-
# Assume that the task has restarted and the previous attempt of the task
|
|
594
|
-
# saved a checkpoint
|
|
595
|
-
checkpoint_path = None
|
|
596
|
-
if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
|
|
597
|
-
print("Loaded checkpoint from the previous attempt")
|
|
598
|
-
checkpoint_path = current.checkpoint.directory
|
|
599
|
-
|
|
600
|
-
model = create_model(self.parameters, checkpoint_path = checkpoint_path)
|
|
601
|
-
for i in range(self.epochs):
|
|
602
|
-
...
|
|
603
|
-
```
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
Parameters
|
|
607
|
-
----------
|
|
608
|
-
load_policy : str, default: "fresh"
|
|
609
|
-
The policy for loading the checkpoint. The following policies are supported:
|
|
610
|
-
- "eager": Loads the the latest available checkpoint within the namespace.
|
|
611
|
-
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
|
612
|
-
will be loaded at the start of the task.
|
|
613
|
-
- "none": Do not load any checkpoint
|
|
614
|
-
- "fresh": Loads the lastest checkpoint created within the running Task.
|
|
615
|
-
This mode helps loading checkpoints across various retry attempts of the same task.
|
|
616
|
-
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
|
617
|
-
created within the task will be loaded when the task is retries execution on failure.
|
|
618
|
-
|
|
619
|
-
temp_dir_root : str, default: None
|
|
620
|
-
The root directory under which `current.checkpoint.directory` will be created.
|
|
621
|
-
"""
|
|
622
|
-
...
|
|
623
|
-
|
|
624
|
-
@typing.overload
|
|
625
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
626
|
-
"""
|
|
627
|
-
Specifies environment variables to be set prior to the execution of a step.
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
Parameters
|
|
631
|
-
----------
|
|
632
|
-
vars : Dict[str, str], default {}
|
|
633
|
-
Dictionary of environment variables to set.
|
|
634
|
-
"""
|
|
635
|
-
...
|
|
636
|
-
|
|
637
|
-
@typing.overload
|
|
638
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
639
|
-
...
|
|
640
|
-
|
|
641
|
-
@typing.overload
|
|
642
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
643
|
-
...
|
|
644
|
-
|
|
645
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
|
646
|
-
"""
|
|
647
|
-
Specifies environment variables to be set prior to the execution of a step.
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
Parameters
|
|
651
|
-
----------
|
|
652
|
-
vars : Dict[str, str], default {}
|
|
653
|
-
Dictionary of environment variables to set.
|
|
654
|
-
"""
|
|
655
|
-
...
|
|
656
|
-
|
|
657
|
-
@typing.overload
|
|
658
|
-
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
322
|
+
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
659
323
|
"""
|
|
660
324
|
Specifies the resources needed when executing this step.
|
|
661
325
|
|
|
@@ -734,61 +398,518 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
|
734
398
|
...
|
|
735
399
|
|
|
736
400
|
@typing.overload
|
|
737
|
-
def
|
|
401
|
+
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
738
402
|
"""
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
Information in this decorator will augment any
|
|
742
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
|
743
|
-
you can use `@conda_base` to set packages required by all
|
|
744
|
-
steps and use `@conda` to specify step-specific overrides.
|
|
745
|
-
|
|
403
|
+
Enables loading / saving of models within a step.
|
|
746
404
|
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
405
|
+
> Examples
|
|
406
|
+
- Saving Models
|
|
407
|
+
```python
|
|
408
|
+
@model
|
|
409
|
+
@step
|
|
410
|
+
def train(self):
|
|
411
|
+
# current.model.save returns a dictionary reference to the model saved
|
|
412
|
+
self.my_model = current.model.save(
|
|
413
|
+
path_to_my_model,
|
|
414
|
+
label="my_model",
|
|
415
|
+
metadata={
|
|
416
|
+
"epochs": 10,
|
|
417
|
+
"batch-size": 32,
|
|
418
|
+
"learning-rate": 0.001,
|
|
419
|
+
}
|
|
420
|
+
)
|
|
421
|
+
self.next(self.test)
|
|
422
|
+
|
|
423
|
+
@model(load="my_model")
|
|
424
|
+
@step
|
|
425
|
+
def test(self):
|
|
426
|
+
# `current.model.loaded` returns a dictionary of the loaded models
|
|
427
|
+
# where the key is the name of the artifact and the value is the path to the model
|
|
428
|
+
print(os.listdir(current.model.loaded["my_model"]))
|
|
429
|
+
self.next(self.end)
|
|
430
|
+
```
|
|
431
|
+
|
|
432
|
+
- Loading models
|
|
433
|
+
```python
|
|
434
|
+
@step
|
|
435
|
+
def train(self):
|
|
436
|
+
# current.model.load returns the path to the model loaded
|
|
437
|
+
checkpoint_path = current.model.load(
|
|
438
|
+
self.checkpoint_key,
|
|
439
|
+
)
|
|
440
|
+
model_path = current.model.load(
|
|
441
|
+
self.model,
|
|
442
|
+
)
|
|
443
|
+
self.next(self.test)
|
|
444
|
+
```
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
Parameters
|
|
448
|
+
----------
|
|
449
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
|
450
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
|
451
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
|
452
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
|
453
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
|
454
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
|
455
|
+
|
|
456
|
+
temp_dir_root : str, default: None
|
|
457
|
+
The root directory under which `current.model.loaded` will store loaded models
|
|
458
|
+
"""
|
|
459
|
+
...
|
|
460
|
+
|
|
461
|
+
@typing.overload
|
|
462
|
+
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
463
|
+
...
|
|
464
|
+
|
|
465
|
+
@typing.overload
|
|
466
|
+
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
467
|
+
...
|
|
468
|
+
|
|
469
|
+
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
|
470
|
+
"""
|
|
471
|
+
Enables loading / saving of models within a step.
|
|
472
|
+
|
|
473
|
+
> Examples
|
|
474
|
+
- Saving Models
|
|
475
|
+
```python
|
|
476
|
+
@model
|
|
477
|
+
@step
|
|
478
|
+
def train(self):
|
|
479
|
+
# current.model.save returns a dictionary reference to the model saved
|
|
480
|
+
self.my_model = current.model.save(
|
|
481
|
+
path_to_my_model,
|
|
482
|
+
label="my_model",
|
|
483
|
+
metadata={
|
|
484
|
+
"epochs": 10,
|
|
485
|
+
"batch-size": 32,
|
|
486
|
+
"learning-rate": 0.001,
|
|
487
|
+
}
|
|
488
|
+
)
|
|
489
|
+
self.next(self.test)
|
|
490
|
+
|
|
491
|
+
@model(load="my_model")
|
|
492
|
+
@step
|
|
493
|
+
def test(self):
|
|
494
|
+
# `current.model.loaded` returns a dictionary of the loaded models
|
|
495
|
+
# where the key is the name of the artifact and the value is the path to the model
|
|
496
|
+
print(os.listdir(current.model.loaded["my_model"]))
|
|
497
|
+
self.next(self.end)
|
|
498
|
+
```
|
|
499
|
+
|
|
500
|
+
- Loading models
|
|
501
|
+
```python
|
|
502
|
+
@step
|
|
503
|
+
def train(self):
|
|
504
|
+
# current.model.load returns the path to the model loaded
|
|
505
|
+
checkpoint_path = current.model.load(
|
|
506
|
+
self.checkpoint_key,
|
|
507
|
+
)
|
|
508
|
+
model_path = current.model.load(
|
|
509
|
+
self.model,
|
|
510
|
+
)
|
|
511
|
+
self.next(self.test)
|
|
512
|
+
```
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
Parameters
|
|
516
|
+
----------
|
|
517
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
|
518
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
|
519
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
|
520
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
|
521
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
|
522
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
|
523
|
+
|
|
524
|
+
temp_dir_root : str, default: None
|
|
525
|
+
The root directory under which `current.model.loaded` will store loaded models
|
|
526
|
+
"""
|
|
527
|
+
...
|
|
528
|
+
|
|
529
|
+
@typing.overload
|
|
530
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
531
|
+
"""
|
|
532
|
+
Specifies that the step will success under all circumstances.
|
|
533
|
+
|
|
534
|
+
The decorator will create an optional artifact, specified by `var`, which
|
|
535
|
+
contains the exception raised. You can use it to detect the presence
|
|
536
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
|
537
|
+
are missing.
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
Parameters
|
|
541
|
+
----------
|
|
542
|
+
var : str, optional, default None
|
|
543
|
+
Name of the artifact in which to store the caught exception.
|
|
544
|
+
If not specified, the exception is not stored.
|
|
545
|
+
print_exception : bool, default True
|
|
546
|
+
Determines whether or not the exception is printed to
|
|
547
|
+
stdout when caught.
|
|
548
|
+
"""
|
|
549
|
+
...
|
|
550
|
+
|
|
551
|
+
@typing.overload
|
|
552
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
553
|
+
...
|
|
554
|
+
|
|
555
|
+
@typing.overload
|
|
556
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
557
|
+
...
|
|
558
|
+
|
|
559
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
|
560
|
+
"""
|
|
561
|
+
Specifies that the step will success under all circumstances.
|
|
562
|
+
|
|
563
|
+
The decorator will create an optional artifact, specified by `var`, which
|
|
564
|
+
contains the exception raised. You can use it to detect the presence
|
|
565
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
|
566
|
+
are missing.
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
Parameters
|
|
570
|
+
----------
|
|
571
|
+
var : str, optional, default None
|
|
572
|
+
Name of the artifact in which to store the caught exception.
|
|
573
|
+
If not specified, the exception is not stored.
|
|
574
|
+
print_exception : bool, default True
|
|
575
|
+
Determines whether or not the exception is printed to
|
|
576
|
+
stdout when caught.
|
|
577
|
+
"""
|
|
578
|
+
...
|
|
579
|
+
|
|
580
|
+
@typing.overload
|
|
581
|
+
def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
582
|
+
"""
|
|
583
|
+
CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
584
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
|
585
|
+
a Neo Cloud like CoreWeave.
|
|
586
|
+
"""
|
|
587
|
+
...
|
|
588
|
+
|
|
589
|
+
@typing.overload
|
|
590
|
+
def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
591
|
+
...
|
|
592
|
+
|
|
593
|
+
def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
594
|
+
"""
|
|
595
|
+
CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
596
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
|
597
|
+
a Neo Cloud like CoreWeave.
|
|
598
|
+
"""
|
|
599
|
+
...
|
|
600
|
+
|
|
601
|
+
@typing.overload
|
|
602
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
603
|
+
"""
|
|
604
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
605
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
606
|
+
"""
|
|
607
|
+
...
|
|
608
|
+
|
|
609
|
+
@typing.overload
|
|
610
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
611
|
+
...
|
|
612
|
+
|
|
613
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
614
|
+
"""
|
|
615
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
616
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
617
|
+
"""
|
|
618
|
+
...
|
|
619
|
+
|
|
620
|
+
def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
621
|
+
"""
|
|
622
|
+
S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
Parameters
|
|
626
|
+
----------
|
|
627
|
+
integration_name : str, optional
|
|
628
|
+
Name of the S3 proxy integration. If not specified, will use the only
|
|
629
|
+
available S3 proxy integration in the namespace (fails if multiple exist).
|
|
630
|
+
write_mode : str, optional
|
|
631
|
+
The desired behavior during write operations to target (origin) S3 bucket.
|
|
632
|
+
allowed options are:
|
|
633
|
+
"origin-and-cache" -> write to both the target S3 bucket and local object
|
|
634
|
+
storage
|
|
635
|
+
"origin" -> only write to the target S3 bucket
|
|
636
|
+
"cache" -> only write to the object storage service used for caching
|
|
637
|
+
debug : bool, optional
|
|
638
|
+
Enable debug logging for proxy operations.
|
|
639
|
+
"""
|
|
640
|
+
...
|
|
641
|
+
|
|
642
|
+
@typing.overload
|
|
643
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
644
|
+
"""
|
|
645
|
+
Specifies environment variables to be set prior to the execution of a step.
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
Parameters
|
|
649
|
+
----------
|
|
650
|
+
vars : Dict[str, str], default {}
|
|
651
|
+
Dictionary of environment variables to set.
|
|
652
|
+
"""
|
|
653
|
+
...
|
|
654
|
+
|
|
655
|
+
@typing.overload
|
|
656
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
657
|
+
...
|
|
658
|
+
|
|
659
|
+
@typing.overload
|
|
660
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
661
|
+
...
|
|
662
|
+
|
|
663
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
|
664
|
+
"""
|
|
665
|
+
Specifies environment variables to be set prior to the execution of a step.
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
Parameters
|
|
669
|
+
----------
|
|
670
|
+
vars : Dict[str, str], default {}
|
|
671
|
+
Dictionary of environment variables to set.
|
|
672
|
+
"""
|
|
673
|
+
...
|
|
674
|
+
|
|
675
|
+
@typing.overload
|
|
676
|
+
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
677
|
+
"""
|
|
678
|
+
Enables checkpointing for a step.
|
|
679
|
+
|
|
680
|
+
> Examples
|
|
681
|
+
|
|
682
|
+
- Saving Checkpoints
|
|
683
|
+
|
|
684
|
+
```python
|
|
685
|
+
@checkpoint
|
|
686
|
+
@step
|
|
687
|
+
def train(self):
|
|
688
|
+
model = create_model(self.parameters, checkpoint_path = None)
|
|
689
|
+
for i in range(self.epochs):
|
|
690
|
+
# some training logic
|
|
691
|
+
loss = model.train(self.dataset)
|
|
692
|
+
if i % 10 == 0:
|
|
693
|
+
model.save(
|
|
694
|
+
current.checkpoint.directory,
|
|
695
|
+
)
|
|
696
|
+
# saves the contents of the `current.checkpoint.directory` as a checkpoint
|
|
697
|
+
# and returns a reference dictionary to the checkpoint saved in the datastore
|
|
698
|
+
self.latest_checkpoint = current.checkpoint.save(
|
|
699
|
+
name="epoch_checkpoint",
|
|
700
|
+
metadata={
|
|
701
|
+
"epoch": i,
|
|
702
|
+
"loss": loss,
|
|
703
|
+
}
|
|
704
|
+
)
|
|
705
|
+
```
|
|
706
|
+
|
|
707
|
+
- Using Loaded Checkpoints
|
|
708
|
+
|
|
709
|
+
```python
|
|
710
|
+
@retry(times=3)
|
|
711
|
+
@checkpoint
|
|
712
|
+
@step
|
|
713
|
+
def train(self):
|
|
714
|
+
# Assume that the task has restarted and the previous attempt of the task
|
|
715
|
+
# saved a checkpoint
|
|
716
|
+
checkpoint_path = None
|
|
717
|
+
if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
|
|
718
|
+
print("Loaded checkpoint from the previous attempt")
|
|
719
|
+
checkpoint_path = current.checkpoint.directory
|
|
720
|
+
|
|
721
|
+
model = create_model(self.parameters, checkpoint_path = checkpoint_path)
|
|
722
|
+
for i in range(self.epochs):
|
|
723
|
+
...
|
|
724
|
+
```
|
|
725
|
+
|
|
726
|
+
|
|
727
|
+
Parameters
|
|
728
|
+
----------
|
|
729
|
+
load_policy : str, default: "fresh"
|
|
730
|
+
The policy for loading the checkpoint. The following policies are supported:
|
|
731
|
+
- "eager": Loads the the latest available checkpoint within the namespace.
|
|
732
|
+
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
|
733
|
+
will be loaded at the start of the task.
|
|
734
|
+
- "none": Do not load any checkpoint
|
|
735
|
+
- "fresh": Loads the lastest checkpoint created within the running Task.
|
|
736
|
+
This mode helps loading checkpoints across various retry attempts of the same task.
|
|
737
|
+
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
|
738
|
+
created within the task will be loaded when the task is retries execution on failure.
|
|
739
|
+
|
|
740
|
+
temp_dir_root : str, default: None
|
|
741
|
+
The root directory under which `current.checkpoint.directory` will be created.
|
|
759
742
|
"""
|
|
760
743
|
...
|
|
761
744
|
|
|
762
745
|
@typing.overload
|
|
763
|
-
def
|
|
746
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
764
747
|
...
|
|
765
748
|
|
|
766
749
|
@typing.overload
|
|
767
|
-
def
|
|
750
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
768
751
|
...
|
|
769
752
|
|
|
770
|
-
def
|
|
753
|
+
def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
|
|
771
754
|
"""
|
|
772
|
-
|
|
755
|
+
Enables checkpointing for a step.
|
|
773
756
|
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
757
|
+
> Examples
|
|
758
|
+
|
|
759
|
+
- Saving Checkpoints
|
|
760
|
+
|
|
761
|
+
```python
|
|
762
|
+
@checkpoint
|
|
763
|
+
@step
|
|
764
|
+
def train(self):
|
|
765
|
+
model = create_model(self.parameters, checkpoint_path = None)
|
|
766
|
+
for i in range(self.epochs):
|
|
767
|
+
# some training logic
|
|
768
|
+
loss = model.train(self.dataset)
|
|
769
|
+
if i % 10 == 0:
|
|
770
|
+
model.save(
|
|
771
|
+
current.checkpoint.directory,
|
|
772
|
+
)
|
|
773
|
+
# saves the contents of the `current.checkpoint.directory` as a checkpoint
|
|
774
|
+
# and returns a reference dictionary to the checkpoint saved in the datastore
|
|
775
|
+
self.latest_checkpoint = current.checkpoint.save(
|
|
776
|
+
name="epoch_checkpoint",
|
|
777
|
+
metadata={
|
|
778
|
+
"epoch": i,
|
|
779
|
+
"loss": loss,
|
|
780
|
+
}
|
|
781
|
+
)
|
|
782
|
+
```
|
|
783
|
+
|
|
784
|
+
- Using Loaded Checkpoints
|
|
785
|
+
|
|
786
|
+
```python
|
|
787
|
+
@retry(times=3)
|
|
788
|
+
@checkpoint
|
|
789
|
+
@step
|
|
790
|
+
def train(self):
|
|
791
|
+
# Assume that the task has restarted and the previous attempt of the task
|
|
792
|
+
# saved a checkpoint
|
|
793
|
+
checkpoint_path = None
|
|
794
|
+
if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
|
|
795
|
+
print("Loaded checkpoint from the previous attempt")
|
|
796
|
+
checkpoint_path = current.checkpoint.directory
|
|
797
|
+
|
|
798
|
+
model = create_model(self.parameters, checkpoint_path = checkpoint_path)
|
|
799
|
+
for i in range(self.epochs):
|
|
800
|
+
...
|
|
801
|
+
```
|
|
778
802
|
|
|
779
803
|
|
|
780
804
|
Parameters
|
|
781
805
|
----------
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
806
|
+
load_policy : str, default: "fresh"
|
|
807
|
+
The policy for loading the checkpoint. The following policies are supported:
|
|
808
|
+
- "eager": Loads the the latest available checkpoint within the namespace.
|
|
809
|
+
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
|
810
|
+
will be loaded at the start of the task.
|
|
811
|
+
- "none": Do not load any checkpoint
|
|
812
|
+
- "fresh": Loads the lastest checkpoint created within the running Task.
|
|
813
|
+
This mode helps loading checkpoints across various retry attempts of the same task.
|
|
814
|
+
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
|
815
|
+
created within the task will be loaded when the task is retries execution on failure.
|
|
816
|
+
|
|
817
|
+
temp_dir_root : str, default: None
|
|
818
|
+
The root directory under which `current.checkpoint.directory` will be created.
|
|
819
|
+
"""
|
|
820
|
+
...
|
|
821
|
+
|
|
822
|
+
@typing.overload
|
|
823
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
824
|
+
"""
|
|
825
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
|
826
|
+
the execution of a step.
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
Parameters
|
|
830
|
+
----------
|
|
831
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
|
832
|
+
List of secret specs, defining how the secrets are to be retrieved
|
|
833
|
+
role : str, optional, default: None
|
|
834
|
+
Role to use for fetching secrets
|
|
835
|
+
"""
|
|
836
|
+
...
|
|
837
|
+
|
|
838
|
+
@typing.overload
|
|
839
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
840
|
+
...
|
|
841
|
+
|
|
842
|
+
@typing.overload
|
|
843
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
844
|
+
...
|
|
845
|
+
|
|
846
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
|
|
847
|
+
"""
|
|
848
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
|
849
|
+
the execution of a step.
|
|
850
|
+
|
|
851
|
+
|
|
852
|
+
Parameters
|
|
853
|
+
----------
|
|
854
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
|
855
|
+
List of secret specs, defining how the secrets are to be retrieved
|
|
856
|
+
role : str, optional, default: None
|
|
857
|
+
Role to use for fetching secrets
|
|
858
|
+
"""
|
|
859
|
+
...
|
|
860
|
+
|
|
861
|
+
@typing.overload
|
|
862
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
863
|
+
"""
|
|
864
|
+
Specifies the number of times the task corresponding
|
|
865
|
+
to a step needs to be retried.
|
|
866
|
+
|
|
867
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
|
868
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
869
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
|
870
|
+
|
|
871
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
|
872
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
|
873
|
+
ensuring that the flow execution can continue.
|
|
874
|
+
|
|
875
|
+
|
|
876
|
+
Parameters
|
|
877
|
+
----------
|
|
878
|
+
times : int, default 3
|
|
879
|
+
Number of times to retry this task.
|
|
880
|
+
minutes_between_retries : int, default 2
|
|
881
|
+
Number of minutes between retries.
|
|
882
|
+
"""
|
|
883
|
+
...
|
|
884
|
+
|
|
885
|
+
@typing.overload
|
|
886
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
887
|
+
...
|
|
888
|
+
|
|
889
|
+
@typing.overload
|
|
890
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
891
|
+
...
|
|
892
|
+
|
|
893
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
|
894
|
+
"""
|
|
895
|
+
Specifies the number of times the task corresponding
|
|
896
|
+
to a step needs to be retried.
|
|
897
|
+
|
|
898
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
|
899
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
900
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
|
901
|
+
|
|
902
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
|
903
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
|
904
|
+
ensuring that the flow execution can continue.
|
|
905
|
+
|
|
906
|
+
|
|
907
|
+
Parameters
|
|
908
|
+
----------
|
|
909
|
+
times : int, default 3
|
|
910
|
+
Number of times to retry this task.
|
|
911
|
+
minutes_between_retries : int, default 2
|
|
912
|
+
Number of minutes between retries.
|
|
792
913
|
"""
|
|
793
914
|
...
|
|
794
915
|
|
|
@@ -904,61 +1025,127 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, cache_scope:
|
|
|
904
1025
|
- It promotes cache reuse across flows.
|
|
905
1026
|
- The blast radius of a bad checkpoint spans every flow that could be using a particular repo.
|
|
906
1027
|
|
|
907
|
-
load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
|
|
908
|
-
The list of repos (models/datasets) to load.
|
|
1028
|
+
load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
|
|
1029
|
+
The list of repos (models/datasets) to load.
|
|
1030
|
+
|
|
1031
|
+
Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
|
|
1032
|
+
|
|
1033
|
+
- If repo (model/dataset) is not found in the datastore:
|
|
1034
|
+
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
|
1035
|
+
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
|
1036
|
+
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
|
1037
|
+
|
|
1038
|
+
- If repo is found in the datastore:
|
|
1039
|
+
- Loads it directly from datastore to local path (can be temporary directory or specified path)
|
|
1040
|
+
"""
|
|
1041
|
+
...
|
|
1042
|
+
|
|
1043
|
+
def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1044
|
+
"""
|
|
1045
|
+
This decorator is used to run Ollama APIs as Metaflow task sidecars.
|
|
1046
|
+
|
|
1047
|
+
User code call
|
|
1048
|
+
--------------
|
|
1049
|
+
@ollama(
|
|
1050
|
+
models=[...],
|
|
1051
|
+
...
|
|
1052
|
+
)
|
|
1053
|
+
|
|
1054
|
+
Valid backend options
|
|
1055
|
+
---------------------
|
|
1056
|
+
- 'local': Run as a separate process on the local task machine.
|
|
1057
|
+
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
|
1058
|
+
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
|
909
1059
|
|
|
910
|
-
|
|
1060
|
+
Valid model options
|
|
1061
|
+
-------------------
|
|
1062
|
+
Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
|
|
911
1063
|
|
|
912
|
-
- If repo (model/dataset) is not found in the datastore:
|
|
913
|
-
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
|
914
|
-
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
|
915
|
-
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
|
916
1064
|
|
|
917
|
-
|
|
918
|
-
|
|
1065
|
+
Parameters
|
|
1066
|
+
----------
|
|
1067
|
+
models: list[str]
|
|
1068
|
+
List of Ollama containers running models in sidecars.
|
|
1069
|
+
backend: str
|
|
1070
|
+
Determines where and how to run the Ollama process.
|
|
1071
|
+
force_pull: bool
|
|
1072
|
+
Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
|
|
1073
|
+
cache_update_policy: str
|
|
1074
|
+
Cache update policy: "auto", "force", or "never".
|
|
1075
|
+
force_cache_update: bool
|
|
1076
|
+
Simple override for "force" cache update policy.
|
|
1077
|
+
debug: bool
|
|
1078
|
+
Whether to turn on verbose debugging logs.
|
|
1079
|
+
circuit_breaker_config: dict
|
|
1080
|
+
Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
|
|
1081
|
+
timeout_config: dict
|
|
1082
|
+
Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
|
|
919
1083
|
"""
|
|
920
1084
|
...
|
|
921
1085
|
|
|
922
1086
|
@typing.overload
|
|
923
|
-
def
|
|
1087
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
924
1088
|
"""
|
|
925
|
-
|
|
926
|
-
It exists to make it easier for users to know that this decorator should only be used with
|
|
927
|
-
a Neo Cloud like CoreWeave.
|
|
1089
|
+
Internal decorator to support Fast bakery
|
|
928
1090
|
"""
|
|
929
1091
|
...
|
|
930
1092
|
|
|
931
1093
|
@typing.overload
|
|
932
|
-
def
|
|
1094
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
933
1095
|
...
|
|
934
1096
|
|
|
935
|
-
def
|
|
1097
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
936
1098
|
"""
|
|
937
|
-
|
|
938
|
-
It exists to make it easier for users to know that this decorator should only be used with
|
|
939
|
-
a Neo Cloud like CoreWeave.
|
|
1099
|
+
Internal decorator to support Fast bakery
|
|
940
1100
|
"""
|
|
941
1101
|
...
|
|
942
1102
|
|
|
943
|
-
|
|
1103
|
+
@typing.overload
|
|
1104
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
944
1105
|
"""
|
|
945
|
-
|
|
1106
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
|
1107
|
+
|
|
1108
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
|
946
1109
|
|
|
947
1110
|
|
|
948
1111
|
Parameters
|
|
949
1112
|
----------
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
1113
|
+
type : str, default 'default'
|
|
1114
|
+
Card type.
|
|
1115
|
+
id : str, optional, default None
|
|
1116
|
+
If multiple cards are present, use this id to identify this card.
|
|
1117
|
+
options : Dict[str, Any], default {}
|
|
1118
|
+
Options passed to the card. The contents depend on the card type.
|
|
1119
|
+
timeout : int, default 45
|
|
1120
|
+
Interrupt reporting if it takes more than this many seconds.
|
|
1121
|
+
"""
|
|
1122
|
+
...
|
|
1123
|
+
|
|
1124
|
+
@typing.overload
|
|
1125
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1126
|
+
...
|
|
1127
|
+
|
|
1128
|
+
@typing.overload
|
|
1129
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1130
|
+
...
|
|
1131
|
+
|
|
1132
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
|
1133
|
+
"""
|
|
1134
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
|
1135
|
+
|
|
1136
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
|
1137
|
+
|
|
1138
|
+
|
|
1139
|
+
Parameters
|
|
1140
|
+
----------
|
|
1141
|
+
type : str, default 'default'
|
|
1142
|
+
Card type.
|
|
1143
|
+
id : str, optional, default None
|
|
1144
|
+
If multiple cards are present, use this id to identify this card.
|
|
1145
|
+
options : Dict[str, Any], default {}
|
|
1146
|
+
Options passed to the card. The contents depend on the card type.
|
|
1147
|
+
timeout : int, default 45
|
|
1148
|
+
Interrupt reporting if it takes more than this many seconds.
|
|
962
1149
|
"""
|
|
963
1150
|
...
|
|
964
1151
|
|
|
@@ -1052,346 +1239,294 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
|
1052
1239
|
...
|
|
1053
1240
|
|
|
1054
1241
|
@typing.overload
|
|
1055
|
-
def
|
|
1242
|
+
def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1056
1243
|
"""
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
This decorator is useful if this step may hang indefinitely.
|
|
1060
|
-
|
|
1061
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
|
1062
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
|
1063
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
|
1064
|
-
|
|
1065
|
-
Note that all the values specified in parameters are added together so if you specify
|
|
1066
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
Parameters
|
|
1070
|
-
----------
|
|
1071
|
-
seconds : int, default 0
|
|
1072
|
-
Number of seconds to wait prior to timing out.
|
|
1073
|
-
minutes : int, default 0
|
|
1074
|
-
Number of minutes to wait prior to timing out.
|
|
1075
|
-
hours : int, default 0
|
|
1076
|
-
Number of hours to wait prior to timing out.
|
|
1244
|
+
A simple decorator that demonstrates using CardDecoratorInjector
|
|
1245
|
+
to inject a card and render simple markdown content.
|
|
1077
1246
|
"""
|
|
1078
1247
|
...
|
|
1079
1248
|
|
|
1080
1249
|
@typing.overload
|
|
1081
|
-
def
|
|
1082
|
-
...
|
|
1083
|
-
|
|
1084
|
-
@typing.overload
|
|
1085
|
-
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1250
|
+
def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1086
1251
|
...
|
|
1087
1252
|
|
|
1088
|
-
def
|
|
1253
|
+
def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
1089
1254
|
"""
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
This decorator is useful if this step may hang indefinitely.
|
|
1093
|
-
|
|
1094
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
|
1095
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
|
1096
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
|
1097
|
-
|
|
1098
|
-
Note that all the values specified in parameters are added together so if you specify
|
|
1099
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
Parameters
|
|
1103
|
-
----------
|
|
1104
|
-
seconds : int, default 0
|
|
1105
|
-
Number of seconds to wait prior to timing out.
|
|
1106
|
-
minutes : int, default 0
|
|
1107
|
-
Number of minutes to wait prior to timing out.
|
|
1108
|
-
hours : int, default 0
|
|
1109
|
-
Number of hours to wait prior to timing out.
|
|
1255
|
+
A simple decorator that demonstrates using CardDecoratorInjector
|
|
1256
|
+
to inject a card and render simple markdown content.
|
|
1110
1257
|
"""
|
|
1111
1258
|
...
|
|
1112
1259
|
|
|
1113
|
-
|
|
1114
|
-
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1260
|
+
def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1115
1261
|
"""
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
|
1262
|
+
Specifies that this step should execute on DGX cloud.
|
|
1119
1263
|
|
|
1120
1264
|
|
|
1121
1265
|
Parameters
|
|
1122
1266
|
----------
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
timeout : int, default 45
|
|
1130
|
-
Interrupt reporting if it takes more than this many seconds.
|
|
1267
|
+
gpu : int
|
|
1268
|
+
Number of GPUs to use.
|
|
1269
|
+
gpu_type : str
|
|
1270
|
+
Type of Nvidia GPU to use.
|
|
1271
|
+
queue_timeout : int
|
|
1272
|
+
Time to keep the job in NVCF's queue.
|
|
1131
1273
|
"""
|
|
1132
1274
|
...
|
|
1133
1275
|
|
|
1134
1276
|
@typing.overload
|
|
1135
|
-
def
|
|
1277
|
+
def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1278
|
+
"""
|
|
1279
|
+
Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
1280
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
|
1281
|
+
a Neo Cloud like Nebius.
|
|
1282
|
+
"""
|
|
1136
1283
|
...
|
|
1137
1284
|
|
|
1138
1285
|
@typing.overload
|
|
1139
|
-
def
|
|
1140
|
-
...
|
|
1141
|
-
|
|
1142
|
-
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
|
1143
|
-
"""
|
|
1144
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
|
1145
|
-
|
|
1146
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
Parameters
|
|
1150
|
-
----------
|
|
1151
|
-
type : str, default 'default'
|
|
1152
|
-
Card type.
|
|
1153
|
-
id : str, optional, default None
|
|
1154
|
-
If multiple cards are present, use this id to identify this card.
|
|
1155
|
-
options : Dict[str, Any], default {}
|
|
1156
|
-
Options passed to the card. The contents depend on the card type.
|
|
1157
|
-
timeout : int, default 45
|
|
1158
|
-
Interrupt reporting if it takes more than this many seconds.
|
|
1159
|
-
"""
|
|
1286
|
+
def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1160
1287
|
...
|
|
1161
1288
|
|
|
1162
|
-
def
|
|
1289
|
+
def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
1163
1290
|
"""
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
Parameters
|
|
1168
|
-
----------
|
|
1169
|
-
gpu : int
|
|
1170
|
-
Number of GPUs to use.
|
|
1171
|
-
gpu_type : str
|
|
1172
|
-
Type of Nvidia GPU to use.
|
|
1173
|
-
queue_timeout : int
|
|
1174
|
-
Time to keep the job in NVCF's queue.
|
|
1291
|
+
Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
1292
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
|
1293
|
+
a Neo Cloud like Nebius.
|
|
1175
1294
|
"""
|
|
1176
1295
|
...
|
|
1177
1296
|
|
|
1178
1297
|
@typing.overload
|
|
1179
|
-
def
|
|
1180
|
-
"""
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
> Examples
|
|
1184
|
-
- Saving Models
|
|
1185
|
-
```python
|
|
1186
|
-
@model
|
|
1187
|
-
@step
|
|
1188
|
-
def train(self):
|
|
1189
|
-
# current.model.save returns a dictionary reference to the model saved
|
|
1190
|
-
self.my_model = current.model.save(
|
|
1191
|
-
path_to_my_model,
|
|
1192
|
-
label="my_model",
|
|
1193
|
-
metadata={
|
|
1194
|
-
"epochs": 10,
|
|
1195
|
-
"batch-size": 32,
|
|
1196
|
-
"learning-rate": 0.001,
|
|
1197
|
-
}
|
|
1198
|
-
)
|
|
1199
|
-
self.next(self.test)
|
|
1200
|
-
|
|
1201
|
-
@model(load="my_model")
|
|
1202
|
-
@step
|
|
1203
|
-
def test(self):
|
|
1204
|
-
# `current.model.loaded` returns a dictionary of the loaded models
|
|
1205
|
-
# where the key is the name of the artifact and the value is the path to the model
|
|
1206
|
-
print(os.listdir(current.model.loaded["my_model"]))
|
|
1207
|
-
self.next(self.end)
|
|
1208
|
-
```
|
|
1298
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1299
|
+
"""
|
|
1300
|
+
Specifies the PyPI packages for the step.
|
|
1209
1301
|
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
# current.model.load returns the path to the model loaded
|
|
1215
|
-
checkpoint_path = current.model.load(
|
|
1216
|
-
self.checkpoint_key,
|
|
1217
|
-
)
|
|
1218
|
-
model_path = current.model.load(
|
|
1219
|
-
self.model,
|
|
1220
|
-
)
|
|
1221
|
-
self.next(self.test)
|
|
1222
|
-
```
|
|
1302
|
+
Information in this decorator will augment any
|
|
1303
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
|
1304
|
+
you can use `@pypi_base` to set packages required by all
|
|
1305
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
1223
1306
|
|
|
1224
1307
|
|
|
1225
1308
|
Parameters
|
|
1226
1309
|
----------
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
temp_dir_root : str, default: None
|
|
1235
|
-
The root directory under which `current.model.loaded` will store loaded models
|
|
1310
|
+
packages : Dict[str, str], default: {}
|
|
1311
|
+
Packages to use for this step. The key is the name of the package
|
|
1312
|
+
and the value is the version to use.
|
|
1313
|
+
python : str, optional, default: None
|
|
1314
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1315
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1236
1316
|
"""
|
|
1237
1317
|
...
|
|
1238
1318
|
|
|
1239
1319
|
@typing.overload
|
|
1240
|
-
def
|
|
1320
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1241
1321
|
...
|
|
1242
1322
|
|
|
1243
1323
|
@typing.overload
|
|
1244
|
-
def
|
|
1324
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1245
1325
|
...
|
|
1246
1326
|
|
|
1247
|
-
def
|
|
1327
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
|
1248
1328
|
"""
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
> Examples
|
|
1252
|
-
- Saving Models
|
|
1253
|
-
```python
|
|
1254
|
-
@model
|
|
1255
|
-
@step
|
|
1256
|
-
def train(self):
|
|
1257
|
-
# current.model.save returns a dictionary reference to the model saved
|
|
1258
|
-
self.my_model = current.model.save(
|
|
1259
|
-
path_to_my_model,
|
|
1260
|
-
label="my_model",
|
|
1261
|
-
metadata={
|
|
1262
|
-
"epochs": 10,
|
|
1263
|
-
"batch-size": 32,
|
|
1264
|
-
"learning-rate": 0.001,
|
|
1265
|
-
}
|
|
1266
|
-
)
|
|
1267
|
-
self.next(self.test)
|
|
1268
|
-
|
|
1269
|
-
@model(load="my_model")
|
|
1270
|
-
@step
|
|
1271
|
-
def test(self):
|
|
1272
|
-
# `current.model.loaded` returns a dictionary of the loaded models
|
|
1273
|
-
# where the key is the name of the artifact and the value is the path to the model
|
|
1274
|
-
print(os.listdir(current.model.loaded["my_model"]))
|
|
1275
|
-
self.next(self.end)
|
|
1276
|
-
```
|
|
1329
|
+
Specifies the PyPI packages for the step.
|
|
1277
1330
|
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
# current.model.load returns the path to the model loaded
|
|
1283
|
-
checkpoint_path = current.model.load(
|
|
1284
|
-
self.checkpoint_key,
|
|
1285
|
-
)
|
|
1286
|
-
model_path = current.model.load(
|
|
1287
|
-
self.model,
|
|
1288
|
-
)
|
|
1289
|
-
self.next(self.test)
|
|
1290
|
-
```
|
|
1331
|
+
Information in this decorator will augment any
|
|
1332
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
|
1333
|
+
you can use `@pypi_base` to set packages required by all
|
|
1334
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
1291
1335
|
|
|
1292
1336
|
|
|
1293
1337
|
Parameters
|
|
1294
1338
|
----------
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
temp_dir_root : str, default: None
|
|
1303
|
-
The root directory under which `current.model.loaded` will store loaded models
|
|
1339
|
+
packages : Dict[str, str], default: {}
|
|
1340
|
+
Packages to use for this step. The key is the name of the package
|
|
1341
|
+
and the value is the version to use.
|
|
1342
|
+
python : str, optional, default: None
|
|
1343
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1344
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1304
1345
|
"""
|
|
1305
1346
|
...
|
|
1306
1347
|
|
|
1307
|
-
|
|
1308
|
-
def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1348
|
+
def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1309
1349
|
"""
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1350
|
+
This decorator is used to run vllm APIs as Metaflow task sidecars.
|
|
1351
|
+
|
|
1352
|
+
User code call
|
|
1353
|
+
--------------
|
|
1354
|
+
@vllm(
|
|
1355
|
+
model="...",
|
|
1356
|
+
...
|
|
1357
|
+
)
|
|
1358
|
+
|
|
1359
|
+
Valid backend options
|
|
1360
|
+
---------------------
|
|
1361
|
+
- 'local': Run as a separate process on the local task machine.
|
|
1362
|
+
|
|
1363
|
+
Valid model options
|
|
1364
|
+
-------------------
|
|
1365
|
+
Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
|
|
1366
|
+
|
|
1367
|
+
NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
|
|
1368
|
+
If you need multiple models, you must create multiple @vllm decorators.
|
|
1369
|
+
|
|
1370
|
+
|
|
1371
|
+
Parameters
|
|
1372
|
+
----------
|
|
1373
|
+
model: str
|
|
1374
|
+
HuggingFace model identifier to be served by vLLM.
|
|
1375
|
+
backend: str
|
|
1376
|
+
Determines where and how to run the vLLM process.
|
|
1377
|
+
openai_api_server: bool
|
|
1378
|
+
Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
|
|
1379
|
+
Default is False (uses native engine).
|
|
1380
|
+
Set to True for backward compatibility with existing code.
|
|
1381
|
+
debug: bool
|
|
1382
|
+
Whether to turn on verbose debugging logs.
|
|
1383
|
+
card_refresh_interval: int
|
|
1384
|
+
Interval in seconds for refreshing the vLLM status card.
|
|
1385
|
+
Only used when openai_api_server=True.
|
|
1386
|
+
max_retries: int
|
|
1387
|
+
Maximum number of retries checking for vLLM server startup.
|
|
1388
|
+
Only used when openai_api_server=True.
|
|
1389
|
+
retry_alert_frequency: int
|
|
1390
|
+
Frequency of alert logs for vLLM server startup retries.
|
|
1391
|
+
Only used when openai_api_server=True.
|
|
1392
|
+
engine_args : dict
|
|
1393
|
+
Additional keyword arguments to pass to the vLLM engine.
|
|
1394
|
+
For example, `tensor_parallel_size=2`.
|
|
1313
1395
|
"""
|
|
1314
1396
|
...
|
|
1315
1397
|
|
|
1316
|
-
|
|
1317
|
-
def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1318
|
-
...
|
|
1319
|
-
|
|
1320
|
-
def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
1398
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1321
1399
|
"""
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1400
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
|
1401
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
|
1402
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
|
1403
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
|
1404
|
+
starts only after all sensors finish.
|
|
1405
|
+
|
|
1406
|
+
|
|
1407
|
+
Parameters
|
|
1408
|
+
----------
|
|
1409
|
+
timeout : int
|
|
1410
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
|
1411
|
+
poke_interval : int
|
|
1412
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
|
1413
|
+
mode : str
|
|
1414
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
|
1415
|
+
exponential_backoff : bool
|
|
1416
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
|
1417
|
+
pool : str
|
|
1418
|
+
the slot pool this task should run in,
|
|
1419
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
|
1420
|
+
soft_fail : bool
|
|
1421
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
|
1422
|
+
name : str
|
|
1423
|
+
Name of the sensor on Airflow
|
|
1424
|
+
description : str
|
|
1425
|
+
Description of sensor in the Airflow UI
|
|
1426
|
+
bucket_key : Union[str, List[str]]
|
|
1427
|
+
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
|
1428
|
+
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
|
1429
|
+
bucket_name : str
|
|
1430
|
+
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
|
1431
|
+
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
|
1432
|
+
wildcard_match : bool
|
|
1433
|
+
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
|
1434
|
+
aws_conn_id : str
|
|
1435
|
+
a reference to the s3 connection on Airflow. (Default: None)
|
|
1436
|
+
verify : bool
|
|
1437
|
+
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
|
1325
1438
|
"""
|
|
1326
1439
|
...
|
|
1327
1440
|
|
|
1328
1441
|
@typing.overload
|
|
1329
|
-
def
|
|
1442
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1330
1443
|
"""
|
|
1331
|
-
|
|
1332
|
-
|
|
1444
|
+
Specifies the times when the flow should be run when running on a
|
|
1445
|
+
production scheduler.
|
|
1446
|
+
|
|
1447
|
+
|
|
1448
|
+
Parameters
|
|
1449
|
+
----------
|
|
1450
|
+
hourly : bool, default False
|
|
1451
|
+
Run the workflow hourly.
|
|
1452
|
+
daily : bool, default True
|
|
1453
|
+
Run the workflow daily.
|
|
1454
|
+
weekly : bool, default False
|
|
1455
|
+
Run the workflow weekly.
|
|
1456
|
+
cron : str, optional, default None
|
|
1457
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
|
1458
|
+
specified by this expression.
|
|
1459
|
+
timezone : str, optional, default None
|
|
1460
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
|
1461
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
|
1333
1462
|
"""
|
|
1334
1463
|
...
|
|
1335
1464
|
|
|
1336
1465
|
@typing.overload
|
|
1337
|
-
def
|
|
1466
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1338
1467
|
...
|
|
1339
1468
|
|
|
1340
|
-
def
|
|
1469
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
|
1341
1470
|
"""
|
|
1342
|
-
|
|
1343
|
-
|
|
1471
|
+
Specifies the times when the flow should be run when running on a
|
|
1472
|
+
production scheduler.
|
|
1473
|
+
|
|
1474
|
+
|
|
1475
|
+
Parameters
|
|
1476
|
+
----------
|
|
1477
|
+
hourly : bool, default False
|
|
1478
|
+
Run the workflow hourly.
|
|
1479
|
+
daily : bool, default True
|
|
1480
|
+
Run the workflow daily.
|
|
1481
|
+
weekly : bool, default False
|
|
1482
|
+
Run the workflow weekly.
|
|
1483
|
+
cron : str, optional, default None
|
|
1484
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
|
1485
|
+
specified by this expression.
|
|
1486
|
+
timezone : str, optional, default None
|
|
1487
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
|
1488
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
|
1344
1489
|
"""
|
|
1345
1490
|
...
|
|
1346
1491
|
|
|
1347
1492
|
@typing.overload
|
|
1348
|
-
def
|
|
1493
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1349
1494
|
"""
|
|
1350
|
-
Specifies
|
|
1351
|
-
|
|
1352
|
-
The decorator will create an optional artifact, specified by `var`, which
|
|
1353
|
-
contains the exception raised. You can use it to detect the presence
|
|
1354
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
|
1355
|
-
are missing.
|
|
1495
|
+
Specifies the PyPI packages for all steps of the flow.
|
|
1356
1496
|
|
|
1497
|
+
Use `@pypi_base` to set common packages required by all
|
|
1498
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
1357
1499
|
|
|
1358
1500
|
Parameters
|
|
1359
1501
|
----------
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1502
|
+
packages : Dict[str, str], default: {}
|
|
1503
|
+
Packages to use for this flow. The key is the name of the package
|
|
1504
|
+
and the value is the version to use.
|
|
1505
|
+
python : str, optional, default: None
|
|
1506
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1507
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1366
1508
|
"""
|
|
1367
1509
|
...
|
|
1368
1510
|
|
|
1369
1511
|
@typing.overload
|
|
1370
|
-
def
|
|
1371
|
-
...
|
|
1372
|
-
|
|
1373
|
-
@typing.overload
|
|
1374
|
-
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1512
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1375
1513
|
...
|
|
1376
1514
|
|
|
1377
|
-
def
|
|
1515
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
|
1378
1516
|
"""
|
|
1379
|
-
Specifies
|
|
1380
|
-
|
|
1381
|
-
The decorator will create an optional artifact, specified by `var`, which
|
|
1382
|
-
contains the exception raised. You can use it to detect the presence
|
|
1383
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
|
1384
|
-
are missing.
|
|
1517
|
+
Specifies the PyPI packages for all steps of the flow.
|
|
1385
1518
|
|
|
1519
|
+
Use `@pypi_base` to set common packages required by all
|
|
1520
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
1386
1521
|
|
|
1387
1522
|
Parameters
|
|
1388
1523
|
----------
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1524
|
+
packages : Dict[str, str], default: {}
|
|
1525
|
+
Packages to use for this flow. The key is the name of the package
|
|
1526
|
+
and the value is the version to use.
|
|
1527
|
+
python : str, optional, default: None
|
|
1528
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1529
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1395
1530
|
"""
|
|
1396
1531
|
...
|
|
1397
1532
|
|
|
@@ -1484,243 +1619,58 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
|
1484
1619
|
- `test.my_experiment`
|
|
1485
1620
|
- `prod.staging`
|
|
1486
1621
|
|
|
1487
|
-
|
|
1488
|
-
Parameters
|
|
1489
|
-
----------
|
|
1490
|
-
flow : Union[str, Dict[str, str]], optional, default None
|
|
1491
|
-
Upstream flow dependency for this flow.
|
|
1492
|
-
flows : List[Union[str, Dict[str, str]]], default []
|
|
1493
|
-
Upstream flow dependencies for this flow.
|
|
1494
|
-
options : Dict[str, Any], default {}
|
|
1495
|
-
Backend-specific configuration for tuning eventing behavior.
|
|
1496
|
-
"""
|
|
1497
|
-
...
|
|
1498
|
-
|
|
1499
|
-
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1500
|
-
"""
|
|
1501
|
-
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
|
1502
|
-
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
|
1503
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
|
1504
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
|
1505
|
-
starts only after all sensors finish.
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
Parameters
|
|
1509
|
-
----------
|
|
1510
|
-
timeout : int
|
|
1511
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
|
1512
|
-
poke_interval : int
|
|
1513
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
|
1514
|
-
mode : str
|
|
1515
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
|
1516
|
-
exponential_backoff : bool
|
|
1517
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
|
1518
|
-
pool : str
|
|
1519
|
-
the slot pool this task should run in,
|
|
1520
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
|
1521
|
-
soft_fail : bool
|
|
1522
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
|
1523
|
-
name : str
|
|
1524
|
-
Name of the sensor on Airflow
|
|
1525
|
-
description : str
|
|
1526
|
-
Description of sensor in the Airflow UI
|
|
1527
|
-
bucket_key : Union[str, List[str]]
|
|
1528
|
-
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
|
1529
|
-
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
|
1530
|
-
bucket_name : str
|
|
1531
|
-
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
|
1532
|
-
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
|
1533
|
-
wildcard_match : bool
|
|
1534
|
-
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
|
1535
|
-
aws_conn_id : str
|
|
1536
|
-
a reference to the s3 connection on Airflow. (Default: None)
|
|
1537
|
-
verify : bool
|
|
1538
|
-
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
|
1539
|
-
"""
|
|
1540
|
-
...
|
|
1541
|
-
|
|
1542
|
-
@typing.overload
|
|
1543
|
-
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1544
|
-
"""
|
|
1545
|
-
Specifies the event(s) that this flow depends on.
|
|
1546
|
-
|
|
1547
|
-
```
|
|
1548
|
-
@trigger(event='foo')
|
|
1549
|
-
```
|
|
1550
|
-
or
|
|
1551
|
-
```
|
|
1552
|
-
@trigger(events=['foo', 'bar'])
|
|
1553
|
-
```
|
|
1554
|
-
|
|
1555
|
-
Additionally, you can specify the parameter mappings
|
|
1556
|
-
to map event payload to Metaflow parameters for the flow.
|
|
1557
|
-
```
|
|
1558
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
|
1559
|
-
```
|
|
1560
|
-
or
|
|
1561
|
-
```
|
|
1562
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
|
1563
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
|
1564
|
-
```
|
|
1565
|
-
|
|
1566
|
-
'parameters' can also be a list of strings and tuples like so:
|
|
1567
|
-
```
|
|
1568
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
1569
|
-
```
|
|
1570
|
-
This is equivalent to:
|
|
1571
|
-
```
|
|
1572
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
|
1573
|
-
```
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
Parameters
|
|
1577
|
-
----------
|
|
1578
|
-
event : Union[str, Dict[str, Any]], optional, default None
|
|
1579
|
-
Event dependency for this flow.
|
|
1580
|
-
events : List[Union[str, Dict[str, Any]]], default []
|
|
1581
|
-
Events dependency for this flow.
|
|
1582
|
-
options : Dict[str, Any], default {}
|
|
1583
|
-
Backend-specific configuration for tuning eventing behavior.
|
|
1584
|
-
"""
|
|
1585
|
-
...
|
|
1586
|
-
|
|
1587
|
-
@typing.overload
|
|
1588
|
-
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1589
|
-
...
|
|
1590
|
-
|
|
1591
|
-
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
|
1592
|
-
"""
|
|
1593
|
-
Specifies the event(s) that this flow depends on.
|
|
1594
|
-
|
|
1595
|
-
```
|
|
1596
|
-
@trigger(event='foo')
|
|
1597
|
-
```
|
|
1598
|
-
or
|
|
1599
|
-
```
|
|
1600
|
-
@trigger(events=['foo', 'bar'])
|
|
1601
|
-
```
|
|
1602
|
-
|
|
1603
|
-
Additionally, you can specify the parameter mappings
|
|
1604
|
-
to map event payload to Metaflow parameters for the flow.
|
|
1605
|
-
```
|
|
1606
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
|
1607
|
-
```
|
|
1608
|
-
or
|
|
1609
|
-
```
|
|
1610
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
|
1611
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
|
1612
|
-
```
|
|
1613
|
-
|
|
1614
|
-
'parameters' can also be a list of strings and tuples like so:
|
|
1615
|
-
```
|
|
1616
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
1617
|
-
```
|
|
1618
|
-
This is equivalent to:
|
|
1619
|
-
```
|
|
1620
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
|
1621
|
-
```
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
Parameters
|
|
1625
|
-
----------
|
|
1626
|
-
event : Union[str, Dict[str, Any]], optional, default None
|
|
1627
|
-
Event dependency for this flow.
|
|
1628
|
-
events : List[Union[str, Dict[str, Any]]], default []
|
|
1629
|
-
Events dependency for this flow.
|
|
1630
|
-
options : Dict[str, Any], default {}
|
|
1631
|
-
Backend-specific configuration for tuning eventing behavior.
|
|
1632
|
-
"""
|
|
1633
|
-
...
|
|
1634
|
-
|
|
1635
|
-
@typing.overload
|
|
1636
|
-
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1637
|
-
"""
|
|
1638
|
-
Specifies the Conda environment for all steps of the flow.
|
|
1639
|
-
|
|
1640
|
-
Use `@conda_base` to set common libraries required by all
|
|
1641
|
-
steps and use `@conda` to specify step-specific additions.
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
Parameters
|
|
1645
|
-
----------
|
|
1646
|
-
packages : Dict[str, str], default {}
|
|
1647
|
-
Packages to use for this flow. The key is the name of the package
|
|
1648
|
-
and the value is the version to use.
|
|
1649
|
-
libraries : Dict[str, str], default {}
|
|
1650
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1651
|
-
python : str, optional, default None
|
|
1652
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1653
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1654
|
-
disabled : bool, default False
|
|
1655
|
-
If set to True, disables Conda.
|
|
1656
|
-
"""
|
|
1657
|
-
...
|
|
1658
|
-
|
|
1659
|
-
@typing.overload
|
|
1660
|
-
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1661
|
-
...
|
|
1662
|
-
|
|
1663
|
-
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
|
1664
|
-
"""
|
|
1665
|
-
Specifies the Conda environment for all steps of the flow.
|
|
1666
|
-
|
|
1667
|
-
Use `@conda_base` to set common libraries required by all
|
|
1668
|
-
steps and use `@conda` to specify step-specific additions.
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
Parameters
|
|
1672
|
-
----------
|
|
1673
|
-
packages : Dict[str, str], default {}
|
|
1674
|
-
Packages to use for this flow. The key is the name of the package
|
|
1675
|
-
and the value is the version to use.
|
|
1676
|
-
libraries : Dict[str, str], default {}
|
|
1677
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1678
|
-
python : str, optional, default None
|
|
1679
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1680
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1681
|
-
disabled : bool, default False
|
|
1682
|
-
If set to True, disables Conda.
|
|
1683
|
-
"""
|
|
1684
|
-
...
|
|
1685
|
-
|
|
1686
|
-
@typing.overload
|
|
1687
|
-
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1688
|
-
"""
|
|
1689
|
-
Specifies the PyPI packages for all steps of the flow.
|
|
1690
|
-
|
|
1691
|
-
Use `@pypi_base` to set common packages required by all
|
|
1692
|
-
steps and use `@pypi` to specify step-specific overrides.
|
|
1693
|
-
|
|
1694
|
-
Parameters
|
|
1695
|
-
----------
|
|
1696
|
-
packages : Dict[str, str], default: {}
|
|
1697
|
-
Packages to use for this flow. The key is the name of the package
|
|
1698
|
-
and the value is the version to use.
|
|
1699
|
-
python : str, optional, default: None
|
|
1700
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1701
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1622
|
+
|
|
1623
|
+
Parameters
|
|
1624
|
+
----------
|
|
1625
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
|
1626
|
+
Upstream flow dependency for this flow.
|
|
1627
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
|
1628
|
+
Upstream flow dependencies for this flow.
|
|
1629
|
+
options : Dict[str, Any], default {}
|
|
1630
|
+
Backend-specific configuration for tuning eventing behavior.
|
|
1702
1631
|
"""
|
|
1703
1632
|
...
|
|
1704
1633
|
|
|
1705
|
-
|
|
1706
|
-
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1707
|
-
...
|
|
1708
|
-
|
|
1709
|
-
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
|
1634
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1710
1635
|
"""
|
|
1711
|
-
|
|
1636
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
|
1637
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
|
1712
1638
|
|
|
1713
|
-
Use `@pypi_base` to set common packages required by all
|
|
1714
|
-
steps and use `@pypi` to specify step-specific overrides.
|
|
1715
1639
|
|
|
1716
1640
|
Parameters
|
|
1717
1641
|
----------
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1642
|
+
timeout : int
|
|
1643
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
|
1644
|
+
poke_interval : int
|
|
1645
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
|
1646
|
+
mode : str
|
|
1647
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
|
1648
|
+
exponential_backoff : bool
|
|
1649
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
|
1650
|
+
pool : str
|
|
1651
|
+
the slot pool this task should run in,
|
|
1652
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
|
1653
|
+
soft_fail : bool
|
|
1654
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
|
1655
|
+
name : str
|
|
1656
|
+
Name of the sensor on Airflow
|
|
1657
|
+
description : str
|
|
1658
|
+
Description of sensor in the Airflow UI
|
|
1659
|
+
external_dag_id : str
|
|
1660
|
+
The dag_id that contains the task you want to wait for.
|
|
1661
|
+
external_task_ids : List[str]
|
|
1662
|
+
The list of task_ids that you want to wait for.
|
|
1663
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
|
1664
|
+
allowed_states : List[str]
|
|
1665
|
+
Iterable of allowed states, (Default: ['success'])
|
|
1666
|
+
failed_states : List[str]
|
|
1667
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
|
1668
|
+
execution_delta : datetime.timedelta
|
|
1669
|
+
time difference with the previous execution to look at,
|
|
1670
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
|
1671
|
+
check_existence: bool
|
|
1672
|
+
Set to True to check if the external task exists or check if
|
|
1673
|
+
the DAG to wait for exists. (Default: True)
|
|
1724
1674
|
"""
|
|
1725
1675
|
...
|
|
1726
1676
|
|
|
@@ -1838,49 +1788,6 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
|
|
|
1838
1788
|
"""
|
|
1839
1789
|
...
|
|
1840
1790
|
|
|
1841
|
-
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1842
|
-
"""
|
|
1843
|
-
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
|
1844
|
-
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
Parameters
|
|
1848
|
-
----------
|
|
1849
|
-
timeout : int
|
|
1850
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
|
1851
|
-
poke_interval : int
|
|
1852
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
|
1853
|
-
mode : str
|
|
1854
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
|
1855
|
-
exponential_backoff : bool
|
|
1856
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
|
1857
|
-
pool : str
|
|
1858
|
-
the slot pool this task should run in,
|
|
1859
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
|
1860
|
-
soft_fail : bool
|
|
1861
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
|
1862
|
-
name : str
|
|
1863
|
-
Name of the sensor on Airflow
|
|
1864
|
-
description : str
|
|
1865
|
-
Description of sensor in the Airflow UI
|
|
1866
|
-
external_dag_id : str
|
|
1867
|
-
The dag_id that contains the task you want to wait for.
|
|
1868
|
-
external_task_ids : List[str]
|
|
1869
|
-
The list of task_ids that you want to wait for.
|
|
1870
|
-
If None (default value) the sensor waits for the DAG. (Default: None)
|
|
1871
|
-
allowed_states : List[str]
|
|
1872
|
-
Iterable of allowed states, (Default: ['success'])
|
|
1873
|
-
failed_states : List[str]
|
|
1874
|
-
Iterable of failed or dis-allowed states. (Default: None)
|
|
1875
|
-
execution_delta : datetime.timedelta
|
|
1876
|
-
time difference with the previous execution to look at,
|
|
1877
|
-
the default is the same logical date as the current task or DAG. (Default: None)
|
|
1878
|
-
check_existence: bool
|
|
1879
|
-
Set to True to check if the external task exists or check if
|
|
1880
|
-
the DAG to wait for exists. (Default: True)
|
|
1881
|
-
"""
|
|
1882
|
-
...
|
|
1883
|
-
|
|
1884
1791
|
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1885
1792
|
"""
|
|
1886
1793
|
Specifies what flows belong to the same project.
|
|
@@ -1917,53 +1824,146 @@ def project(*, name: str, branch: typing.Optional[str] = None, production: bool
|
|
|
1917
1824
|
...
|
|
1918
1825
|
|
|
1919
1826
|
@typing.overload
|
|
1920
|
-
def
|
|
1827
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1921
1828
|
"""
|
|
1922
|
-
Specifies the
|
|
1923
|
-
|
|
1829
|
+
Specifies the Conda environment for all steps of the flow.
|
|
1830
|
+
|
|
1831
|
+
Use `@conda_base` to set common libraries required by all
|
|
1832
|
+
steps and use `@conda` to specify step-specific additions.
|
|
1924
1833
|
|
|
1925
1834
|
|
|
1926
1835
|
Parameters
|
|
1927
1836
|
----------
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
|
1939
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
|
1837
|
+
packages : Dict[str, str], default {}
|
|
1838
|
+
Packages to use for this flow. The key is the name of the package
|
|
1839
|
+
and the value is the version to use.
|
|
1840
|
+
libraries : Dict[str, str], default {}
|
|
1841
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1842
|
+
python : str, optional, default None
|
|
1843
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1844
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1845
|
+
disabled : bool, default False
|
|
1846
|
+
If set to True, disables Conda.
|
|
1940
1847
|
"""
|
|
1941
1848
|
...
|
|
1942
1849
|
|
|
1943
1850
|
@typing.overload
|
|
1944
|
-
def
|
|
1851
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1945
1852
|
...
|
|
1946
1853
|
|
|
1947
|
-
def
|
|
1854
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
|
1948
1855
|
"""
|
|
1949
|
-
Specifies the
|
|
1950
|
-
|
|
1856
|
+
Specifies the Conda environment for all steps of the flow.
|
|
1857
|
+
|
|
1858
|
+
Use `@conda_base` to set common libraries required by all
|
|
1859
|
+
steps and use `@conda` to specify step-specific additions.
|
|
1951
1860
|
|
|
1952
1861
|
|
|
1953
1862
|
Parameters
|
|
1954
1863
|
----------
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1864
|
+
packages : Dict[str, str], default {}
|
|
1865
|
+
Packages to use for this flow. The key is the name of the package
|
|
1866
|
+
and the value is the version to use.
|
|
1867
|
+
libraries : Dict[str, str], default {}
|
|
1868
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1869
|
+
python : str, optional, default None
|
|
1870
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1871
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1872
|
+
disabled : bool, default False
|
|
1873
|
+
If set to True, disables Conda.
|
|
1874
|
+
"""
|
|
1875
|
+
...
|
|
1876
|
+
|
|
1877
|
+
@typing.overload
|
|
1878
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1879
|
+
"""
|
|
1880
|
+
Specifies the event(s) that this flow depends on.
|
|
1881
|
+
|
|
1882
|
+
```
|
|
1883
|
+
@trigger(event='foo')
|
|
1884
|
+
```
|
|
1885
|
+
or
|
|
1886
|
+
```
|
|
1887
|
+
@trigger(events=['foo', 'bar'])
|
|
1888
|
+
```
|
|
1889
|
+
|
|
1890
|
+
Additionally, you can specify the parameter mappings
|
|
1891
|
+
to map event payload to Metaflow parameters for the flow.
|
|
1892
|
+
```
|
|
1893
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
|
1894
|
+
```
|
|
1895
|
+
or
|
|
1896
|
+
```
|
|
1897
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
|
1898
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
|
1899
|
+
```
|
|
1900
|
+
|
|
1901
|
+
'parameters' can also be a list of strings and tuples like so:
|
|
1902
|
+
```
|
|
1903
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
1904
|
+
```
|
|
1905
|
+
This is equivalent to:
|
|
1906
|
+
```
|
|
1907
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
|
1908
|
+
```
|
|
1909
|
+
|
|
1910
|
+
|
|
1911
|
+
Parameters
|
|
1912
|
+
----------
|
|
1913
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
|
1914
|
+
Event dependency for this flow.
|
|
1915
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
|
1916
|
+
Events dependency for this flow.
|
|
1917
|
+
options : Dict[str, Any], default {}
|
|
1918
|
+
Backend-specific configuration for tuning eventing behavior.
|
|
1919
|
+
"""
|
|
1920
|
+
...
|
|
1921
|
+
|
|
1922
|
+
@typing.overload
|
|
1923
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1924
|
+
...
|
|
1925
|
+
|
|
1926
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
|
1927
|
+
"""
|
|
1928
|
+
Specifies the event(s) that this flow depends on.
|
|
1929
|
+
|
|
1930
|
+
```
|
|
1931
|
+
@trigger(event='foo')
|
|
1932
|
+
```
|
|
1933
|
+
or
|
|
1934
|
+
```
|
|
1935
|
+
@trigger(events=['foo', 'bar'])
|
|
1936
|
+
```
|
|
1937
|
+
|
|
1938
|
+
Additionally, you can specify the parameter mappings
|
|
1939
|
+
to map event payload to Metaflow parameters for the flow.
|
|
1940
|
+
```
|
|
1941
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
|
1942
|
+
```
|
|
1943
|
+
or
|
|
1944
|
+
```
|
|
1945
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
|
1946
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
|
1947
|
+
```
|
|
1948
|
+
|
|
1949
|
+
'parameters' can also be a list of strings and tuples like so:
|
|
1950
|
+
```
|
|
1951
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
1952
|
+
```
|
|
1953
|
+
This is equivalent to:
|
|
1954
|
+
```
|
|
1955
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
|
1956
|
+
```
|
|
1957
|
+
|
|
1958
|
+
|
|
1959
|
+
Parameters
|
|
1960
|
+
----------
|
|
1961
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
|
1962
|
+
Event dependency for this flow.
|
|
1963
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
|
1964
|
+
Events dependency for this flow.
|
|
1965
|
+
options : Dict[str, Any], default {}
|
|
1966
|
+
Backend-specific configuration for tuning eventing behavior.
|
|
1967
1967
|
"""
|
|
1968
1968
|
...
|
|
1969
1969
|
|